* lib/rdoc: Update to 3.9.1. Fixes === lines in verbatim sections.

	  Fixes :nodoc: on class aliases.  Fixes :stopdoc: creating references
	  to Object.  Fixes spacing when class comments are merged in ri.
	  Fixes `ri []` crash.  Fixes bug report URL when rdoc crashes.  Adds
	  :doc: and :nodoc: to allow hiding of implementation details in ruby.
	  Makes `rdoc` and `ri` gem-aware.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_1_9_3@32868 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

20 files changed, 526 insertions, 289 deletions

diff --git a/lib/rdoc.rb b/lib/rdoc.rb
index 187b7d4d51..86c194f8ba 100644
--- a/lib/rdoc.rb
+++ b/lib/rdoc.rb
@@ -104,7 +104,7 @@ module RDoc
   ##
   # RDoc version you are using
 
-  VERSION = '3.8'
+  VERSION = '3.9.1'
 
   ##
   # Method visibilities
diff --git a/lib/rdoc/class_module.rb b/lib/rdoc/class_module.rb
index e104101dcc..27066d8bd7 100644
--- a/lib/rdoc/class_module.rb
+++ b/lib/rdoc/class_module.rb
@@ -222,6 +222,9 @@ class RDoc::ClassModule < RDoc::Context
                    end
   end
 
+  ##
+  # TODO: filter included items by #display?
+
   def marshal_dump # :nodoc:
     attrs = attributes.sort.map do |attr|
       [ attr.name, attr.rw,
diff --git a/lib/rdoc/code_object.rb b/lib/rdoc/code_object.rb
index 606fd8ff49..54826fffbd 100644
--- a/lib/rdoc/code_object.rb
+++ b/lib/rdoc/code_object.rb
@@ -114,6 +114,7 @@ class RDoc::CodeObject
     @done_documenting    = false
     @force_documentation = false
     @received_nodoc      = false
+    @ignored             = false
   end
 
   ##
@@ -140,6 +141,13 @@ class RDoc::CodeObject
   end
 
   ##
+  # Should this CodeObject be shown in documentation?
+
+  def display?
+    @document_self and not @ignored
+  end
+
+  ##
   # Enables or disables documentation of this CodeObject's children unless it
   # has been turned off by :enddoc:
 
@@ -195,6 +203,11 @@ class RDoc::CodeObject
     self
   end
 
+  ##
+  # File name where this CodeObject was found.
+  #
+  # See also RDoc::Context#in_files
+
   def file_name
     return unless @file
 
@@ -221,6 +234,34 @@ class RDoc::CodeObject
   end
 
   ##
+  # Use this to ignore a CodeObject and all its children until found again
+  # (#record_location is called).  An ignored item will not be shown in
+  # documentation.
+  #
+  # See github issue #55
+  #
+  # The ignored status is temporary in order to allow implementation details
+  # to be hidden.  At the end of processing a file RDoc allows all classes
+  # and modules to add new documentation to previously created classes.
+  #
+  # If a class was ignored (via stopdoc) then reopened later with additional
+  # documentation it should be shown.  If a class was ignored and never
+  # reopened it should not be shown.  The ignore flag allows this to occur.
+
+  def ignore
+    @ignored = true
+
+    stop_doc
+  end
+
+  ##
+  # Has this class been ignored?
+
+  def ignored?
+    @ignored
+  end
+
+  ##
   # File name of our parent
 
   def parent_file_name
@@ -238,6 +279,7 @@ class RDoc::CodeObject
   # Records the RDoc::TopLevel (file) where this code object was defined
 
   def record_location top_level
+    @ignored = false
     @file = top_level
   end
 
@@ -250,6 +292,7 @@ class RDoc::CodeObject
 
     @document_self = true
     @document_children = true
+    @ignored = false
   end
 
   ##
diff --git a/lib/rdoc/context.rb b/lib/rdoc/context.rb
index d2552c647c..abdab2026d 100644
--- a/lib/rdoc/context.rb
+++ b/lib/rdoc/context.rb
@@ -423,6 +423,7 @@ class RDoc::Context < RDoc::CodeObject
     if klass then
       # if TopLevel, it may not be registered in the classes:
       enclosing.classes_hash[name] = klass
+
       # update the superclass if needed
       if superclass then
         existing = klass.superclass
@@ -623,8 +624,10 @@ class RDoc::Context < RDoc::CodeObject
 
   ##
   # Is there any content?
-  # This means any of: comment, aliases, methods, attributes,
-  # external aliases, require, constant.
+  #
+  # This means any of: comment, aliases, methods, attributes, external
+  # aliases, require, constant.
+  #
   # Includes are also checked unless <tt>includes == false</tt>.
 
   def any_content(includes = true)
diff --git a/lib/rdoc/cross_reference.rb b/lib/rdoc/cross_reference.rb
new file mode 100644
index 0000000000..adeef2661a
--- /dev/null
+++ b/lib/rdoc/cross_reference.rb
@@ -0,0 +1,173 @@
+##
+# RDoc::CrossReference is a reusable way to create cross references for names.
+
+class RDoc::CrossReference
+
+  ##
+  # Regular expression to match class references
+  #
+  # 1. There can be a '\\' in front of text to suppress the cross-reference
+  # 2. There can be a '::' in front of class names to reference from the
+  #    top-level namespace.
+  # 3. The method can be followed by parenthesis (not recommended)
+
+  CLASS_REGEXP_STR = '\\\\?((?:\:{2})?[A-Z]\w*(?:\:\:\w+)*)'
+
+  ##
+  # Regular expression to match method references.
+  #
+  # See CLASS_REGEXP_STR
+
+  METHOD_REGEXP_STR = '([a-z]\w*[!?=]?)(?:\([\w.+*/=<>-]*\))?'
+
+  ##
+  # Regular expressions matching text that should potentially have
+  # cross-reference links generated are passed to add_special.  Note that
+  # these expressions are meant to pick up text for which cross-references
+  # have been suppressed, since the suppression characters are removed by the
+  # code that is triggered.
+
+  CROSSREF_REGEXP = /(
+                      # A::B::C.meth
+                      #{CLASS_REGEXP_STR}(?:[.#]|::)#{METHOD_REGEXP_STR}
+
+                      # Stand-alone method (preceded by a #)
+                      | \\?\##{METHOD_REGEXP_STR}
+
+                      # Stand-alone method (preceded by ::)
+                      | ::#{METHOD_REGEXP_STR}
+
+                      # A::B::C
+                      # The stuff after CLASS_REGEXP_STR is a
+                      # nasty hack.  CLASS_REGEXP_STR unfortunately matches
+                      # words like dog and cat (these are legal "class"
+                      # names in Fortran 95).  When a word is flagged as a
+                      # potential cross-reference, limitations in the markup
+                      # engine suppress other processing, such as typesetting.
+                      # This is particularly noticeable for contractions.
+                      # In order that words like "can't" not
+                      # be flagged as potential cross-references, only
+                      # flag potential class cross-references if the character
+                      # after the cross-reference is a space, sentence
+                      # punctuation, tag start character, or attribute
+                      # marker.
+                      | #{CLASS_REGEXP_STR}(?=[\s\)\.\?\!\,\;<\000]|\z)
+
+                      # Things that look like filenames
+                      # The key thing is that there must be at least
+                      # one special character (period, slash, or
+                      # underscore).
+                      | (?:\.\.\/)*[-\/\w]+[_\/\.][-\w\/\.]+
+
+                      # Things that have markup suppressed
+                      # Don't process things like '\<' in \<tt>, though.
+                      # TODO: including < is a hack, not very satisfying.
+                      | \\[^\s<]
+                      )/x
+
+  ##
+  # Version of CROSSREF_REGEXP used when <tt>--hyperlink-all</tt> is specified.
+
+  ALL_CROSSREF_REGEXP = /(
+                      # A::B::C.meth
+                      #{CLASS_REGEXP_STR}(?:[.#]|::)#{METHOD_REGEXP_STR}
+
+                      # Stand-alone method
+                      | \\?#{METHOD_REGEXP_STR}
+
+                      # A::B::C
+                      | #{CLASS_REGEXP_STR}(?=[\s\)\.\?\!\,\;<\000]|\z)
+
+                      # Things that look like filenames
+                      | (?:\.\.\/)*[-\/\w]+[_\/\.][-\w\/\.]+
+
+                      # Things that have markup suppressed
+                      | \\[^\s<]
+                      )/x
+
+  attr_accessor :seen
+
+  ##
+  # Allows cross-references to be created based on the given +context+
+  # (RDoc::Context).
+
+  def initialize context
+    @context = context
+
+    @seen = {}
+  end
+
+  ##
+  # Returns a reference to +name+.
+  #
+  # If the reference is found and +name+ is not documented +text+ will be
+  # returned.  If +name+ is escaped +name+ is returned.  If +name+ is not
+  # found +text+ is returned.
+
+  def resolve name, text
+    return @seen[name] if @seen.include? name
+
+    # Find class, module, or method in class or module.
+    #
+    # Do not, however, use an if/elsif/else chain to do so.  Instead, test
+    # each possible pattern until one matches.  The reason for this is that a
+    # string like "YAML.txt" could be the txt() class method of class YAML (in
+    # which case it would match the first pattern, which splits the string
+    # into container and method components and looks up both) or a filename
+    # (in which case it would match the last pattern, which just checks
+    # whether the string as a whole is a known symbol).
+
+    if /#{CLASS_REGEXP_STR}([.#]|::)#{METHOD_REGEXP_STR}/o =~ name then
+      type = $2
+      type = '' if type == '.'  # will find either #method or ::method
+      method = "#{type}#{$3}"
+      container = @context.find_symbol_module($1)
+    elsif /^([.#]|::)#{METHOD_REGEXP_STR}/o =~ name then
+      type = $1
+      type = '' if type == '.'
+      method = "#{type}#{$2}"
+      container = @context
+    else
+      container = nil
+    end
+
+    if container then
+      ref = container.find_local_symbol method
+
+      unless ref || RDoc::TopLevel === container then
+        ref = container.find_ancestor_local_symbol method
+      end
+    end
+
+    ref = case name
+          when /^\\(#{CLASS_REGEXP_STR})$/o then
+            ref = @context.find_symbol $1
+          else
+            ref = @context.find_symbol name
+          end unless ref
+
+    ref = nil if RDoc::Alias === ref # external alias: can't link to it
+
+    out = if name == '\\' then
+            name
+          elsif name =~ /^\\/ then
+            # we remove the \ only in front of what we know:
+            # other backslashes are treated later, only outside of <tt>
+            ref ? $' : name
+          elsif ref then
+            if ref.display? then
+              ref
+            else
+              text
+            end
+          else
+            text
+          end
+
+    @seen[name] = out
+
+    out
+  end
+
+end
+
diff --git a/lib/rdoc/generator/darkfish.rb b/lib/rdoc/generator/darkfish.rb
index 299e6b7dec..a3ffea0ce8 100644
--- a/lib/rdoc/generator/darkfish.rb
+++ b/lib/rdoc/generator/darkfish.rb
@@ -192,7 +192,7 @@ class RDoc::Generator::Darkfish
       top_level = klass.full_name.gsub( /::.*/, '' )
       [nscounts[top_level] * -1, klass.full_name]
     end.select do |klass|
-      klass.document_self
+      klass.display?
     end
   end
 
diff --git a/lib/rdoc/generator/template/darkfish/classpage.rhtml b/lib/rdoc/generator/template/darkfish/classpage.rhtml
index 856321532b..9c74cacf0f 100644
--- a/lib/rdoc/generator/template/darkfish/classpage.rhtml
+++ b/lib/rdoc/generator/template/darkfish/classpage.rhtml
@@ -176,6 +176,8 @@
     </div><!-- description -->
 
     <% klass.each_section do |section, constants, attributes| %>
+    <% constants = constants.select { |const| const.display? } %>
+    <% attributes = attributes.select { |attr| attr.display? } %>
     <div id="<%= section.aref %>" class="documentation-section">
       <% if section.title then %>
       <h2 class="section-header">
diff --git a/lib/rdoc/markup.rb b/lib/rdoc/markup.rb
index 6122fcac65..3dd2459e61 100644
--- a/lib/rdoc/markup.rb
+++ b/lib/rdoc/markup.rb
@@ -269,40 +269,43 @@ require 'rdoc'
 # preceding the first character with a backslash (see <i>Escaping
 # Text Markup</i>, below).
 #
-# === Hyperlinks
+# === Links
 #
-# Hyperlinks to the web starting with +http:+, +mailto:+, +ftp:+ or +www.+
+# Links to starting with +http:+, +https:+, +mailto:+, +ftp:+ or +www.+
 # are recognized.  An HTTP url that references an external image file is
-# converted into an inline <img...>.  Hyperlinks starting with +link:+ are
-# assumed to refer to local files whose path is relative to the <tt>--op</tt>
-# directory.
+# converted into an inline image element.
 #
-# Hyperlinks can also be of the form _label_[_url_], in which
-# case _label_ is used in the displayed text, and _url_ is
-# used as the target.  If _label_ contains multiple words,
-# put it in braces: {<em>multi word label</em>}[url].
+# Links starting with <tt>rdoc-ref:</tt> will link to the referenced class,
+# module, method, file, etc.  If the referenced item is not documented the
+# text will be and no link will be generated.
 #
-# Example hyperlinks:
+# Links starting with +link:+ refer to local files whose path is relative to
+# the <tt>--op</tt> directory.
 #
-#   link:RDoc.html
-#   http://rdoc.rubyforge.org
+# Links can also be of the form <tt>label[url]</tt>, in which case +label+ is
+# used in the displayed text, and +url+ is used as the target.  If +label+
+# contains multiple words, put it in braces: <tt>{multi word label}[url]<tt>.
+#
+# Example links:
+#
+#   https://github.com/rdoc/rdoc
 #   mailto:user@example.com
 #   {RDoc Documentation}[http://rdoc.rubyforge.org]
-#   {RDoc Markup}[link:RDoc/Markup.html]
+#   {RDoc Markup}[rdoc-ref:RDoc::Markup]
 #
 # === Escaping Text Markup
 #
 # Text markup can be escaped with a backslash, as in \<tt>, which was obtained
-# with "<tt>\\<tt></tt>". Except in verbatim sections and between \<tt> tags,
-# to produce a backslash, you have to double it unless it is followed by a
+# with <tt>\\<tt></tt>.  Except in verbatim sections and between \<tt> tags,
+# to produce a backslash you have to double it unless it is followed by a
 # space, tab or newline. Otherwise, the HTML formatter will discard it, as it
-# is used to escape potential hyperlinks:
+# is used to escape potential links:
 #
 #   * The \ must be doubled if not followed by white space: \\.
 #   * But not in \<tt> tags: in a Regexp, <tt>\S</tt> matches non-space.
 #   * This is a link to {ruby-lang}[www.ruby-lang.org].
 #   * This is not a link, however: \{ruby-lang.org}[www.ruby-lang.org].
-#   * This will not be hyperlinked to \RDoc::RDoc#document
+#   * This will not be linked to \RDoc::RDoc#document
 #
 # generates:
 #
@@ -310,16 +313,16 @@ require 'rdoc'
 # * But not in \<tt> tags: in a Regexp, <tt>\S</tt> matches non-space.
 # * This is a link to {ruby-lang}[www.ruby-lang.org]
 # * This is not a link, however: \{ruby-lang.org}[www.ruby-lang.org]
-# * This will not be hyperlinked to \RDoc::RDoc#document
+# * This will not be linked to \RDoc::RDoc#document
 #
-# Inside \<tt> tags, more precisely, leading backslashes are removed
-# only if followed by a markup character (<tt><*_+</tt>), a backslash,
-# or a known hyperlink reference (a known class or method). So in the
-# example above, the backslash of <tt>\S</tt> would be removed
-# if there was a class or module named +S+ in the current context.
+# Inside \<tt> tags, more precisely, leading backslashes are removed only if
+# followed by a markup character (<tt><*_+</tt>), a backslash, or a known link
+# reference (a known class or method). So in the example above, the backslash
+# of <tt>\S</tt> would be removed if there was a class or module named +S+ in
+# the current context.
 #
-# This behavior is inherited from RDoc version 1, and has been kept
-# for compatibility with existing RDoc documentation.
+# This behavior is inherited from RDoc version 1, and has been kept for
+# compatibility with existing RDoc documentation.
 #
 # === Conversion of characters
 #
@@ -378,11 +381,10 @@ require 'rdoc'
 #     # ...
 #   end
 #
-# Names of classes, files, and any method names containing an
-# underscore or preceded by a hash character are automatically hyperlinked
-# from comment text to their description. This hyperlinking works inside
-# the current class or module, and with ancestor methods (in included modules
-# or in the superclass).
+# Names of classes, files, and any method names containing an underscore or
+# preceded by a hash character are automatically linked from comment text to
+# their description. This linking works inside the current class or module,
+# and with ancestor methods (in included modules or in the superclass).
 #
 # Method parameter lists are extracted and displayed with the method
 # description.  If a method calls +yield+, then the parameters passed to yield
diff --git a/lib/rdoc/markup/document.rb b/lib/rdoc/markup/document.rb
index 6fbef33ae3..7077f357d3 100644
--- a/lib/rdoc/markup/document.rb
+++ b/lib/rdoc/markup/document.rb
@@ -71,9 +71,7 @@ class RDoc::Markup::Document
   # Does this document have no parts?
 
   def empty?
-    @parts.empty? or
-      (@parts.length == 1 and RDoc::Markup::Document === @parts.first and
-       @parts.first.empty?)
+    @parts.empty? or (@parts.length == 1 and merged? and @parts.first.empty?)
   end
 
   ##
@@ -85,6 +83,11 @@ class RDoc::Markup::Document
   # The information in +other+ is preferred over the receiver
 
   def merge other
+    if empty? then
+      @parts = other.parts
+      return self
+    end
+
     other.parts.each do |other_part|
       self.parts.delete_if do |self_part|
         self_part.file and self_part.file == other_part.file
@@ -96,6 +99,13 @@ class RDoc::Markup::Document
     self
   end
 
+  ##
+  # Does this Document contain other Documents?
+
+  def merged?
+    RDoc::Markup::Document === @parts.first
+  end
+
   def pretty_print q # :nodoc:
     start = @file ? "[doc (#{@file}): " : '[doc: '
 
diff --git a/lib/rdoc/markup/formatter.rb b/lib/rdoc/markup/formatter.rb
index b6e12f82e7..bf904bba68 100644
--- a/lib/rdoc/markup/formatter.rb
+++ b/lib/rdoc/markup/formatter.rb
@@ -4,6 +4,10 @@ require 'rdoc/markup'
 # Base class for RDoc markup formatters
 #
 # Formatters use a visitor pattern to convert content into output.
+#
+# If you'd like to write your own Formatter use
+# RDoc::Markup::FormatterTestCase.  If you're writing a text-output formatter
+# use RDoc::Markup::TextFormatterTestCase which provides extra test cases.
 
 class RDoc::Markup::Formatter
 
diff --git a/lib/rdoc/markup/parser.rb b/lib/rdoc/markup/parser.rb
index 68616d7787..c18ce821fb 100644
--- a/lib/rdoc/markup/parser.rb
+++ b/lib/rdoc/markup/parser.rb
@@ -86,11 +86,18 @@ class RDoc::Markup::Parser
   # Builds a Heading of +level+
 
   def build_heading level
-    _, text, = get  # TEXT
-    heading = RDoc::Markup::Heading.new level, text
-    skip :NEWLINE
-
-    heading
+    type, text, = get
+
+    text = case type
+           when :TEXT then
+             skip :NEWLINE
+             text
+           else
+             unget
+             ''
+           end
+
+    RDoc::Markup::Heading.new level, text
   end
 
   ##
@@ -405,13 +412,19 @@ class RDoc::Markup::Parser
                    @line += 1
                    token
                  # === text => :HEADER then :TEXT
-                 when s.scan(/(=+)\s*/) then
+                 when s.scan(/(=+)(\s*)/) then
                    level = s[1].length
-                   level = 6 if level > 6
-                   @tokens << [:HEADER, level, *token_pos(pos)]
-                   pos = s.pos
-                   s.scan(/.*/)
-                   [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                   header = [:HEADER, level, *token_pos(pos)]
+
+                   if s[2] =~ /^\r?\n/ then
+                     s.pos -= s[2].length
+                     header
+                   else
+                     pos = s.pos
+                     s.scan(/.*/)
+                     @tokens << header
+                     [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                   end
                  # --- (at least 3) and nothing else on the line => :RULE
                  when s.scan(/(-{3,}) *$/) then
                    [:RULE, s[1].length - 2, *token_pos(pos)]
diff --git a/lib/rdoc/markup/pre_process.rb b/lib/rdoc/markup/pre_process.rb
index 03f919aa0e..53e8e38ec1 100644
--- a/lib/rdoc/markup/pre_process.rb
+++ b/lib/rdoc/markup/pre_process.rb
@@ -13,6 +13,8 @@ require 'rdoc/encoding'
 
 class RDoc::Markup::PreProcess
 
+  attr_accessor :options
+
   @registered = {}
 
   ##
@@ -38,6 +40,7 @@ class RDoc::Markup::PreProcess
   def initialize(input_file_name, include_path)
     @input_file_name = input_file_name
     @include_path = include_path
+    @options = nil
   end
 
   ##
@@ -54,54 +57,120 @@ class RDoc::Markup::PreProcess
   # If +code_object+ is given and the param is set as metadata on the
   # +code_object+.  See RDoc::CodeObject#metadata
 
-  def handle text, code_object = nil
+  def handle text, code_object = nil, &block
+    encoding = if defined?(Encoding) then text.encoding else nil end
     # regexp helper (square brackets for optional)
     # $1      $2  $3        $4      $5
     # [prefix][\]:directive:[spaces][param]newline
-    text.gsub!(/^([ \t]*#?[ \t]*)(\\?):(\w+):([ \t]*)(.+)?\n/) do
+    text.gsub!(/^([ \t]*(?:#|\/?\*)?[ \t]*)(\\?):(\w+):([ \t]*)(.+)?\n/) do
       # skip something like ':toto::'
       next $& if $4.empty? and $5 and $5[0, 1] == ':'
 
       # skip if escaped
       next "#$1:#$3:#$4#$5\n" unless $2.empty?
 
-      prefix    = $1
-      directive = $3.downcase
-      param     = $5
-
-      case directive
-      when 'include' then
-        filename = param.split[0]
-        encoding = if defined?(Encoding) then text.encoding else nil end
-        include_file filename, prefix, encoding
-      when 'category' then
-        if RDoc::Context === code_object then
-          section = code_object.add_section param, ''
-          code_object.temporary_section = section
-        end
+      handle_directive $1, $3, $5, code_object, encoding, &block
+    end
+
+    text
+  end
+
+  #--
+  # When 1.8.7 support is ditched prefix can be defaulted to ''
+
+  def handle_directive prefix, directive, param, code_object = nil,
+                       encoding = nil
+    blankline = "#{prefix.strip}\n"
+    directive = directive.downcase
+
+    case directive
+    when 'arg', 'args' then
+      return blankline unless code_object
+
+      code_object.params = param
+
+      blankline
+    when 'category' then
+      if RDoc::Context === code_object then
+        section = code_object.add_section param, ''
+        code_object.temporary_section = section
+      end
+
+      blankline # ignore category if we're not on an RDoc::Context
+    when 'doc' then
+      return blankline unless code_object
+      code_object.document_self = true
+      code_object.force_documentation = true
+
+      blankline
+    when 'enddoc' then
+      return blankline unless code_object
+      code_object.done_documenting = true
+
+      blankline
+    when 'include' then
+      filename = param.split.first
+      include_file filename, prefix, encoding
+    when 'main' then
+      @options.main_page = param if @options.respond_to? :main_page
+
+      blankline
+    when 'nodoc' then
+      return blankline unless code_object
+      code_object.document_self = nil # notify nodoc
+      code_object.document_children = param !~ /all/i
 
-        '' # ignore category if we're not on an RDoc::Context
-      else
-        result = yield directive, param if block_given?
-
-        case result
-        when nil then
-          code_object.metadata[directive] = param if code_object
-          if RDoc::Markup::PreProcess.registered.include? directive then
-            handler = RDoc::Markup::PreProcess.registered[directive]
-            result = handler.call directive, param if handler
-          else
-            result = "#{prefix}:#{directive}: #{param}\n"
-          end
-        when false then
+      blankline
+    when 'notnew', 'not_new', 'not-new' then
+      return blankline unless RDoc::AnyMethod === code_object
+
+      code_object.dont_rename_initialize = true
+
+      blankline
+    when 'startdoc' then
+      return blankline unless code_object
+
+      code_object.start_doc
+      code_object.force_documentation = true
+
+      blankline
+    when 'stopdoc' then
+      return blankline unless code_object
+
+      code_object.stop_doc
+
+      blankline
+    when 'title' then
+      @options.default_title = param if @options.respond_to? :default_title=
+
+      blankline
+    when 'yield', 'yields' then
+      return blankline unless code_object
+      # remove parameter &block
+      code_object.params.sub!(/,?\s*&\w+/, '') if code_object.params
+
+      code_object.block_params = param
+
+      blankline
+    else
+      result = yield directive, param if block_given?
+
+      case result
+      when nil then
+        code_object.metadata[directive] = param if code_object
+
+        if RDoc::Markup::PreProcess.registered.include? directive then
+          handler = RDoc::Markup::PreProcess.registered[directive]
+          result = handler.call directive, param if handler
+        else
           result = "#{prefix}:#{directive}: #{param}\n"
         end
-
-        result
+      when false then
+        result = "#{prefix}:#{directive}: #{param}\n"
       end
-    end
 
-    text
+      result
+    end
   end
 
   ##
diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb
index b518fbf265..f87aabad6f 100644
--- a/lib/rdoc/markup/to_html.rb
+++ b/lib/rdoc/markup/to_html.rb
@@ -70,7 +70,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     @list = nil
     @from_path = ''
 
-    # external hyperlinks
+    # external links
     @markup.add_special(/((link:|https?:|mailto:|ftp:|www\.)\S+\w)/, :HYPERLINK)
 
     # and links of the form  <text>[<url>]
@@ -84,7 +84,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # These methods handle special markup added by RDoc::Markup#add_special.
 
   ##
-  # +special+ is a potential hyperlink.  The following schemes are handled:
+  # +special+ is a potential link.  The following schemes are handled:
   #
   # <tt>mailto:</tt>::
   #   Inserted as-is.
@@ -97,12 +97,13 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
   def handle_special_HYPERLINK(special)
     url = special.text
+
     gen_url url, url
   end
 
   ##
-  # This +special+ is a hyperlink where the label is different from the URL
-  # label[url] or {long label}[url]
+  # This +special+ is a link where the label is different from the URL
+  # <tt>label[url]</tt> or <tt>{long label}[url]</tt>
 
   def handle_special_TIDYLINK(special)
     text = special.text
@@ -232,8 +233,8 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   end
 
   ##
-  # Generate a hyperlink for +url+, labeled with +text+.  Handles the special
-  # cases for img: and link: described under handle_special_HYPERLINK
+  # Generate a link for +url+, labeled with +text+.  Handles the special cases
+  # for img: and link: described under handle_special_HYPERLINK
 
   def gen_url(url, text)
     if url =~ /([A-Za-z]+):(.*)/ then
diff --git a/lib/rdoc/markup/to_html_crossref.rb b/lib/rdoc/markup/to_html_crossref.rb
index aeecc3f4d0..450ea960b5 100644
--- a/lib/rdoc/markup/to_html_crossref.rb
+++ b/lib/rdoc/markup/to_html_crossref.rb
@@ -1,92 +1,19 @@
 require 'rdoc/markup/to_html'
+require 'rdoc/cross_reference'
 
 ##
-# Subclass of the RDoc::Markup::ToHtml class that supports looking up words
-# from a context.  Those that are found will be hyperlinked.
+# Subclass of the RDoc::Markup::ToHtml class that supports looking up method
+# names, classes, etc to create links.  RDoc::CrossReference is used to
+# generate those links based on the current context.
 
 class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
 
-  ##
-  # Regular expression to match class references
-  #
-  # 1. There can be a '\\' in front of text to suppress the cross-reference
-  # 2. There can be a '::' in front of class names to reference from the
-  #    top-level namespace.
-  # 3. The method can be followed by parenthesis (not recommended)
-
-  CLASS_REGEXP_STR = '\\\\?((?:\:{2})?[A-Z]\w*(?:\:\:\w+)*)'
-
-  ##
-  # Regular expression to match method references.
-  #
-  # See CLASS_REGEXP_STR
-
-  METHOD_REGEXP_STR = '([a-z]\w*[!?=]?)(?:\([\w.+*/=<>-]*\))?'
-
-  ##
-  # Regular expressions matching text that should potentially have
-  # cross-reference links generated are passed to add_special.  Note that
-  # these expressions are meant to pick up text for which cross-references
-  # have been suppressed, since the suppression characters are removed by the
-  # code that is triggered.
-
-  CROSSREF_REGEXP = /(
-                      # A::B::C.meth
-                      #{CLASS_REGEXP_STR}(?:[.#]|::)#{METHOD_REGEXP_STR}
-
-                      # Stand-alone method (preceded by a #)
-                      | \\?\##{METHOD_REGEXP_STR}
-
-                      # Stand-alone method (preceded by ::)
-                      | ::#{METHOD_REGEXP_STR}
-
-                      # A::B::C
-                      # The stuff after CLASS_REGEXP_STR is a
-                      # nasty hack.  CLASS_REGEXP_STR unfortunately matches
-                      # words like dog and cat (these are legal "class"
-                      # names in Fortran 95).  When a word is flagged as a
-                      # potential cross-reference, limitations in the markup
-                      # engine suppress other processing, such as typesetting.
-                      # This is particularly noticeable for contractions.
-                      # In order that words like "can't" not
-                      # be flagged as potential cross-references, only
-                      # flag potential class cross-references if the character
-                      # after the cross-reference is a space, sentence
-                      # punctuation, tag start character, or attribute
-                      # marker.
-                      | #{CLASS_REGEXP_STR}(?=[\s\)\.\?\!\,\;<\000]|\z)
-
-                      # Things that look like filenames
-                      # The key thing is that there must be at least
-                      # one special character (period, slash, or
-                      # underscore).
-                      | (?:\.\.\/)*[-\/\w]+[_\/\.][-\w\/\.]+
-
-                      # Things that have markup suppressed
-                      # Don't process things like '\<' in \<tt>, though.
-                      # TODO: including < is a hack, not very satisfying.
-                      | \\[^\s<]
-                      )/x
-
-  ##
-  # Version of CROSSREF_REGEXP used when <tt>--hyperlink-all</tt> is specified.
-
-  ALL_CROSSREF_REGEXP = /(
-                      # A::B::C.meth
-                      #{CLASS_REGEXP_STR}(?:[.#]|::)#{METHOD_REGEXP_STR}
-
-                      # Stand-alone method
-                      | \\?#{METHOD_REGEXP_STR}
-
-                      # A::B::C
-                      | #{CLASS_REGEXP_STR}(?=[\s\)\.\?\!\,\;<\000]|\z)
-
-                      # Things that look like filenames
-                      | (?:\.\.\/)*[-\/\w]+[_\/\.][-\w\/\.]+
-
-                      # Things that have markup suppressed
-                      | \\[^\s<]
-                      )/x
+  # :stopdoc:
+  ALL_CROSSREF_REGEXP = RDoc::CrossReference::ALL_CROSSREF_REGEXP
+  CLASS_REGEXP_STR    = RDoc::CrossReference::CLASS_REGEXP_STR
+  CROSSREF_REGEXP     = RDoc::CrossReference::CROSSREF_REGEXP
+  METHOD_REGEXP_STR   = RDoc::CrossReference::METHOD_REGEXP_STR
+  # :startdoc:
 
   ##
   # RDoc::CodeObject for generating references
@@ -102,7 +29,7 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
   # Creates a new crossref resolver that generates links relative to +context+
   # which lives at +from_path+ in the generated files.  '#' characters on
   # references are removed unless +show_hash+ is true.  Only method names
-  # preceded by '#' or '::' are hyperlinked, unless +hyperlink_all+ is true.
+  # preceded by '#' or '::' are linked, unless +hyperlink_all+ is true.
 
   def initialize(from_path, context, show_hash, hyperlink_all = false,
                  markup = nil)
@@ -111,22 +38,36 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
 
     crossref_re = hyperlink_all ? ALL_CROSSREF_REGEXP : CROSSREF_REGEXP
 
+    @cross_reference = RDoc::CrossReference.new context
+
     @markup.add_special crossref_re, :CROSSREF
+    @markup.add_special(/rdoc-ref:\S+\w/, :HYPERLINK)
 
-    @from_path = from_path
-    @context = context
-    @show_hash = show_hash
+    @from_path     = from_path
     @hyperlink_all = hyperlink_all
+    @show_hash     = show_hash
+  end
+
+  ##
+  # Creates a link to the reference +name+ if the name exists.  If +text+ is
+  # given it is used as the link text, otherwise +name+ is used.
+
+  def cross_reference name, text = nil
+    lookup = name
 
-    @seen = {}
+    name = name[1..-1] unless @show_hash if name[0, 1] == '#'
+
+    text = name unless text
+
+    link lookup, text
   end
 
   ##
   # We're invoked when any text matches the CROSSREF pattern.  If we find the
-  # corresponding reference, generate a hyperlink.  If the name we're looking
-  # for contains no punctuation, we look for it up the module/class chain.
-  # For example, ToHtml is found, even without the <tt>RDoc::Markup::</tt>
-  # prefix, because we look for it in module Markup first.
+  # corresponding reference, generate a link.  If the name we're looking for
+  # contains no punctuation, we look for it up the module/class chain.  For
+  # example, ToHtml is found, even without the <tt>RDoc::Markup::</tt> prefix,
+  # because we look for it in module Markup first.
 
   def handle_special_CROSSREF(special)
     name = special.text
@@ -138,66 +79,41 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
       return name if name =~ /\A[a-z]*\z/
     end
 
-    return @seen[name] if @seen.include? name
+    cross_reference name
+  end
 
-    lookup = name
+  ##
+  # Handles <tt>rdoc-ref:</tt> scheme links and allows RDoc::Markup::ToHtml to
+  # handle other schemes.
 
-    name = name[1..-1] unless @show_hash if name[0, 1] == '#'
+  def handle_special_HYPERLINK special
+    return cross_reference $' if special.text =~ /\Ardoc-ref:/
 
-    # Find class, module, or method in class or module.
-    #
-    # Do not, however, use an if/elsif/else chain to do so.  Instead, test
-    # each possible pattern until one matches.  The reason for this is that a
-    # string like "YAML.txt" could be the txt() class method of class YAML (in
-    # which case it would match the first pattern, which splits the string
-    # into container and method components and looks up both) or a filename
-    # (in which case it would match the last pattern, which just checks
-    # whether the string as a whole is a known symbol).
-
-    if /#{CLASS_REGEXP_STR}([.#]|::)#{METHOD_REGEXP_STR}/ =~ lookup then
-      type = $2
-      type = '' if type == '.'  # will find either #method or ::method
-      method = "#{type}#{$3}"
-      container = @context.find_symbol_module($1)
-    elsif /^([.#]|::)#{METHOD_REGEXP_STR}/ =~ lookup then
-      type = $1
-      type = '' if type == '.'
-      method = "#{type}#{$2}"
-      container = @context
-    else
-      container = nil
-    end
+    super
+  end
 
-    if container then
-      ref = container.find_local_symbol method
+  ##
+  # Generates links for <tt>rdoc-ref:</tt> scheme URLs and allows
+  # RDoc::Markup::ToHtml to handle other schemes.
 
-      unless ref || RDoc::TopLevel === container then
-        ref = container.find_ancestor_local_symbol method
-      end
-    end
+  def gen_url url, text
+    return super unless url =~ /\Ardoc-ref:/
 
-    ref = @context.find_symbol lookup unless ref
-    ref = nil if RDoc::Alias === ref # external alias: can't link to it
-
-    out = if lookup == '\\' then
-            lookup
-          elsif lookup =~ /^\\/ then
-            # we remove the \ only in front of what we know:
-            # other backslashes are treated later, only outside of <tt>
-            ref ? $' : lookup
-          elsif ref then
-            if ref.document_self then
-              "<a href=\"#{ref.as_href @from_path}\">#{name}</a>"
-            else
-              name
-            end
-          else
-            lookup
-          end
-
-    @seen[lookup] = out
-
-    out
+    cross_reference $', text
+  end
+
+  ##
+  # Creates an HTML link to +name+ with the given +text+.
+
+  def link name, text
+    ref = @cross_reference.resolve name, text
+
+    case ref
+    when String then
+      ref
+    else
+      "<a href=\"#{ref.as_href @from_path}\">#{text}</a>"
+    end
   end
 
 end
diff --git a/lib/rdoc/parser.rb b/lib/rdoc/parser.rb
index b2559fa3a9..ba7b858595 100644
--- a/lib/rdoc/parser.rb
+++ b/lib/rdoc/parser.rb
@@ -106,6 +106,8 @@ class RDoc::Parser
   # Applies +directive+'s +value+ to +code_object+, if appropriate
 
   def self.process_directive code_object, directive, value
+    warn "RDoc::Parser::process_directive is deprecated and wil be removed in RDoc 4.  Use RDoc::Markup::PreProcess#handle_directive instead" if $-w
+
     case directive
     when 'nodoc' then
       code_object.document_self = nil # notify nodoc
@@ -196,6 +198,9 @@ class RDoc::Parser
     @content = content
     @options = options
     @stats = stats
+
+    @preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
+    @preprocess.options = @options
   end
 
 end
diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
index a0282d69f1..3da1820c50 100644
--- a/lib/rdoc/parser/c.rb
+++ b/lib/rdoc/parser/c.rb
@@ -645,9 +645,7 @@ class RDoc::Parser::C < RDoc::Parser
       meth_obj.call_seq = $1.strip
     end
 
-    if comment.sub!(/\s*:(nodoc|doc|yields?|args?):\s*(.*)/, '') then
-      RDoc::Parser.process_directive meth_obj, $1, $2
-    end
+    look_for_directives_in meth_obj, comment
   end
 
   ##
@@ -913,12 +911,10 @@ class RDoc::Parser::C < RDoc::Parser
   #    * :title: My Awesome Project
   #    */
   #
-  # This routine modifies its parameter
-
-  def look_for_directives_in(context, comment)
-    preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
+  # This method modifies the +comment+
 
-    preprocess.handle comment, context do |directive, param|
+  def look_for_directives_in context, comment
+    @preprocess.handle comment, context do |directive, param|
       case directive
       when 'main' then
         @options.main_page = param
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
index 8b7c9c3eff..c9a12a8fe8 100644
--- a/lib/rdoc/parser/ruby.rb
+++ b/lib/rdoc/parser/ruby.rb
@@ -405,17 +405,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   #
   # This routine modifies its +comment+ parameter.
 
-  def look_for_directives_in(context, comment)
-    preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
-
-    preprocess.handle comment, context do |directive, param|
+  def look_for_directives_in context, comment
+    @preprocess.handle comment, context do |directive, param|
       case directive
-      when 'enddoc' then
-        context.done_documenting = true
-        ''
-      when 'main' then
-        @options.main_page = param if @options.respond_to? :main_page
-        ''
       when 'method', 'singleton-method',
            'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
         false # handled elsewhere
@@ -423,16 +415,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
         context.set_current_section param, comment
         comment.replace ''
         break
-      when 'startdoc' then
-        context.start_doc
-        context.force_documentation = true
-        ''
-      when 'stopdoc' then
-        context.stop_doc
-        ''
-      when 'title' then
-        @options.default_title = param if @options.respond_to? :default_title=
-        ''
       end
     end
 
@@ -629,6 +611,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
       cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
       cls = declaration_context.add_class cls_type, given_name, superclass
+      cls.ignore unless container.document_children
 
       read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
       cls.record_location @top_level
@@ -679,7 +662,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   ##
   # Parses a constant in +context+ with +comment+
 
-  def parse_constant(container, tk, comment)
+  def parse_constant container, tk, comment
     offset  = tk.seek
     line_no = tk.line_no
 
@@ -718,7 +701,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
       when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
         nest -= 1
       when TkCOMMENT then
-        if nest <= 0 && @scanner.lex_state == EXPR_END
+        if nest <= 0 &&
+           (@scanner.lex_state == EXPR_END || !@scanner.continue) then
           unget_tk tk
           break
         end
@@ -733,7 +717,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
                 end
 
           container.add_module_alias mod, name, @top_level if mod
-          get_tk # TkNL
           break
         end
       when TkNL then
@@ -1327,11 +1310,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         keep_comment = true
 
       when TkCLASS then
-        if container.document_children then
-          parse_class container, single, tk, comment
-        else
-          nest += 1
-        end
+        parse_class container, single, tk, comment
 
       when TkMODULE then
         if container.document_children then
@@ -1516,11 +1495,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
   ##
   # Parses statements in the top-level +container+
 
-  def parse_top_level_statements(container)
+  def parse_top_level_statements container
     comment = collect_first_comment
-    look_for_directives_in(container, comment)
+    look_for_directives_in container, comment
+
     # HACK move if to RDoc::Context#comment=
     container.comment = comment if container.document_self unless comment.empty?
+
     parse_statements container, NORMAL, nil, comment
   end
 
@@ -1643,16 +1624,17 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Handles the directive for +context+ if the directive is listed in +allow+.
   # This method is called for directives following a definition.
 
-  def read_documentation_modifiers(context, allow)
+  def read_documentation_modifiers context, allow
     directive, value = read_directive allow
 
     return unless directive
 
-    case directive
-    when 'notnew', 'not_new', 'not-new' then
-      context.dont_rename_initialize = true
-    else
-      RDoc::Parser.process_directive context, directive, value
+    @preprocess.handle_directive '', directive, value, context do |dir, param|
+      if %w[notnew not_new not-new].include? dir then
+        context.dont_rename_initialize = true
+
+        true
+      end
     end
   end
 
diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb
index 3f6190884e..678f721624 100644
--- a/lib/rdoc/parser/ruby_tools.rb
+++ b/lib/rdoc/parser/ruby_tools.rb
@@ -153,6 +153,8 @@ module RDoc::Parser::RubyTools
     @token_listeners.each do |obj|
       obj.pop_token
     end if @token_listeners
+
+    nil
   end
 
 end
diff --git a/lib/rdoc/ri/driver.rb b/lib/rdoc/ri/driver.rb
index a99f96cb56..26304dca96 100644
--- a/lib/rdoc/ri/driver.rb
+++ b/lib/rdoc/ri/driver.rb
@@ -387,6 +387,8 @@ Options may also be set in the 'RI' environment variable.
                   klass.superclass unless klass.module?
                 end.compact.shift || 'Object'
 
+                superclass = superclass.full_name unless String === superclass
+
                 "#{name} < #{superclass}"
               end
 
@@ -451,7 +453,7 @@ Options may also be set in the 'RI' environment variable.
   # Adds a list of +methods+ to +out+ with a heading of +name+
 
   def add_method_list out, methods, name
-    return unless methods
+    return if methods.empty?
 
     out << RDoc::Markup::Heading.new(1, "#{name}:")
     out << RDoc::Markup::BlankLine.new
@@ -518,11 +520,13 @@ Options may also be set in the 'RI' environment variable.
 
     found.each do |store, klass|
       comment = klass.comment
-      class_methods    = store.class_methods[klass.full_name]
-      instance_methods = store.instance_methods[klass.full_name]
-      attributes       = store.attributes[klass.full_name]
+      # TODO the store's cache should always return an empty Array
+      class_methods    = store.class_methods[klass.full_name]    || []
+      instance_methods = store.instance_methods[klass.full_name] || []
+      attributes       = store.attributes[klass.full_name]       || []
 
-      if comment.empty? and !(instance_methods or class_methods) then
+      if comment.empty? and
+         instance_methods.empty? and class_methods.empty? then
         also_in << store
         next
       end
@@ -531,7 +535,17 @@ Options may also be set in the 'RI' environment variable.
 
       unless comment.empty? then
         out << RDoc::Markup::Rule.new(1)
-        out << comment
+
+        if comment.merged? then
+          parts = comment.parts
+          parts = parts.zip [RDoc::Markup::BlankLine.new] * parts.length
+          parts.flatten!
+          parts.pop
+
+          out.push(*parts)
+        else
+          out << comment
+        end
       end
 
       if class_methods or instance_methods or not klass.constants.empty? then
@@ -554,13 +568,12 @@ Options may also be set in the 'RI' environment variable.
         end)
 
         out << list
+        out << RDoc::Markup::BlankLine.new
       end
 
       add_method_list out, class_methods,    'Class methods'
       add_method_list out, instance_methods, 'Instance methods'
       add_method_list out, attributes,       'Attributes'
-
-      out << RDoc::Markup::BlankLine.new
     end
 
     add_also_in out, also_in
@@ -1090,11 +1103,11 @@ Options may also be set in the 'RI' environment variable.
   # NOTE: Given Foo::Bar, Bar is considered a class even though it may be a
   #       method
 
-  def parse_name(name)
+  def parse_name name
     parts = name.split(/(::|#|\.)/)
 
     if parts.length == 1 then
-      if parts.first =~ /^[a-z]/ then
+      if parts.first =~ /^[a-z]|^([%&*+\/<>^`|~-]|\+@|-@|<<|<=>?|===?|=>|=~|>>|\[\]=?|~@)$/ then
         type = '.'
         meth = parts.pop
       else
diff --git a/lib/rdoc/text.rb b/lib/rdoc/text.rb
index aec334b545..3ac55ed560 100644
--- a/lib/rdoc/text.rb
+++ b/lib/rdoc/text.rb
@@ -126,7 +126,7 @@ Ruby #{RUBY_VERSION}-p#{RUBY_PATCHLEVEL} #{RUBY_RELEASE_DATE}
 
 Please file a bug report with the above information at:
 
-http://rubyforge.org/tracker/?atid=2472&group_id=627&func=browse
+https://github.com/rdoc/rdoc/issues
 
     EOF
     raise