* lib/rdoc*: Updated to RDoc 4.0 (pre-release)

* bin/rdoc:  ditto
* test/rdoc:  ditto
* NEWS:  Updated with RDoc 4.0 information


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@37889 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

34 files changed, 1664 insertions, 376 deletions

diff --git a/lib/rdoc/markup/attr_changer.rb b/lib/rdoc/markup/attr_changer.rb
new file mode 100644
index 0000000000..1772f18b2b
--- /dev/null
+++ b/lib/rdoc/markup/attr_changer.rb
@@ -0,0 +1,22 @@
+class RDoc::Markup
+
+  AttrChanger = Struct.new :turn_on, :turn_off # :nodoc:
+
+end
+
+##
+# An AttrChanger records a change in attributes. It contains a bitmap of the
+# attributes to turn on, and a bitmap of those to turn off.
+
+class RDoc::Markup::AttrChanger
+
+  def to_s # :nodoc:
+    "Attr: +#{turn_on}/-#{turn_off}"
+  end
+
+  def inspect # :nodoc:
+    '+%d/-%d' % [turn_on, turn_off]
+  end
+
+end
+
diff --git a/lib/rdoc/markup/attr_span.rb b/lib/rdoc/markup/attr_span.rb
new file mode 100644
index 0000000000..b5c1b3b7b7
--- /dev/null
+++ b/lib/rdoc/markup/attr_span.rb
@@ -0,0 +1,29 @@
+##
+# An array of attributes which parallels the characters in a string.
+
+class RDoc::Markup::AttrSpan
+
+  ##
+  # Creates a new AttrSpan for +length+ characters
+
+  def initialize(length)
+    @attrs = Array.new(length, 0)
+  end
+
+  ##
+  # Toggles +bits+ from +start+ to +length+
+  def set_attrs(start, length, bits)
+    for i in start ... (start+length)
+      @attrs[i] |= bits
+    end
+  end
+
+  ##
+  # Accesses flags for character +n+
+
+  def [](n)
+    @attrs[n]
+  end
+
+end
+
diff --git a/lib/rdoc/markup/attribute_manager.rb b/lib/rdoc/markup/attribute_manager.rb
index d2402f1b1d..71d5e2b2cc 100644
--- a/lib/rdoc/markup/attribute_manager.rb
+++ b/lib/rdoc/markup/attribute_manager.rb
@@ -23,6 +23,11 @@ class RDoc::Markup::AttributeManager
   PROTECT_ATTR = A_PROTECT.chr # :nodoc:
 
   ##
+  # The attributes enabled for this markup object.
+
+  attr_reader :attributes
+
+  ##
   # This maps delimiters that occur around words (such as *bold* or +tt+)
   # where the start and end delimiters and the same. This lets us optimize
   # the regexp
@@ -60,8 +65,9 @@ class RDoc::Markup::AttributeManager
     @html_tags = {}
     @matching_word_pairs = {}
     @protectable = %w[<]
-    @special = {}
+    @special = []
     @word_pair_map = {}
+    @attributes = RDoc::Markup::Attributes.new
 
     add_word_pair "*", "*", :BOLD
     add_word_pair "_", "_", :EM
@@ -96,11 +102,11 @@ class RDoc::Markup::AttributeManager
   def changed_attribute_by_name current_set, new_set
     current = new = 0
     current_set.each do |name|
-      current |= RDoc::Markup::Attribute.bitmap_for(name)
+      current |= @attributes.bitmap_for(name)
     end
 
     new_set.each do |name|
-      new |= RDoc::Markup::Attribute.bitmap_for(name)
+      new |= @attributes.bitmap_for(name)
     end
 
     change_attribute(current, new)
@@ -161,12 +167,15 @@ class RDoc::Markup::AttributeManager
   ##
   # Converts special sequences to RDoc attributes
 
-  def convert_specials(str, attrs)
+  def convert_specials str, attrs
     unless @special.empty?
-      @special.each do |regexp, attr|
+      @special.each do |regexp, attribute|
         str.scan(regexp) do
-          attrs.set_attrs($`.length, $&.length,
-                          attr | RDoc::Markup::Attribute::SPECIAL)
+          capture = $~.size == 1 ? 0 : 1
+
+          s, e = $~.offset capture
+
+          attrs.set_attrs s, e - s, attribute | @attributes.special
         end
       end
     end
@@ -200,7 +209,7 @@ class RDoc::Markup::AttributeManager
     raise ArgumentError, "Word flags may not start with '<'" if
       start[0,1] == '<'
 
-    bitmap = RDoc::Markup::Attribute.bitmap_for name
+    bitmap = @attributes.bitmap_for name
 
     if start == stop then
       @matching_word_pairs[start] = bitmap
@@ -220,7 +229,7 @@ class RDoc::Markup::AttributeManager
   #   am.add_html 'em', :EM
 
   def add_html(tag, name)
-    @html_tags[tag.downcase] = RDoc::Markup::Attribute.bitmap_for name
+    @html_tags[tag.downcase] = @attributes.bitmap_for name
   end
 
   ##
@@ -229,14 +238,14 @@ class RDoc::Markup::AttributeManager
   #
   #   @am.add_special(/((https?:)\S+\w)/, :HYPERLINK)
 
-  def add_special(pattern, name)
-    @special[pattern] = RDoc::Markup::Attribute.bitmap_for name
+  def add_special pattern, name
+    @special << [pattern, @attributes.bitmap_for(name)]
   end
 
   ##
   # Processes +str+ converting attributes, HTML and specials
 
-  def flow(str)
+  def flow str
     @str = str
 
     mask_protected_sequences
@@ -303,9 +312,9 @@ class RDoc::Markup::AttributeManager
         res << change_attribute(current_attr, new_attr)
         current_attr = new_attr
 
-        if (current_attr & RDoc::Markup::Attribute::SPECIAL) != 0 then
+        if (current_attr & @attributes.special) != 0 then
           i += 1 while
-            i < str_len and (@attrs[i] & RDoc::Markup::Attribute::SPECIAL) != 0
+            i < str_len and (@attrs[i] & @attributes.special) != 0
 
           res << RDoc::Markup::Special.new(current_attr,
                                            copy_string(start_pos, i))
diff --git a/lib/rdoc/markup/attributes.rb b/lib/rdoc/markup/attributes.rb
new file mode 100644
index 0000000000..3423f10ca7
--- /dev/null
+++ b/lib/rdoc/markup/attributes.rb
@@ -0,0 +1,70 @@
+##
+# We manage a set of attributes.  Each attribute has a symbol name and a bit
+# value.
+
+class RDoc::Markup::Attributes
+
+  ##
+  # The special attribute type.  See RDoc::Markup#add_special
+
+  attr_reader :special
+
+  ##
+  # Creates a new attributes set.
+
+  def initialize
+    @special = 1
+
+    @name_to_bitmap = [
+      [:_SPECIAL_, @special],
+    ]
+
+    @next_bitmap = @special << 1
+  end
+
+  ##
+  # Returns a unique bit for +name+
+
+  def bitmap_for name
+    bitmap = @name_to_bitmap.assoc name
+
+    unless bitmap then
+      bitmap = @next_bitmap
+      @next_bitmap <<= 1
+      @name_to_bitmap << [name, bitmap]
+    else
+      bitmap = bitmap.last
+    end
+
+    bitmap
+  end
+
+  ##
+  # Returns a string representation of +bitmap+
+
+  def as_string bitmap
+    return 'none' if bitmap.zero?
+    res = []
+
+    @name_to_bitmap.each do |name, bit|
+      res << name if (bitmap & bit) != 0
+    end
+
+    res.join ','
+  end
+
+  ##
+  # yields each attribute name in +bitmap+
+
+  def each_name_of bitmap
+    return enum_for __method__, bitmap unless block_given?
+
+    @name_to_bitmap.each do |name, bit|
+      next if bit == @special
+
+      yield name.to_s if (bitmap & bit) != 0
+    end
+  end
+
+end
+
diff --git a/lib/rdoc/markup/block_quote.rb b/lib/rdoc/markup/block_quote.rb
new file mode 100644
index 0000000000..552f0c4baa
--- /dev/null
+++ b/lib/rdoc/markup/block_quote.rb
@@ -0,0 +1,14 @@
+##
+# A quoted section which contains markup items.
+
+class RDoc::Markup::BlockQuote < RDoc::Markup::Raw
+
+  ##
+  # Calls #accept_block_quote on +visitor+
+
+  def accept visitor
+    visitor.accept_block_quote self
+  end
+
+end
+
diff --git a/lib/rdoc/markup/document.rb b/lib/rdoc/markup/document.rb
index 7077f357d3..198cef9ed9 100644
--- a/lib/rdoc/markup/document.rb
+++ b/lib/rdoc/markup/document.rb
@@ -3,11 +3,13 @@
 
 class RDoc::Markup::Document
 
+  include Enumerable
+
   ##
   # The file this document was created from.  See also
   # RDoc::ClassModule#add_comment
 
-  attr_accessor :file
+  attr_reader :file
 
   ##
   # The parts of the Document
@@ -19,7 +21,7 @@ class RDoc::Markup::Document
 
   def initialize *parts
     @parts = []
-    @parts.push(*parts)
+    @parts.concat parts
 
     @file = nil
   end
@@ -31,7 +33,7 @@ class RDoc::Markup::Document
     case part
     when RDoc::Markup::Document then
       unless part.empty? then
-        parts.push(*part.parts)
+        parts.concat part.parts
         parts << RDoc::Markup::BlankLine.new
       end
     when String then
@@ -68,6 +70,20 @@ class RDoc::Markup::Document
   end
 
   ##
+  # Concatenates the given +parts+ onto the document
+
+  def concat parts
+    self.parts.concat parts
+  end
+
+  ##
+  # Enumerator for the parts of this document
+
+  def each &block
+    @parts.each(&block)
+  end
+
+  ##
   # Does this document have no parts?
 
   def empty?
@@ -75,6 +91,18 @@ class RDoc::Markup::Document
   end
 
   ##
+  # The file this Document was created from.
+
+  def file= location
+    @file = case location
+            when RDoc::TopLevel then
+              location.absolute_name
+            else
+              location
+            end
+  end
+
+  ##
   # When this is a collection of documents (#file is not set and this document
   # contains only other documents as its direct children) #merge replaces
   # documents in this class with documents from +other+ when the file matches
@@ -120,7 +148,16 @@ class RDoc::Markup::Document
   # Appends +parts+ to the document
 
   def push *parts
-    self.parts.push(*parts)
+    self.parts.concat parts
+  end
+
+  ##
+  # Returns an Array of headings in the document.
+  #
+  # Require 'rdoc/markup/formatter' before calling this method.
+
+  def table_of_contents
+    accept RDoc::Markup::ToTableOfContents.to_toc
   end
 
 end
diff --git a/lib/rdoc/markup/formatter.rb b/lib/rdoc/markup/formatter.rb
index f42b3fc6ea..ac76db3536 100644
--- a/lib/rdoc/markup/formatter.rb
+++ b/lib/rdoc/markup/formatter.rb
@@ -1,9 +1,9 @@
-require 'rdoc/markup'
-
 ##
 # Base class for RDoc markup formatters
 #
-# Formatters use a visitor pattern to convert content into output.
+# Formatters are a visitor that converts an RDoc::Markup tree (from a comment)
+# into some kind of output.  RDoc ships with formatters for converting back to
+# rdoc, ANSI text, HTML, a Table of Contents and other formats.
 #
 # If you'd like to write your own Formatter use
 # RDoc::Markup::FormatterTestCase.  If you're writing a text-output formatter
@@ -20,14 +20,21 @@ class RDoc::Markup::Formatter
   ##
   # Creates a new Formatter
 
-  def initialize markup = nil
+  def initialize options, markup = nil
+    @options = options
+
     @markup = markup || RDoc::Markup.new
     @am     = @markup.attribute_manager
+    @am.add_special(/<br>/, :HARD_BREAK)
+
+    @attributes = @am.attributes
 
     @attr_tags = []
 
     @in_tt = 0
-    @tt_bit = RDoc::Markup::Attribute.bitmap_for :TT
+    @tt_bit = @attributes.bitmap_for :TT
+
+    @hard_break = ''
   end
 
   ##
@@ -44,7 +51,7 @@ class RDoc::Markup::Formatter
   # tags for flexibility
 
   def add_tag(name, start, stop)
-    attr = RDoc::Markup::Attribute.bitmap_for name
+    attr = @attributes.bitmap_for name
     @attr_tags << InlineTag.new(attr, start, stop)
   end
 
@@ -58,7 +65,7 @@ class RDoc::Markup::Formatter
   ##
   # Marks up +content+
 
-  def convert(content)
+  def convert content
     @markup.convert content, self
   end
 
@@ -93,7 +100,7 @@ class RDoc::Markup::Formatter
 
     handled = false
 
-    RDoc::Markup::Attribute.each_name_of special.type do |name|
+    @attributes.each_name_of special.type do |name|
       method_name = "handle_special_#{name}"
 
       if respond_to? method_name then
@@ -102,7 +109,11 @@ class RDoc::Markup::Formatter
       end
     end
 
-    raise "Unhandled special: #{special}" unless handled
+    unless handled then
+      special_name = @attributes.as_string special.type
+
+      raise RDoc::Error, "Unhandled special #{special_name}: #{special}"
+    end
 
     special.text
   end
@@ -115,6 +126,17 @@ class RDoc::Markup::Formatter
   end
 
   ##
+  # Use ignore in your subclass to ignore the content of a node.
+  #
+  #   ##
+  #   # We don't support raw nodes in ToNoRaw
+  #
+  #   alias accept_raw ignore
+
+  def ignore *node
+  end
+
+  ##
   # Are we currently inside tt tags?
 
   def in_tt?
@@ -160,10 +182,3 @@ class RDoc::Markup::Formatter
 
 end
 
-class RDoc::Markup
-  autoload :ToAnsi,         'rdoc/markup/to_ansi'
-  autoload :ToBs,           'rdoc/markup/to_bs'
-  autoload :ToHtml,         'rdoc/markup/to_html'
-  autoload :ToHtmlCrossref, 'rdoc/markup/to_html_crossref'
-  autoload :ToRdoc,         'rdoc/markup/to_rdoc'
-end
diff --git a/lib/rdoc/markup/formatter_test_case.rb b/lib/rdoc/markup/formatter_test_case.rb
index c739f990b3..6616a75898 100644
--- a/lib/rdoc/markup/formatter_test_case.rb
+++ b/lib/rdoc/markup/formatter_test_case.rb
@@ -1,5 +1,4 @@
 require 'minitest/unit'
-require 'rdoc/markup/formatter'
 
 ##
 # Test case for creating new RDoc::Markup formatters.  See
@@ -35,7 +34,7 @@ require 'rdoc/markup/formatter'
 #
 #  end
 
-class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
+class RDoc::Markup::FormatterTestCase < RDoc::TestCase
 
   ##
   # Call #setup when inheriting from this test case.
@@ -54,8 +53,9 @@ class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
   def setup
     super
 
-    @m = RDoc::Markup.new
-    @RM = RDoc::Markup
+    @options = RDoc::Options.new
+
+    @m = @RM.new
 
     @bullet_list = @RM::List.new(:BULLET,
       @RM::ListItem.new(nil, @RM::Paragraph.new('l1')),
@@ -86,7 +86,7 @@ class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
   # Call to add the visitor tests to your test case
 
   def self.add_visitor_tests
-    self.class_eval do
+    class_eval do
 
       ##
       # Calls start_accepting which needs to verify startup state
@@ -120,6 +120,16 @@ class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
       end
 
       ##
+      # Calls accept_block_quote
+
+      def test_accept_block_quote
+        @to.start_accepting
+
+        @to.accept_block_quote block para 'quote'
+
+        accept_block_quote
+      end
+      ##
       # Test case that calls <tt>@to.accept_document</tt>
 
       def test_accept_document
@@ -234,6 +244,29 @@ class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
       end
 
       ##
+      # Calls accept_paragraph_br with a RDoc::Markup::Paragraph containing
+      # a \<br>
+
+      def test_accept_paragraph_br
+        @to.start_accepting
+
+        @to.accept_paragraph para 'one<br>two'
+
+        accept_paragraph_br
+      end
+
+      ##
+      # Calls accept_paragraph with a Paragraph containing a hard break
+
+      def test_accept_paragraph_break
+        @to.start_accepting
+
+        @to.accept_paragraph para('hello', hard_break, 'world')
+
+        accept_paragraph_break
+      end
+
+      ##
       # Calls accept_paragraph_i with a RDoc::Markup::Paragraph containing
       # emphasized words
 
@@ -374,9 +407,9 @@ class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
       # Calls accept_list_item_start_note_2
 
       def test_accept_list_item_start_note_2
-        list = @RM::List.new(:NOTE,
-                 @RM::ListItem.new('<tt>teletype</tt>',
-                   @RM::Paragraph.new('teletype description')))
+        list = list(:NOTE,
+                 item('<tt>teletype</tt>',
+                   para('teletype description')))
 
         @to.start_accepting
 
@@ -388,6 +421,41 @@ class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
       end
 
       ##
+      # Calls accept_list_item_start_note_multi_description
+
+      def test_accept_list_item_start_note_multi_description
+        list = list(:NOTE,
+                 item(%w[label],
+                   para('description one')),
+                 item(nil, para('description two')))
+
+        @to.start_accepting
+
+        list.accept @to
+
+        @to.end_accepting
+
+        accept_list_item_start_note_multi_description
+      end
+
+      ##
+      # Calls accept_list_item_start_note_multi_label
+
+      def test_accept_list_item_start_note_multi_label
+        list = list(:NOTE,
+                 item(%w[one two],
+                   para('two headers')))
+
+        @to.start_accepting
+
+        list.accept @to
+
+        @to.end_accepting
+
+        accept_list_item_start_note_multi_label
+      end
+
+      ##
       # Calls accept_list_item_start_number
 
       def test_accept_list_item_start_number
@@ -635,7 +703,7 @@ class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
       end
 
       ##
-      # Calls accept_list_end_ulpha
+      # Calls accept_list_end_ualpha
 
       def test_accept_list_end_ualpha
         @to.start_accepting
@@ -670,28 +738,28 @@ class RDoc::Markup::FormatterTestCase < MiniTest::Unit::TestCase
       # Calls list_verbatim with a list containing a verbatim block
 
       def test_list_verbatim # HACK overblown
-        doc = @RM::Document.new(
-                @RM::List.new(:BULLET,
-                  @RM::ListItem.new(nil,
-                    @RM::Paragraph.new('list', 'stuff'),
-                    @RM::BlankLine.new,
-                    @RM::Verbatim.new("* list\n",
-                                      "  with\n",
-                                      "\n",
-                                      "  second\n",
-                                      "\n",
-                                      "  1. indented\n",
-                                      "  2. numbered\n",
-                                      "\n",
-                                      "  third\n",
-                                      "\n",
-                                      "* second\n"))))
+        doc =
+          doc(
+            list(:BULLET,
+              item(nil,
+                para('list stuff'),
+                blank_line,
+                verb("* list\n",
+                     "  with\n",
+                     "\n",
+                     "  second\n",
+                     "\n",
+                     "  1. indented\n",
+                     "  2. numbered\n",
+                     "\n",
+                     "  third\n",
+                     "\n",
+                     "* second\n"))))
 
         doc.accept @to
 
         list_verbatim
       end
-
     end
   end
 
diff --git a/lib/rdoc/markup/hard_break.rb b/lib/rdoc/markup/hard_break.rb
new file mode 100644
index 0000000000..8445ad26e7
--- /dev/null
+++ b/lib/rdoc/markup/hard_break.rb
@@ -0,0 +1,31 @@
+##
+# A hard-break in the middle of a paragraph.
+
+class RDoc::Markup::HardBreak
+
+  @instance = new
+
+  ##
+  # RDoc::Markup::HardBreak is a singleton
+
+  def self.new
+    @instance
+  end
+
+  ##
+  # Calls #accept_hard_break on +visitor+
+
+  def accept visitor
+    visitor.accept_hard_break self
+  end
+
+  def == other # :nodoc:
+    self.class === other
+  end
+
+  def pretty_print q # :nodoc:
+    q.text "[break]"
+  end
+
+end
+
diff --git a/lib/rdoc/markup/heading.rb b/lib/rdoc/markup/heading.rb
index 3bda77a1e1..b72c3e2b14 100644
--- a/lib/rdoc/markup/heading.rb
+++ b/lib/rdoc/markup/heading.rb
@@ -3,6 +3,35 @@
 
 class RDoc::Markup::Heading < Struct.new :level, :text
 
+  @to_html = nil
+  @to_label = nil
+
+  ##
+  # A singleton RDoc::Markup::ToLabel formatter for headings.
+
+  def self.to_label
+    @to_label ||= RDoc::Markup::ToLabel.new
+  end
+
+  ##
+  # A singleton plain HTML formatter for headings.  Used for creating labels
+  # for the Table of Contents
+
+  def self.to_html
+    return @to_html if @to_html
+
+    markup = RDoc::Markup.new
+    markup.add_special RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
+
+    @to_html = RDoc::Markup::ToHtml.new nil
+
+    def @to_html.handle_special_CROSSREF special
+      special.text.sub(/^\\/, '')
+    end
+
+    @to_html
+  end
+
   ##
   # Calls #accept_heading on +visitor+
 
@@ -10,6 +39,21 @@ class RDoc::Markup::Heading < Struct.new :level, :text
     visitor.accept_heading self
   end
 
+  ##
+  # An HTML-safe anchor reference for this header.
+
+  def aref
+    "label-#{self.class.to_label.convert text.dup}"
+  end
+
+  ##
+  # HTML markup of the text of this label without the surrounding header
+  # element.
+
+  def plain_html
+    self.class.to_html.to_html(text.dup)
+  end
+
   def pretty_print q # :nodoc:
     q.group 2, "[head: #{level} ", ']' do
       q.pp text
diff --git a/lib/rdoc/markup/include.rb b/lib/rdoc/markup/include.rb
new file mode 100644
index 0000000000..a2e8903279
--- /dev/null
+++ b/lib/rdoc/markup/include.rb
@@ -0,0 +1,42 @@
+##
+# A file included at generation time.  Objects of this class are created by
+# RDoc::RD for an extension-less include.
+#
+# This implementation in incomplete.
+
+class RDoc::Markup::Include
+
+  ##
+  # The filename to be included, without extension
+
+  attr_reader :file
+
+  ##
+  # Directories to search for #file
+
+  attr_reader :include_path
+
+  ##
+  # Creates a new include that will import +file+ from +include_path+
+
+  def initialize file, include_path
+    @file = file
+    @include_path = include_path
+  end
+
+  def == other # :nodoc:
+    self.class === other and
+      @file == other.file and @include_path == other.include_path
+  end
+
+  def pretty_print q # :nodoc:
+    q.group 2, '[incl ', ']' do
+      q.text file
+      q.breakable
+      q.text 'from '
+      q.pp include_path
+    end
+  end
+
+end
+
diff --git a/lib/rdoc/markup/indented_paragraph.rb b/lib/rdoc/markup/indented_paragraph.rb
index d995c7d8ed..1b8a8c725d 100644
--- a/lib/rdoc/markup/indented_paragraph.rb
+++ b/lib/rdoc/markup/indented_paragraph.rb
@@ -29,5 +29,19 @@ class RDoc::Markup::IndentedParagraph < RDoc::Markup::Raw
     visitor.accept_indented_paragraph self
   end
 
+  ##
+  # Joins the raw paragraph text and converts inline HardBreaks to the
+  # +hard_break+ text followed by the indent.
+
+  def text hard_break = nil
+    @parts.map do |part|
+      if RDoc::Markup::HardBreak === part then
+        '%1$s%3$*2$s' % [hard_break, @indent, ' '] if hard_break
+      else
+        part
+      end
+    end.join
+  end
+
 end
 
diff --git a/lib/rdoc/markup/inline.rb b/lib/rdoc/markup/inline.rb
index cf598d1583..fb3ab5c74d 100644
--- a/lib/rdoc/markup/inline.rb
+++ b/lib/rdoc/markup/inline.rb
@@ -1,144 +1 @@
-require 'rdoc'
-class RDoc::Markup
-
-  ##
-  # We manage a set of attributes. Each attribute has a symbol name and a bit
-  # value.
-
-  class Attribute
-
-    ##
-    # Special attribute type.  See RDoc::Markup#add_special
-
-    SPECIAL = 1
-
-    @@name_to_bitmap = { :_SPECIAL_ => SPECIAL }
-    @@next_bitmap = 2
-
-    ##
-    # Returns a unique bit for +name+
-
-    def self.bitmap_for(name)
-      bitmap = @@name_to_bitmap[name]
-      unless bitmap then
-        bitmap = @@next_bitmap
-        @@next_bitmap <<= 1
-        @@name_to_bitmap[name] = bitmap
-      end
-      bitmap
-    end
-
-    ##
-    # Returns a string representation of +bitmap+
-
-    def self.as_string(bitmap)
-      return "none" if bitmap.zero?
-      res = []
-      @@name_to_bitmap.each do |name, bit|
-        res << name if (bitmap & bit) != 0
-      end
-      res.join(",")
-    end
-
-    ##
-    # yields each attribute name in +bitmap+
-
-    def self.each_name_of(bitmap)
-      @@name_to_bitmap.each do |name, bit|
-        next if bit == SPECIAL
-        yield name.to_s if (bitmap & bit) != 0
-      end
-    end
-
-  end
-
-  AttrChanger = Struct.new :turn_on, :turn_off # :nodoc:
-
-  ##
-  # An AttrChanger records a change in attributes. It contains a bitmap of the
-  # attributes to turn on, and a bitmap of those to turn off.
-
-  class AttrChanger
-    def to_s # :nodoc:
-      "Attr: +#{Attribute.as_string turn_on}/-#{Attribute.as_string turn_off}"
-    end
-
-    def inspect # :nodoc:
-      "+%s/-%s" % [
-        Attribute.as_string(turn_on),
-        Attribute.as_string(turn_off),
-      ]
-    end
-  end
-
-  ##
-  # An array of attributes which parallels the characters in a string.
-
-  class AttrSpan
-
-    ##
-    # Creates a new AttrSpan for +length+ characters
-
-    def initialize(length)
-      @attrs = Array.new(length, 0)
-    end
-
-    ##
-    # Toggles +bits+ from +start+ to +length+
-    def set_attrs(start, length, bits)
-      for i in start ... (start+length)
-        @attrs[i] |= bits
-      end
-    end
-
-    ##
-    # Accesses flags for character +n+
-
-    def [](n)
-      @attrs[n]
-    end
-
-  end
-
-  ##
-  # Hold details of a special sequence
-
-  class Special
-
-    ##
-    # Special type
-
-    attr_reader   :type
-
-    ##
-    # Special text
-
-    attr_accessor :text
-
-    ##
-    # Creates a new special sequence of +type+ with +text+
-
-    def initialize(type, text)
-      @type, @text = type, text
-    end
-
-    ##
-    # Specials are equal when the have the same text and type
-
-    def ==(o)
-      self.text == o.text && self.type == o.type
-    end
-
-    def inspect # :nodoc:
-      "#<RDoc::Markup::Special:0x%x @type=%p, name=%p @text=%p>" % [
-        object_id, @type, RDoc::Markup::Attribute.as_string(type), text.dump]
-    end
-
-    def to_s # :nodoc:
-      "Special: type=#{type}, name=#{RDoc::Markup::Attribute.as_string type}, text=#{text.dump}"
-    end
-
-  end
-
-end
-
+warn "requiring rdoc/markup/inline is deprecated and will be removed in RDoc 4." if $-w
diff --git a/lib/rdoc/markup/list.rb b/lib/rdoc/markup/list.rb
index 820b4c9645..86ed845634 100644
--- a/lib/rdoc/markup/list.rb
+++ b/lib/rdoc/markup/list.rb
@@ -1,5 +1,24 @@
 ##
-# A List of ListItems
+# A List is a homogeneous set of ListItems.
+#
+# The supported list types include:
+#
+# :BULLET::
+#   An unordered list
+# :LABEL::
+#   An unordered definition list, but using an alternate RDoc::Markup syntax
+# :LALPHA::
+#   An ordered list using increasing lowercase English letters
+# :NOTE::
+#   An unordered definition list
+# :NUMBER::
+#   An ordered list using increasing Arabic numerals
+# :UALPHA::
+#   An ordered list using increasing uppercase English letters
+#
+# Definition lists behave like HTML definition lists.  Each list item can
+# describe multiple terms.  See RDoc::Markup::ListItem for how labels and
+# definition are stored as list items.
 
 class RDoc::Markup::List
 
@@ -14,12 +33,13 @@ class RDoc::Markup::List
   attr_reader :items
 
   ##
-  # Creates a new list of +type+ with +items+
+  # Creates a new list of +type+ with +items+.  Valid list types are:
+  # +:BULLET+, +:LABEL+, +:LALPHA+, +:NOTE+, +:NUMBER+, +:UALPHA+ 
 
   def initialize type = nil, *items
     @type = type
     @items = []
-    @items.push(*items)
+    @items.concat items
   end
 
   ##
@@ -74,7 +94,7 @@ class RDoc::Markup::List
   # Appends +items+ to the list
 
   def push *items
-    @items.push(*items)
+    @items.concat items
   end
 
 end
diff --git a/lib/rdoc/markup/list_item.rb b/lib/rdoc/markup/list_item.rb
index d719c352ec..c5e59fe167 100644
--- a/lib/rdoc/markup/list_item.rb
+++ b/lib/rdoc/markup/list_item.rb
@@ -1,5 +1,12 @@
 ##
 # An item within a List that contains paragraphs, headings, etc.
+#
+# For BULLET, NUMBER, LALPHA and UALPHA lists, the label will always be nil.
+# For NOTE and LABEL lists, the list label may contain:
+#
+# * a single String for a single label
+# * an Array of Strings for a list item with multiple terms
+# * nil for an extra description attached to a previously labeled list item
 
 class RDoc::Markup::ListItem
 
@@ -19,7 +26,7 @@ class RDoc::Markup::ListItem
   def initialize label = nil, *parts
     @label = label
     @parts = []
-    @parts.push(*parts)
+    @parts.concat parts
   end
 
   ##
@@ -64,8 +71,14 @@ class RDoc::Markup::ListItem
 
   def pretty_print q # :nodoc:
     q.group 2, '[item: ', ']' do
-      if @label then
-        q.text @label
+      case @label
+      when Array then
+        q.pp @label
+        q.text ';'
+        q.breakable
+      when String then
+        q.pp @label
+        q.text ';'
         q.breakable
       end
 
@@ -79,7 +92,7 @@ class RDoc::Markup::ListItem
   # Adds +parts+ to the ListItem
 
   def push *parts
-    @parts.push(*parts)
+    @parts.concat parts
   end
 
 end
diff --git a/lib/rdoc/markup/paragraph.rb b/lib/rdoc/markup/paragraph.rb
index 808430d576..7180729f75 100644
--- a/lib/rdoc/markup/paragraph.rb
+++ b/lib/rdoc/markup/paragraph.rb
@@ -10,5 +10,19 @@ class RDoc::Markup::Paragraph < RDoc::Markup::Raw
     visitor.accept_paragraph self
   end
 
+  ##
+  # Joins the raw paragraph text and converts inline HardBreaks to the
+  # +hard_break+ text.
+
+  def text hard_break = ''
+    @parts.map do |part|
+      if RDoc::Markup::HardBreak === part then
+        hard_break
+      else
+        part
+      end
+    end.join
+  end
+
 end
 
diff --git a/lib/rdoc/markup/parser.rb b/lib/rdoc/markup/parser.rb
index c18ce821fb..ca384d0639 100644
--- a/lib/rdoc/markup/parser.rb
+++ b/lib/rdoc/markup/parser.rb
@@ -1,5 +1,4 @@
 require 'strscan'
-require 'rdoc/text'
 
 ##
 # A recursive-descent parser for RDoc markup.
@@ -52,7 +51,9 @@ class RDoc::Markup::Parser
   attr_reader :tokens
 
   ##
-  # Parses +str+ into a Document
+  # Parses +str+ into a Document.
+  #
+  # Use RDoc::Markup#parse instead of this method.
 
   def self.parse str
     parser = new
@@ -74,12 +75,15 @@ class RDoc::Markup::Parser
   # Creates a new Parser.  See also ::parse
 
   def initialize
-    @tokens = []
-    @current_token = nil
-    @debug = false
-
-    @line = 0
-    @line_pos = 0
+    @binary_input   = nil
+    @current_token  = nil
+    @debug          = false
+    @have_encoding  = Object.const_defined? :Encoding
+    @input_encoding = nil
+    @line           = 0
+    @line_pos       = 0
+    @s              = nil
+    @tokens         = []
   end
 
   ##
@@ -107,13 +111,13 @@ class RDoc::Markup::Parser
     p :list_start => margin if @debug
 
     list = RDoc::Markup::List.new
+    label = nil
 
     until @tokens.empty? do
       type, data, column, = get
 
       case type
-      when :BULLET, :LABEL, :LALPHA, :NOTE, :NUMBER, :UALPHA then
-
+      when *LIST_TOKENS then
         if column < margin || (list.type && list.type != type) then
           unget
           break
@@ -124,6 +128,8 @@ class RDoc::Markup::Parser
 
         case type
         when :NOTE, :LABEL then
+          label = [] unless label
+
           if peek_type == :NEWLINE then
             # description not on the same line as LABEL/NOTE
             # skip the trailing newline & any blank lines below
@@ -146,32 +152,35 @@ class RDoc::Markup::Parser
             # In all cases, we have an empty description.
             # In the last case only, we continue.
             if peek_type.nil? || column < margin then
-              empty = 1
+              empty = true
             elsif column == margin then
               case peek_type
               when type
-                empty = 2 # continue
+                empty = :continue
               when *LIST_TOKENS
-                empty = 1
+                empty = true
               else
-                empty = 0
+                empty = false
               end
             else
-              empty = 0
+              empty = false
             end
 
-            if empty > 0 then
-              item = RDoc::Markup::ListItem.new(data)
-              item << RDoc::Markup::BlankLine.new
-              list << item
-              break if empty == 1
-              next
+            if empty then
+              label << data
+              next if empty == :continue
+              break
             end
           end
         else
           data = nil
         end
 
+        if label then
+          data = label << data
+          label = nil
+        end
+
         list_item = RDoc::Markup::ListItem.new data
         parse list_item, column
         list << list_item
@@ -184,7 +193,13 @@ class RDoc::Markup::Parser
 
     p :list_end => margin if @debug
 
-    return nil if list.empty?
+    if list.empty? then
+      return nil unless label
+      return nil unless [:LABEL, :NOTE].include? list.type
+
+      list_item = RDoc::Markup::ListItem.new label, RDoc::Markup::BlankLine.new
+      list << list_item
+    end
 
     list
   end
@@ -200,15 +215,20 @@ class RDoc::Markup::Parser
     until @tokens.empty? do
       type, data, column, = get
 
-      if type == :TEXT && column == margin then
+      if type == :TEXT and column == margin then
         paragraph << data
-        skip :NEWLINE
+
+        break if peek_token.first == :BREAK
+
+        data << ' ' if skip :NEWLINE
       else
         unget
         break
       end
     end
 
+    paragraph.parts.last.sub!(/ \z/, '') # cleanup
+
     p :paragraph_end => margin if @debug
 
     paragraph
@@ -267,7 +287,7 @@ class RDoc::Markup::Parser
         peek_column ||= column + width
         indent = peek_column - column - width
         line << ' ' * indent
-      when :TEXT then
+      when :BREAK, :TEXT then
         line << data
       else # *LIST_TOKENS
         list_marker = case type
@@ -298,6 +318,19 @@ class RDoc::Markup::Parser
   end
 
   ##
+  # The character offset for the input string at the given +byte_offset+
+
+  def char_pos byte_offset
+    if @have_encoding then
+      matched = @binary_input[0, byte_offset]
+      matched.force_encoding @input_encoding
+      matched.length
+    else
+      byte_offset
+    end
+  end
+
+  ##
   # Pulls the next token from the stream.
 
   def get
@@ -321,7 +354,12 @@ class RDoc::Markup::Parser
     until @tokens.empty? do
       type, data, column, = get
 
-      if type == :NEWLINE then
+      case type
+      when :BREAK then
+        parent << RDoc::Markup::BlankLine.new
+        skip :NEWLINE, false
+        next
+      when :NEWLINE then
         # trailing newlines are skipped below, so this is a blank line
         parent << RDoc::Markup::BlankLine.new
         skip :NEWLINE, false
@@ -373,6 +411,21 @@ class RDoc::Markup::Parser
   end
 
   ##
+  # Creates the StringScanner
+
+  def setup_scanner input
+    @line     = 0
+    @line_pos = 0
+
+    if @have_encoding then
+      @input_encoding = input.encoding
+      @binary_input   = input.dup.force_encoding Encoding::BINARY
+    end
+
+    @s = StringScanner.new input
+  end
+
+  ##
   # Skips the next token if its type is +token_type+.
   #
   # Optionally raises an error if the next token is not of the expected type.
@@ -389,58 +442,55 @@ class RDoc::Markup::Parser
   # Turns text +input+ into a stream of tokens
 
   def tokenize input
-    s = StringScanner.new input
+    setup_scanner input
 
-    @line = 0
-    @line_pos = 0
-
-    until s.eos? do
-      pos = s.pos
+    until @s.eos? do
+      pos = @s.pos
 
       # leading spaces will be reflected by the column of the next token
       # the only thing we loose are trailing spaces at the end of the file
-      next if s.scan(/ +/)
+      next if @s.scan(/ +/)
 
       # note: after BULLET, LABEL, etc.,
       # indent will be the column of the next non-newline token
 
       @tokens << case
                  # [CR]LF => :NEWLINE
-                 when s.scan(/\r?\n/) then
-                   token = [:NEWLINE, s.matched, *token_pos(pos)]
-                   @line_pos = s.pos
+                 when @s.scan(/\r?\n/) then
+                   token = [:NEWLINE, @s.matched, *token_pos(pos)]
+                   @line_pos = char_pos @s.pos
                    @line += 1
                    token
                  # === text => :HEADER then :TEXT
-                 when s.scan(/(=+)(\s*)/) then
-                   level = s[1].length
+                 when @s.scan(/(=+)(\s*)/) then
+                   level = @s[1].length
                    header = [:HEADER, level, *token_pos(pos)]
 
-                   if s[2] =~ /^\r?\n/ then
-                     s.pos -= s[2].length
+                   if @s[2] =~ /^\r?\n/ then
+                     @s.pos -= @s[2].length
                      header
                    else
-                     pos = s.pos
-                     s.scan(/.*/)
+                     pos = @s.pos
+                     @s.scan(/.*/)
                      @tokens << header
-                     [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                     [:TEXT, @s.matched.sub(/\r$/, ''), *token_pos(pos)]
                    end
                  # --- (at least 3) and nothing else on the line => :RULE
-                 when s.scan(/(-{3,}) *$/) then
-                   [:RULE, s[1].length - 2, *token_pos(pos)]
+                 when @s.scan(/(-{3,}) *\r?$/) then
+                   [:RULE, @s[1].length - 2, *token_pos(pos)]
                  # * or - followed by white space and text => :BULLET
-                 when s.scan(/([*-]) +(\S)/) then
-                   s.pos -= s[2].bytesize # unget \S
-                   [:BULLET, s[1], *token_pos(pos)]
+                 when @s.scan(/([*-]) +(\S)/) then
+                   @s.pos -= @s[2].bytesize # unget \S
+                   [:BULLET, @s[1], *token_pos(pos)]
                  # A. text, a. text, 12. text => :UALPHA, :LALPHA, :NUMBER
-                 when s.scan(/([a-z]|\d+)\. +(\S)/i) then
+                 when @s.scan(/([a-z]|\d+)\. +(\S)/i) then
                    # FIXME if tab(s), the column will be wrong
                    # either support tabs everywhere by first expanding them to
                    # spaces, or assume that they will have been replaced
                    # before (and provide a check for that at least in debug
                    # mode)
-                   list_label = s[1]
-                   s.pos -= s[2].bytesize # unget \S
+                   list_label = @s[1]
+                   @s.pos -= @s[2].bytesize # unget \S
                    list_type =
                      case list_label
                      when /[a-z]/ then :LALPHA
@@ -451,14 +501,21 @@ class RDoc::Markup::Parser
                      end
                    [list_type, list_label, *token_pos(pos)]
                  # [text] followed by spaces or end of line => :LABEL
-                 when s.scan(/\[(.*?)\]( +|$)/) then
-                   [:LABEL, s[1], *token_pos(pos)]
+                 when @s.scan(/\[(.*?)\]( +|\r?$)/) then
+                   [:LABEL, @s[1], *token_pos(pos)]
                  # text:: followed by spaces or end of line => :NOTE
-                 when s.scan(/(.*?)::( +|$)/) then
-                   [:NOTE, s[1], *token_pos(pos)]
+                 when @s.scan(/(.*?)::( +|\r?$)/) then
+                   [:NOTE, @s[1], *token_pos(pos)]
                  # anything else: :TEXT
-                 else s.scan(/.*/)
-                   [:TEXT, s.matched.sub(/\r$/, ''), *token_pos(pos)]
+                 else @s.scan(/(.*?)(  )?\r?$/)
+                   token = [:TEXT, @s[1], *token_pos(pos)]
+
+                   if @s[2] then
+                     @tokens << token
+                     [:BREAK, @s[2], *token_pos(pos + @s[1].length)]
+                   else
+                     token
+                   end
                  end
     end
 
@@ -466,9 +523,12 @@ class RDoc::Markup::Parser
   end
 
   ##
-  # Calculates the column and line of the current token based on +offset+.
+  # Calculates the column (by character) and line of the current token from
+  # +scanner+ based on +byte_offset+.
+
+  def token_pos byte_offset
+    offset = char_pos byte_offset
 
-  def token_pos offset
     [offset - @line_pos, @line]
   end
 
@@ -484,14 +544,3 @@ class RDoc::Markup::Parser
 
 end
 
-require 'rdoc/markup/blank_line'
-require 'rdoc/markup/document'
-require 'rdoc/markup/heading'
-require 'rdoc/markup/list'
-require 'rdoc/markup/list_item'
-require 'rdoc/markup/raw'
-require 'rdoc/markup/paragraph'
-require 'rdoc/markup/indented_paragraph'
-require 'rdoc/markup/rule'
-require 'rdoc/markup/verbatim'
-
diff --git a/lib/rdoc/markup/pre_process.rb b/lib/rdoc/markup/pre_process.rb
index 53e8e38ec1..6024edcd27 100644
--- a/lib/rdoc/markup/pre_process.rb
+++ b/lib/rdoc/markup/pre_process.rb
@@ -1,6 +1,3 @@
-require 'rdoc/markup'
-require 'rdoc/encoding'
-
 ##
 # Handle common directives that can occur in a block of text:
 #
@@ -9,18 +6,48 @@ require 'rdoc/encoding'
 # Directives can be escaped by preceding them with a backslash.
 #
 # RDoc plugin authors can register additional directives to be handled by
-# using RDoc::Markup::PreProcess::register
+# using RDoc::Markup::PreProcess::register.
+#
+# Any directive that is not built-in to RDoc (including those registered via
+# plugins) will be stored in the metadata hash on the CodeObject the comment
+# is attached to.  See RDoc::Markup@Directives for the list of built-in
+# directives.
 
 class RDoc::Markup::PreProcess
 
+  ##
+  # An RDoc::Options instance that will be filled in with overrides from
+  # directives
+
   attr_accessor :options
 
-  @registered = {}
+  ##
+  # Adds a post-process handler for directives.  The handler will be called
+  # with the result RDoc::Comment (or text String) and the code object for the
+  # comment (if any).
+
+  def self.post_process &block
+    @post_processors << block
+  end
+
+  ##
+  # Registered post-processors
+
+  def self.post_processors
+    @post_processors
+  end
 
   ##
   # Registers +directive+ as one handled by RDoc.  If a block is given the
   # directive will be replaced by the result of the block, otherwise the
   # directive will be removed from the processed text.
+  #
+  # The block will be called with the directive name and the directive
+  # parameter:
+  #
+  #   RDoc::Markup::PreProcess.register 'my-directive' do |directive, param|
+  #     # replace text, etc.
+  #   end
 
   def self.register directive, &block
     @registered[directive] = block
@@ -34,6 +61,16 @@ class RDoc::Markup::PreProcess
   end
 
   ##
+  # Clears all registered directives and post-processors
+
+  def self.reset
+    @post_processors = []
+    @registered = {}
+  end
+
+  reset
+
+  ##
   # Creates a new pre-processor for +input_file_name+ that will look for
   # included files in +include_path+
 
@@ -44,7 +81,7 @@ class RDoc::Markup::PreProcess
   end
 
   ##
-  # Look for directives in a chunk of +text+.
+  # Look for directives in the given +text+.
   #
   # Options that we don't handle are yielded.  If the block returns false the
   # directive is restored to the text.  If the block returns nil or no block
@@ -54,27 +91,56 @@ class RDoc::Markup::PreProcess
   # If no matching directive was registered the directive is restored to the
   # text.
   #
-  # If +code_object+ is given and the param is set as metadata on the
-  # +code_object+.  See RDoc::CodeObject#metadata
+  # If +code_object+ is given and the directive is unknown then the
+  # directive's parameter is set as metadata on the +code_object+.  See
+  # RDoc::CodeObject#metadata for details.
 
   def handle text, code_object = nil, &block
-    encoding = if defined?(Encoding) then text.encoding else nil end
+    if RDoc::Comment === text then
+      comment = text
+      text = text.text
+    end
+
+    encoding = text.encoding if defined?(Encoding)
+
     # regexp helper (square brackets for optional)
     # $1      $2  $3        $4      $5
     # [prefix][\]:directive:[spaces][param]newline
-    text.gsub!(/^([ \t]*(?:#|\/?\*)?[ \t]*)(\\?):(\w+):([ \t]*)(.+)?\n/) do
+    text.gsub!(/^([ \t]*(?:#|\/?\*)?[ \t]*)(\\?):(\w+):([ \t]*)(.+)?(\r?\n|$)/) do
       # skip something like ':toto::'
       next $& if $4.empty? and $5 and $5[0, 1] == ':'
 
       # skip if escaped
       next "#$1:#$3:#$4#$5\n" unless $2.empty?
 
+      # This is not in handle_directive because I didn't want to pass another
+      # argument into it
+      if comment and $3 == 'markup' then
+        next "#{$1.strip}\n" unless $5
+        comment.format = $5.downcase
+        next "#{$1.strip}\n"
+      end
+
       handle_directive $1, $3, $5, code_object, encoding, &block
     end
 
+    comment = text unless comment
+
+    self.class.post_processors.each do |handler|
+      handler.call comment, code_object
+    end
+
     text
   end
 
+  ##
+  # Performs the actions described by +directive+ and its parameter +param+.
+  #
+  # +code_object+ is used for directives that operate on a class or module.
+  # +prefix+ is used to ensure the replacement for handled directives is
+  # correct.  +encoding+ is used for the <tt>include</tt> directive.
+  #
+  # For a list of directives in RDoc see RDoc::Markup.
   #--
   # When 1.8.7 support is ditched prefix can be defaulted to ''
 
@@ -92,7 +158,7 @@ class RDoc::Markup::PreProcess
       blankline
     when 'category' then
       if RDoc::Context === code_object then
-        section = code_object.add_section param, ''
+        section = code_object.add_section param
         code_object.temporary_section = section
       end
 
diff --git a/lib/rdoc/markup/raw.rb b/lib/rdoc/markup/raw.rb
index ca877c79af..e11e8efff4 100644
--- a/lib/rdoc/markup/raw.rb
+++ b/lib/rdoc/markup/raw.rb
@@ -13,7 +13,7 @@ class RDoc::Markup::Raw
 
   def initialize *parts
     @parts = []
-    @parts.push(*parts)
+    @parts.concat parts
   end
 
   ##
@@ -24,7 +24,7 @@ class RDoc::Markup::Raw
   end
 
   def == other # :nodoc:
-    self.class == other.class and text == other.text
+    self.class == other.class and @parts == other.parts
   end
 
   ##
@@ -38,11 +38,11 @@ class RDoc::Markup::Raw
   # Appends +other+'s parts
 
   def merge other
-    @parts.push(*other.parts)
+    @parts.concat other.parts
   end
 
   def pretty_print q # :nodoc:
-    self.class.name =~ /.*::(\w{4})/i
+    self.class.name =~ /.*::(\w{1,4})/i
 
     q.group 2, "[#{$1.downcase}: ", ']' do
       q.seplist @parts do |part|
@@ -55,7 +55,7 @@ class RDoc::Markup::Raw
   # Appends +texts+ onto this Paragraph
 
   def push *texts
-    self.parts.push(*texts)
+    self.parts.concat texts
   end
 
   ##
diff --git a/lib/rdoc/markup/special.rb b/lib/rdoc/markup/special.rb
new file mode 100644
index 0000000000..1c0fc03eea
--- /dev/null
+++ b/lib/rdoc/markup/special.rb
@@ -0,0 +1,40 @@
+##
+# Hold details of a special sequence
+
+class RDoc::Markup::Special
+
+  ##
+  # Special type
+
+  attr_reader   :type
+
+  ##
+  # Special text
+
+  attr_accessor :text
+
+  ##
+  # Creates a new special sequence of +type+ with +text+
+
+  def initialize(type, text)
+    @type, @text = type, text
+  end
+
+  ##
+  # Specials are equal when the have the same text and type
+
+  def ==(o)
+    self.text == o.text && self.type == o.type
+  end
+
+  def inspect # :nodoc:
+    "#<RDoc::Markup::Special:0x%x @type=%p, @text=%p>" % [
+      object_id, @type, text.dump]
+  end
+
+  def to_s # :nodoc:
+    "Special: type=#{type} text=#{text.dump}"
+  end
+
+end
+
diff --git a/lib/rdoc/markup/text_formatter_test_case.rb b/lib/rdoc/markup/text_formatter_test_case.rb
index ba9e7c6187..4abf42563b 100644
--- a/lib/rdoc/markup/text_formatter_test_case.rb
+++ b/lib/rdoc/markup/text_formatter_test_case.rb
@@ -1,5 +1,3 @@
-require 'rdoc/markup/formatter_test_case'
-
 ##
 # Test case for creating new plain-text RDoc::Markup formatters.  See also
 # RDoc::Markup::FormatterTestCase
diff --git a/lib/rdoc/markup/to_ansi.rb b/lib/rdoc/markup/to_ansi.rb
index 1e8a0289d9..4d439ce88d 100644
--- a/lib/rdoc/markup/to_ansi.rb
+++ b/lib/rdoc/markup/to_ansi.rb
@@ -1,5 +1,3 @@
-require 'rdoc/markup/to_rdoc'
-
 ##
 # Outputs RDoc markup with vibrant ANSI color!
 
@@ -34,6 +32,11 @@ class RDoc::Markup::ToAnsi < RDoc::Markup::ToRdoc
             when :BULLET then
               2
             when :NOTE, :LABEL then
+              if @prefix then
+                @res << @prefix.strip
+                @prefix = nil
+              end
+
               @res << "\n" unless res.length == 1
               2
             else
@@ -53,7 +56,13 @@ class RDoc::Markup::ToAnsi < RDoc::Markup::ToRdoc
              when :BULLET then
                '*'
              when :NOTE, :LABEL then
-               attributes(list_item.label) + ":\n"
+               labels = Array(list_item.label).map do |label|
+                 attributes(label).strip
+               end.join "\n"
+
+               labels << ":\n" unless labels.empty?
+
+               labels
              else
                @list_index.last.to_s + '.'
              end
diff --git a/lib/rdoc/markup/to_bs.rb b/lib/rdoc/markup/to_bs.rb
index 32b1bbb9eb..10c31854d2 100644
--- a/lib/rdoc/markup/to_bs.rb
+++ b/lib/rdoc/markup/to_bs.rb
@@ -1,5 +1,3 @@
-require 'rdoc/markup/to_rdoc'
-
 ##
 # Outputs RDoc markup with hot backspace action!  You will probably need a
 # pager to use this output format.
diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb
index bd5fdb493e..9cd94a5945 100644
--- a/lib/rdoc/markup/to_html.rb
+++ b/lib/rdoc/markup/to_html.rb
@@ -1,10 +1,7 @@
-require 'rdoc/markup/formatter'
-require 'rdoc/markup/inline'
-
 require 'cgi'
 
 ##
-# Outputs RDoc markup as HTML
+# Outputs RDoc markup as HTML.
 
 class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
@@ -16,12 +13,12 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # Maps RDoc::Markup::Parser::LIST_TOKENS types to HTML tags
 
   LIST_TYPE_TO_HTML = {
-    :BULLET => ['<ul>', '</ul>'],
-    :LABEL  => ['<dl class="rdoc-list">', '</dl>'],
-    :LALPHA => ['<ol style="display: lower-alpha">', '</ol>'],
-    :NOTE   => ['<table class="rdoc-list">', '</table>'],
-    :NUMBER => ['<ol>', '</ol>'],
-    :UALPHA => ['<ol style="display: upper-alpha">', '</ol>'],
+    :BULLET => ['<ul>',                                      '</ul>'],
+    :LABEL  => ['<dl class="rdoc-list label-list">',         '</dl>'],
+    :LALPHA => ['<ol style="list-style-type: lower-alpha">', '</ol>'],
+    :NOTE   => ['<dl class="rdoc-list note-list">',          '</dl>'],
+    :NUMBER => ['<ol>',                                      '</ol>'],
+    :UALPHA => ['<ol style="list-style-type: upper-alpha">', '</ol>'],
   }
 
   attr_reader :res # :nodoc:
@@ -29,6 +26,12 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   attr_reader :list # :nodoc:
 
   ##
+  # The RDoc::CodeObject HTML is being generated for.  This is used to
+  # generate namespaced URI fragments
+
+  attr_accessor :code_object
+
+  ##
   # Path to this document for relative links
 
   attr_accessor :from_path
@@ -62,19 +65,31 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   ##
   # Creates a new formatter that will output HTML
 
-  def initialize markup = nil
+  def initialize options, markup = nil
     super
 
-    @th = nil
+    @code_object = nil
+    @from_path = ''
     @in_list_entry = nil
     @list = nil
-    @from_path = ''
+    @th = nil
+    @hard_break = "<br>\n"
 
     # external links
-    @markup.add_special(/((link:|https?:|mailto:|ftp:|www\.)\S+\w)/, :HYPERLINK)
+    @markup.add_special(/(?:link:|https?:|mailto:|ftp:|irc:|www\.)\S+\w/,
+                        :HYPERLINK)
+
+    # internal links
+    @markup.add_special(/rdoc-[a-z]+:\S+/, :RDOCLINK)
 
     # and links of the form  <text>[<url>]
-    @markup.add_special(/(((\{.*?\})|\b\S+?)\[\S+?\])/, :TIDYLINK)
+    @markup.add_special(/(?:
+                          \{.*?\} |   # multi-word label
+                          \b[^\s{}]+? # single-word label
+                         )
+
+                         \[\S+?\]     # link target
+                        /x, :TIDYLINK)
 
     init_tags
   end
@@ -84,6 +99,13 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # These methods handle special markup added by RDoc::Markup#add_special.
 
   ##
+  # +special+ is a <code><br></code>
+
+  def handle_special_HARD_BREAK special
+    '<br>'
+  end
+
+  ##
   # +special+ is a potential link.  The following schemes are handled:
   #
   # <tt>mailto:</tt>::
@@ -102,6 +124,39 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   end
 
   ##
+  # +special+ is an rdoc-schemed link that will be converted into a hyperlink.
+  #
+  # For the +rdoc-ref+ scheme the named reference will be returned without
+  # creating a link.
+  #
+  # For the +rdoc-label+ scheme the footnote and label prefixes are stripped
+  # when creating a link.  All other contents will be linked verbatim.
+
+  def handle_special_RDOCLINK special
+    url = special.text
+
+    case url
+    when /\Ardoc-ref:/
+      $'
+    when /\Ardoc-label:/
+      text = $'
+
+      text = case text
+             when /\Alabel-/    then $'
+             when /\Afootmark-/ then "^#{$'}"
+             when /\Afoottext-/ then "*#{$'}"
+             else                    text
+             end
+
+      gen_url url, text
+    else
+      url =~ /\Ardoc-[a-z]+:/
+
+      $'
+    end
+  end
+
+  ##
   # This +special+ is a link where the label is different from the URL
   # <tt>label[url]</tt> or <tt>{long label}[url]</tt>
 
@@ -136,21 +191,47 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   end
 
   ##
+  # Adds +block_quote+ to the output
+
+  def accept_block_quote block_quote
+    @res << "\n<blockquote>"
+
+    block_quote.parts.each do |part|
+      part.accept self
+    end
+
+    @res << "</blockquote>\n"
+  end
+
+  ##
   # Adds +paragraph+ to the output
 
-  def accept_paragraph(paragraph)
+  def accept_paragraph paragraph
     @res << "\n<p>"
-    @res << wrap(to_html(paragraph.text))
+    text = paragraph.text @hard_break
+    @res << wrap(to_html(text))
     @res << "</p>\n"
   end
 
   ##
   # Adds +verbatim+ to the output
 
-  def accept_verbatim(verbatim)
-    @res << "\n<pre>"
-    @res << CGI.escapeHTML(verbatim.text.rstrip)
-    @res << "</pre>\n"
+  def accept_verbatim verbatim
+    text = verbatim.text.rstrip
+
+    @res << if verbatim.ruby? or parseable? text then
+              begin
+                tokens = RDoc::RubyLex.tokenize text, @options
+
+                html = RDoc::TokenStream.to_html tokens
+
+                "\n<pre class=\"ruby\">#{html}</pre>\n"
+              rescue RDoc::RubyLex::Error
+                "\n<pre>#{CGI.escapeHTML text}</pre>\n"
+              end
+            else
+              "\n<pre>#{CGI.escapeHTML text}</pre>\n"
+            end
   end
 
   ##
@@ -208,12 +289,19 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   end
 
   ##
-  # Adds +heading+ to the output
+  # Adds +heading+ to the output.  The headings greater than 6 are trimmed to
+  # level 6.
+
+  def accept_heading heading
+    level = [6, heading.level].min
 
-  def accept_heading(heading)
-    @res << "\n<h#{heading.level}>"
+    label = heading.aref
+    label = [@code_object.aref, label].compact.join '-' if
+      @code_object and @code_object.respond_to? :aref
+
+    @res << "\n<h#{level} id=\"#{label}\">"
     @res << to_html(heading.text)
-    @res << "</h#{heading.level}>\n"
+    @res << "</h#{level}>\n"
   end
 
   ##
@@ -226,18 +314,22 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # :section: Utilities
 
   ##
-  # CGI escapes +text+
+  # CGI-escapes +text+
 
   def convert_string(text)
     CGI.escapeHTML text
   end
 
   ##
-  # Generate a link for +url+, labeled with +text+.  Handles the special cases
+  # Generate a link to +url+ with content +text+.  Handles the special cases
   # for img: and link: described under handle_special_HYPERLINK
 
-  def gen_url(url, text)
-    if url =~ /([A-Za-z]+):(.*)/ then
+  def gen_url url, text
+    if url =~ /^rdoc-label:([^:]*)(?::(.*))?/ then
+      type = "link"
+      path = "##{$1}"
+      id   = " id=\"#{$2}\"" if $2
+    elsif url =~ /([A-Za-z]+):(.*)/ then
       type = $1
       path = $2
     else
@@ -258,7 +350,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
        url =~ /\.(gif|png|jpg|jpeg|bmp)$/ then
       "<img src=\"#{url}\" />"
     else
-      "<a href=\"#{url}\">#{text.sub(%r{^#{type}:/*}, '')}</a>"
+      "<a#{id} href=\"#{url}\">#{text.sub(%r{^#{type}:/*}, '')}</a>"
     end
   end
 
@@ -275,9 +367,9 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
   # Maps attributes to HTML tags
 
   def init_tags
-    add_tag :BOLD, "<b>",  "</b>"
-    add_tag :TT,   "<tt>", "</tt>"
-    add_tag :EM,   "<em>", "</em>"
+    add_tag :BOLD, "<strong>", "</strong>"
+    add_tag :TT,   "<code>",   "</code>"
+    add_tag :EM,   "<em>",     "</em>"
   end
 
   ##
@@ -288,10 +380,10 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     case list_type
     when :BULLET, :LALPHA, :NUMBER, :UALPHA then
       "<li>"
-    when :LABEL then
-      "<dt>#{to_html list_item.label}</dt>\n<dd>"
-    when :NOTE then
-      "<tr><td class=\"rdoc-term\"><p>#{to_html list_item.label}</p></td>\n<td>"
+    when :LABEL, :NOTE then
+      Array(list_item.label).map do |label|
+        "<dt>#{to_html label}\n"
+      end.join << "<dd>"
     else
       raise RDoc::Error, "Invalid list type: #{list_type.inspect}"
     end
@@ -304,16 +396,22 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
     case list_type
     when :BULLET, :LALPHA, :NUMBER, :UALPHA then
       "</li>"
-    when :LABEL then
+    when :LABEL, :NOTE then
       "</dd>"
-    when :NOTE then
-      "</td></tr>"
     else
       raise RDoc::Error, "Invalid list type: #{list_type.inspect}"
     end
   end
 
   ##
+  # Returns true if Ripper is available it can create a sexp from +text+
+
+  def parseable? text
+    text =~ /\b(def|class|module|require) |=>|\{\s?\||do \|/ and
+      text !~ /<%|%>/
+  end
+
+  ##
   # Converts +item+ to HTML using RDoc::Text#to_html
 
   def to_html item
diff --git a/lib/rdoc/markup/to_html_crossref.rb b/lib/rdoc/markup/to_html_crossref.rb
index 450ea960b5..405f68c14f 100644
--- a/lib/rdoc/markup/to_html_crossref.rb
+++ b/lib/rdoc/markup/to_html_crossref.rb
@@ -1,6 +1,3 @@
-require 'rdoc/markup/to_html'
-require 'rdoc/cross_reference'
-
 ##
 # Subclass of the RDoc::Markup::ToHtml class that supports looking up method
 # names, classes, etc to create links.  RDoc::CrossReference is used to
@@ -31,21 +28,20 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
   # references are removed unless +show_hash+ is true.  Only method names
   # preceded by '#' or '::' are linked, unless +hyperlink_all+ is true.
 
-  def initialize(from_path, context, show_hash, hyperlink_all = false,
-                 markup = nil)
+  def initialize(options, from_path, context, markup = nil)
     raise ArgumentError, 'from_path cannot be nil' if from_path.nil?
-    super markup
 
-    crossref_re = hyperlink_all ? ALL_CROSSREF_REGEXP : CROSSREF_REGEXP
+    super options, markup
 
-    @cross_reference = RDoc::CrossReference.new context
+    @context       = context
+    @from_path     = from_path
+    @hyperlink_all = @options.hyperlink_all
+    @show_hash     = @options.show_hash
 
+    crossref_re = @hyperlink_all ? ALL_CROSSREF_REGEXP : CROSSREF_REGEXP
     @markup.add_special crossref_re, :CROSSREF
-    @markup.add_special(/rdoc-ref:\S+\w/, :HYPERLINK)
 
-    @from_path     = from_path
-    @hyperlink_all = hyperlink_all
-    @show_hash     = show_hash
+    @cross_reference = RDoc::CrossReference.new @context
   end
 
   ##
@@ -57,6 +53,8 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
 
     name = name[1..-1] unless @show_hash if name[0, 1] == '#'
 
+    name = "#{CGI.unescape $'} at #{$1}" if name =~ /(.*[^#:])@/
+
     text = name unless text
 
     link lookup, text
@@ -72,6 +70,8 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
   def handle_special_CROSSREF(special)
     name = special.text
 
+    return name if name =~ /@[\w-]+\.[\w-]/ # labels that look like emails
+
     unless @hyperlink_all then
       # This ensures that words entirely consisting of lowercase letters will
       # not have cross-references generated (to suppress lots of erroneous
@@ -93,6 +93,25 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
   end
 
   ##
+  # +special+ is an rdoc-schemed link that will be converted into a hyperlink.
+  # For the rdoc-ref scheme the cross-reference will be looked up and the
+  # given name will be used.
+  #
+  # All other contents are handled by
+  # {the superclass}[rdoc-ref:RDoc::Markup::ToHtml#handle_special_RDOCLINK]
+
+  def handle_special_RDOCLINK special
+    url = special.text
+
+    case url
+    when /\Ardoc-ref:/ then
+      cross_reference $'
+    else
+      super
+    end
+  end
+
+  ##
   # Generates links for <tt>rdoc-ref:</tt> scheme URLs and allows
   # RDoc::Markup::ToHtml to handle other schemes.
 
@@ -106,13 +125,31 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml
   # Creates an HTML link to +name+ with the given +text+.
 
   def link name, text
+    original_name = name
+
+    if name =~ /(.*[^#:])@/ then
+      name = $1
+      label = $'
+    end
+
     ref = @cross_reference.resolve name, text
 
+    text = ref.output_name @context if
+      RDoc::MethodAttr === ref and text == original_name
+
     case ref
     when String then
       ref
     else
-      "<a href=\"#{ref.as_href @from_path}\">#{text}</a>"
+      path = ref.as_href @from_path
+
+      if path =~ /#/ then
+        path << "-label-#{label}"
+      else
+        path << "#label-#{label}"
+      end if label
+
+      "<a href=\"#{path}\">#{text}</a>"
     end
   end
 
diff --git a/lib/rdoc/markup/to_html_snippet.rb b/lib/rdoc/markup/to_html_snippet.rb
new file mode 100644
index 0000000000..4ad0a9a4b9
--- /dev/null
+++ b/lib/rdoc/markup/to_html_snippet.rb
@@ -0,0 +1,284 @@
+##
+# Outputs RDoc markup as paragraphs with inline markup only.
+
+class RDoc::Markup::ToHtmlSnippet < RDoc::Markup::ToHtml
+
+  ##
+  # After this many characters the input will be cut off.
+
+  attr_reader :character_limit
+
+  ##
+  # The number of characters seen so far.
+
+  attr_reader :characters # :nodoc:
+
+  ##
+  # The attribute bitmask
+
+  attr_reader :mask
+
+  ##
+  # After this many paragraphs the input will be cut off.
+
+  attr_reader :paragraph_limit
+
+  ##
+  # Count of paragraphs found
+
+  attr_reader :paragraphs
+
+  ##
+  # Creates a new ToHtmlSnippet formatter that will cut off the input on the
+  # next word boundary after the given number of +characters+ or +paragraphs+
+  # of text have been encountered.
+
+  def initialize options, characters = 100, paragraphs = 3, markup = nil
+    super options, markup
+
+    @character_limit = characters
+    @paragraph_limit = paragraphs
+
+    @characters = 0
+    @mask       = 0
+    @paragraphs = 0
+
+    @markup.add_special RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
+  end
+
+  ##
+  # Adds +heading+ to the output as a paragraph
+
+  def accept_heading heading
+    @res << "<p>#{to_html heading.text}\n"
+
+    add_paragraph
+  end
+
+  ##
+  # Raw sections are untrusted and ignored
+
+  alias accept_raw ignore
+
+  ##
+  # Rules are ignored
+
+  alias accept_rule ignore
+
+  def accept_paragraph paragraph
+    para = @in_list_entry.last || "<p>"
+
+    text = paragraph.text @hard_break
+
+    @res << "#{para}#{wrap to_html text}\n"
+
+    add_paragraph
+  end
+
+  ##
+  # Finishes consumption of +list_item+
+
+  def accept_list_item_end list_item
+  end
+
+  ##
+  # Prepares the visitor for consuming +list_item+
+
+  def accept_list_item_start list_item
+    @res << list_item_start(list_item, @list.last)
+  end
+
+  ##
+  # Prepares the visitor for consuming +list+
+
+  def accept_list_start list
+    @list << list.type
+    @res << html_list_name(list.type, true)
+    @in_list_entry.push ''
+  end
+
+  ##
+  # Adds +verbatim+ to the output
+
+  def accept_verbatim verbatim
+    throw :done if @characters >= @character_limit
+    input = verbatim.text.rstrip
+
+    text = truncate input
+    text << ' ...' unless text == input
+
+    super RDoc::Markup::Verbatim.new text
+
+    add_paragraph
+  end
+
+  ##
+  # Prepares the visitor for HTML snippet generation
+
+  def start_accepting
+    super
+
+    @characters = 0
+  end
+
+  ##
+  # Removes escaping from the cross-references in +special+
+
+  def handle_special_CROSSREF special
+    special.text.sub(/\A\\/, '')
+  end
+
+  ##
+  # +special+ is a <code><br></code>
+
+  def handle_special_HARD_BREAK special
+    @characters -= 4
+    '<br>'
+  end
+
+  ##
+  # Lists are paragraphs, but notes and labels have a separator
+
+  def list_item_start list_item, list_type
+    throw :done if @characters >= @character_limit
+
+    case list_type
+    when :BULLET, :LALPHA, :NUMBER, :UALPHA then
+      "<p>"
+    when :LABEL, :NOTE then
+      labels = Array(list_item.label).map do |label|
+        to_html label
+      end.join ', '
+
+      labels << " &mdash; " unless labels.empty?
+
+      start = "<p>#{labels}"
+      @characters += 1 # try to include the label
+      start
+    else
+      raise RDoc::Error, "Invalid list type: #{list_type.inspect}"
+    end
+  end
+
+  ##
+  # Returns just the text of +link+, +url+ is only used to determine the link
+  # type.
+
+  def gen_url url, text
+    if url =~ /^rdoc-label:([^:]*)(?::(.*))?/ then
+      type = "link"
+    elsif url =~ /([A-Za-z]+):(.*)/ then
+      type = $1
+    else
+      type = "http"
+    end
+
+    if (type == "http" or type == "https" or type == "link") and
+       url =~ /\.(gif|png|jpg|jpeg|bmp)$/ then
+      ''
+    else
+      text.sub(%r%^#{type}:/*%, '')
+    end
+  end
+
+  ##
+  # In snippets, there are no lists
+
+  def html_list_name list_type, open_tag
+    ''
+  end
+
+  ##
+  # Throws +:done+ when paragraph_limit paragraphs have been encountered
+
+  def add_paragraph
+    @paragraphs += 1
+
+    throw :done if @paragraphs >= @paragraph_limit
+  end
+
+  ##
+  # Marks up +content+
+
+  def convert content
+    catch :done do
+      return super
+    end
+
+    end_accepting
+  end
+
+  ##
+  # Converts flow items +flow+
+
+  def convert_flow flow
+    throw :done if @characters >= @character_limit
+
+    res = []
+    @mask = 0
+
+    flow.each do |item|
+      case item
+      when RDoc::Markup::AttrChanger then
+        off_tags res, item
+        on_tags  res, item
+      when String then
+        text = convert_string item
+        res << truncate(text)
+      when RDoc::Markup::Special then
+        text = convert_special item
+        res << truncate(text)
+      else
+        raise "Unknown flow element: #{item.inspect}"
+      end
+
+      if @characters >= @character_limit then
+        off_tags res, RDoc::Markup::AttrChanger.new(0, @mask)
+        break
+      end
+    end
+
+    res << ' ...' if @characters >= @character_limit
+
+    res.join
+  end
+
+  ##
+  # Maintains a bitmask to allow HTML elements to be closed properly.  See
+  # RDoc::Markup::Formatter.
+
+  def on_tags res, item
+    @mask ^= item.turn_on
+
+    super
+  end
+
+  ##
+  # Maintains a bitmask to allow HTML elements to be closed properly.  See
+  # RDoc::Markup::Formatter.
+
+  def off_tags res, item
+    @mask ^= item.turn_off
+
+    super
+  end
+
+  ##
+  # Truncates +text+ at the end of the first word after the character_limit.
+
+  def truncate text
+    length = text.length
+    characters = @characters
+    @characters += length
+
+    return text if @characters < @character_limit
+
+    remaining = @character_limit - characters
+
+    text =~ /\A(.{#{remaining},}?)(\s|$)/m # TODO word-break instead of \s?
+
+    $1
+  end
+
+end
+
diff --git a/lib/rdoc/markup/to_joined_paragraph.rb b/lib/rdoc/markup/to_joined_paragraph.rb
new file mode 100644
index 0000000000..d91eb439f0
--- /dev/null
+++ b/lib/rdoc/markup/to_joined_paragraph.rb
@@ -0,0 +1,68 @@
+##
+# Joins the parts of an RDoc::Markup::Paragraph into a single String.
+#
+# This allows for easier maintenance and testing of Markdown support.
+#
+# This formatter only works on Paragraph instances.  Attempting to process
+# other markup syntax items will not work.
+
+class RDoc::Markup::ToJoinedParagraph < RDoc::Markup::Formatter
+
+  def initialize # :nodoc:
+    super nil
+  end
+
+  def start_accepting
+  end
+
+  def end_accepting
+  end
+
+  def accept_paragraph paragraph
+    parts = []
+    string = false
+
+    paragraph.parts.each do |part|
+      if String === part then
+        if string then
+          string << part
+        else
+          parts << part
+          string = part
+        end
+      else
+        parts << part
+        string = false
+      end
+    end
+
+    parts = parts.map do |part|
+      if String === part then
+        part.rstrip
+      else
+        part
+      end
+    end
+
+    # TODO use Enumerable#chunk when ruby 1.8 support is dropped
+    #parts = paragraph.parts.chunk do |part|
+    #  String === part
+    #end.map do |string, chunk|
+    #  string ? chunk.join.rstrip : chunk
+    #end.flatten
+
+    paragraph.parts.replace parts
+  end
+
+  alias accept_block_quote     ignore
+  alias accept_heading         ignore
+  alias accept_list_end        ignore
+  alias accept_list_item_end   ignore
+  alias accept_list_item_start ignore
+  alias accept_list_start      ignore
+  alias accept_raw             ignore
+  alias accept_rule            ignore
+  alias accept_verbatim        ignore
+
+end
+
diff --git a/lib/rdoc/markup/to_label.rb b/lib/rdoc/markup/to_label.rb
new file mode 100644
index 0000000000..ace89c324a
--- /dev/null
+++ b/lib/rdoc/markup/to_label.rb
@@ -0,0 +1,74 @@
+require 'cgi'
+
+##
+# Creates HTML-safe labels suitable for use in id attributes.  Tidylinks are
+# converted to their link part and cross-reference links have the suppression
+# marks removed (\\SomeClass is converted to SomeClass).
+
+class RDoc::Markup::ToLabel < RDoc::Markup::Formatter
+
+  attr_reader :res # :nodoc:
+
+  ##
+  # Creates a new formatter that will output HTML-safe labels
+
+  def initialize markup = nil
+    super nil, markup
+
+    @markup.add_special RDoc::CrossReference::CROSSREF_REGEXP, :CROSSREF
+    @markup.add_special(/(((\{.*?\})|\b\S+?)\[\S+?\])/, :TIDYLINK)
+
+    add_tag :BOLD, '', ''
+    add_tag :TT,   '', ''
+    add_tag :EM,   '', ''
+
+    @res = []
+  end
+
+  ##
+  # Converts +text+ to an HTML-safe label
+
+  def convert text
+    label = convert_flow @am.flow text
+
+    CGI.escape label
+  end
+
+  ##
+  # Converts the CROSSREF +special+ to plain text, removing the suppression
+  # marker, if any
+
+  def handle_special_CROSSREF special
+    text = special.text
+
+    text.sub(/^\\/, '')
+  end
+
+  ##
+  # Converts the TIDYLINK +special+ to just the text part
+
+  def handle_special_TIDYLINK special
+    text = special.text
+
+    return text unless text =~ /\{(.*?)\}\[(.*?)\]/ or text =~ /(\S+)\[(.*?)\]/
+
+    $1
+  end
+
+  alias accept_blank_line         ignore
+  alias accept_block_quote        ignore
+  alias accept_heading            ignore
+  alias accept_list_end           ignore
+  alias accept_list_item_end      ignore
+  alias accept_list_item_start    ignore
+  alias accept_list_start         ignore
+  alias accept_paragraph          ignore
+  alias accept_raw                ignore
+  alias accept_rule               ignore
+  alias accept_verbatim           ignore
+  alias end_accepting             ignore
+  alias handle_special_HARD_BREAK ignore
+  alias start_accepting           ignore
+
+end
+
diff --git a/lib/rdoc/markup/to_markdown.rb b/lib/rdoc/markup/to_markdown.rb
new file mode 100644
index 0000000000..e984776399
--- /dev/null
+++ b/lib/rdoc/markup/to_markdown.rb
@@ -0,0 +1,134 @@
+# :markup: markdown
+
+##
+# Outputs parsed markup as Markdown
+
+class RDoc::Markup::ToMarkdown < RDoc::Markup::ToRdoc
+
+  ##
+  # Creates a new formatter that will output Markdown format text
+
+  def initialize markup = nil
+    super
+
+    @headings[1] = ['# ',      '']
+    @headings[2] = ['## ',     '']
+    @headings[3] = ['### ',    '']
+    @headings[4] = ['#### ',   '']
+    @headings[5] = ['##### ',  '']
+    @headings[6] = ['###### ', '']
+
+    @hard_break = "  \n"
+  end
+
+  ##
+  # Maps attributes to HTML sequences
+
+  def init_tags
+    add_tag :BOLD, '**', '**'
+    add_tag :EM,   '*',  '*'
+    add_tag :TT,   '`',  '`'
+  end
+
+  ##
+  # Adds a newline to the output
+
+  def handle_special_HARD_BREAK special
+    "  \n"
+  end
+
+  ##
+  # Finishes consumption of `list`
+
+  def accept_list_end list
+    @res << "\n"
+
+    super
+  end
+
+  ##
+  # Finishes consumption of `list_item`
+
+  def accept_list_item_end list_item
+    width = case @list_type.last
+            when :BULLET then
+              4
+            when :NOTE, :LABEL then
+              use_prefix
+
+              4
+            else
+              @list_index[-1] = @list_index.last.succ
+              4
+            end
+
+    @indent -= width
+  end
+
+  ##
+  # Prepares the visitor for consuming `list_item`
+
+  def accept_list_item_start list_item
+    type = @list_type.last
+
+    case type
+    when :NOTE, :LABEL then
+      bullets = Array(list_item.label).map do |label|
+        attributes(label).strip
+      end.join "\n"
+
+      bullets << "\n:"
+
+      @prefix = ' ' * @indent
+      @indent += 4
+      @prefix << bullets + (' ' * (@indent - 1))
+    else
+      bullet = type == :BULLET ? '*' : @list_index.last.to_s + '.'
+      @prefix = (' ' * @indent) + bullet.ljust(4)
+
+      @indent += 4
+    end
+  end
+
+  ##
+  # Prepares the visitor for consuming `list`
+
+  def accept_list_start list
+    case list.type
+    when :BULLET, :LABEL, :NOTE then
+      @list_index << nil
+    when :LALPHA, :NUMBER, :UALPHA then
+      @list_index << 1
+    else
+      raise RDoc::Error, "invalid list type #{list.type}"
+    end
+
+    @list_width << 4
+    @list_type << list.type
+  end
+
+  ##
+  # Adds `rule` to the output
+
+  def accept_rule rule
+    use_prefix or @res << ' ' * @indent
+    @res << '-' * 3
+    @res << "\n"
+  end
+
+  ##
+  # Outputs `verbatim` indented 4 columns
+
+  def accept_verbatim verbatim
+    indent = ' ' * (@indent + 4)
+
+    verbatim.parts.each do |part|
+      @res << indent unless part == "\n"
+      @res << part
+    end
+
+    @res << "\n" unless @res =~ /\n\z/
+  end
+
+end
+
diff --git a/lib/rdoc/markup/to_rdoc.rb b/lib/rdoc/markup/to_rdoc.rb
index 6f2faac2f6..f16b4ed5a3 100644
--- a/lib/rdoc/markup/to_rdoc.rb
+++ b/lib/rdoc/markup/to_rdoc.rb
@@ -1,6 +1,3 @@
-require 'rdoc/markup/formatter'
-require 'rdoc/markup/inline'
-
 ##
 # Outputs RDoc markup as RDoc markup! (mostly)
 
@@ -45,7 +42,7 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
   # Creates a new formatter that will output (mostly) \RDoc markup
 
   def initialize markup = nil
-    super
+    super nil, markup
 
     @markup.add_special(/\\\S/, :SUPPRESSED_CROSSREF)
     @width = 78
@@ -60,6 +57,8 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
     @headings[4] = ['==== ',   '']
     @headings[5] = ['===== ',  '']
     @headings[6] = ['====== ', '']
+
+    @hard_break = "\n"
   end
 
   ##
@@ -79,6 +78,21 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
   end
 
   ##
+  # Adds +paragraph+ to the output
+
+  def accept_block_quote block_quote
+    @indent += 2
+
+    block_quote.parts.each do |part|
+      @prefix = '> '
+
+      part.accept self
+    end
+
+    @indent -= 2
+  end
+
+  ##
   # Adds +heading+ to the output
 
   def accept_heading heading
@@ -106,6 +120,11 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
             when :BULLET then
               2
             when :NOTE, :LABEL then
+              if @prefix then
+                @res << @prefix.strip
+                @prefix = nil
+              end
+
               @res << "\n"
               2
             else
@@ -125,10 +144,15 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
 
     case type
     when :NOTE, :LABEL then
-      bullet = attributes(list_item.label) + ":\n"
+      bullets = Array(list_item.label).map do |label|
+        attributes(label).strip
+      end.join "\n"
+
+      bullets << ":\n" unless bullets.empty?
+
       @prefix = ' ' * @indent
       @indent += 2
-      @prefix << bullet + (' ' * @indent)
+      @prefix << bullets + (' ' * @indent)
     else
       bullet = type == :BULLET ? '*' :  @list_index.last.to_s + '.'
       @prefix = (' ' * @indent) + bullet.ljust(bullet.length + 1)
@@ -168,7 +192,8 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
   # Adds +paragraph+ to the output
 
   def accept_paragraph paragraph
-    wrap attributes(paragraph.text)
+    text = paragraph.text @hard_break
+    wrap attributes text
   end
 
   ##
@@ -176,7 +201,8 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
 
   def accept_indented_paragraph paragraph
     @indent += paragraph.indent
-    wrap attributes(paragraph.text)
+    text = paragraph.text @hard_break
+    wrap attributes text
     @indent -= paragraph.indent
   end
 
@@ -235,6 +261,13 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
   end
 
   ##
+  # Adds a newline to the output
+
+  def handle_special_HARD_BREAK special
+    "\n"
+  end
+
+  ##
   # Prepares the visitor for text generation
 
   def start_accepting
@@ -252,8 +285,7 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter
   # prefix for later consumption.
 
   def use_prefix
-    prefix = @prefix
-    @prefix = nil
+    prefix, @prefix = @prefix, nil
     @res << prefix if prefix
 
     prefix
diff --git a/lib/rdoc/markup/to_table_of_contents.rb b/lib/rdoc/markup/to_table_of_contents.rb
new file mode 100644
index 0000000000..54f2d5f64f
--- /dev/null
+++ b/lib/rdoc/markup/to_table_of_contents.rb
@@ -0,0 +1,61 @@
+##
+# Extracts just the RDoc::Markup::Heading elements from a
+# RDoc::Markup::Document to help build a table of contents
+
+class RDoc::Markup::ToTableOfContents < RDoc::Markup::Formatter
+
+  @to_toc = nil
+
+  ##
+  # Singleton for table-of-contents generation
+
+  def self.to_toc
+    @to_toc ||= new
+  end
+
+  ##
+  # Output accumulator
+
+  attr_reader :res
+
+  def initialize # :nodoc:
+    super nil
+  end
+
+  ##
+  # Adds +heading+ to the table of contents
+
+  def accept_heading heading
+    @res << heading
+  end
+
+  ##
+  # Returns the table of contents
+
+  def end_accepting
+    @res
+  end
+
+  ##
+  # Prepares the visitor for text generation
+
+  def start_accepting
+    @res = []
+  end
+
+  # :stopdoc:
+  alias accept_block_quote     ignore
+  alias accept_raw             ignore
+  alias accept_rule            ignore
+  alias accept_blank_line      ignore
+  alias accept_paragraph       ignore
+  alias accept_verbatim        ignore
+  alias accept_list_end        ignore
+  alias accept_list_item_start ignore
+  alias accept_list_item_end   ignore
+  alias accept_list_end_bullet ignore
+  alias accept_list_start      ignore
+  # :startdoc:
+
+end
+
diff --git a/lib/rdoc/markup/to_test.rb b/lib/rdoc/markup/to_test.rb
index 4847fd29f7..c51f64b917 100644
--- a/lib/rdoc/markup/to_test.rb
+++ b/lib/rdoc/markup/to_test.rb
@@ -1,6 +1,3 @@
-require 'rdoc/markup'
-require 'rdoc/markup/formatter'
-
 ##
 # This Markup outputter is used for testing purposes.
 
diff --git a/lib/rdoc/markup/to_tt_only.rb b/lib/rdoc/markup/to_tt_only.rb
index 078e87db98..e2da20c6f3 100644
--- a/lib/rdoc/markup/to_tt_only.rb
+++ b/lib/rdoc/markup/to_tt_only.rb
@@ -1,6 +1,3 @@
-require 'rdoc/markup/formatter'
-require 'rdoc/markup/inline'
-
 ##
 # Extracts sections of text enclosed in plus, tt or code.  Used to discover
 # undocumented parameters.
@@ -21,12 +18,19 @@ class RDoc::Markup::ToTtOnly < RDoc::Markup::Formatter
   # Creates a new tt-only formatter.
 
   def initialize markup = nil
-    super
+    super nil, markup
 
     add_tag :TT, nil, nil
   end
 
   ##
+  # Adds tts from +block_quote+ to the output
+
+  def accept_block_quote block_quote
+    tt_sections block_quote.text
+  end
+
+  ##
   # Pops the list type for +list+ from #list_type
 
   def accept_list_end list
@@ -46,7 +50,9 @@ class RDoc::Markup::ToTtOnly < RDoc::Markup::Formatter
   def accept_list_item_start list_item
     case @list_type.last
     when :NOTE, :LABEL then
-      tt_sections(list_item.label)
+      Array(list_item.label).map do |label|
+        tt_sections label
+      end.flatten
     end
   end
 
diff --git a/lib/rdoc/markup/verbatim.rb b/lib/rdoc/markup/verbatim.rb
index 8fe2184699..3886bbe8a5 100644
--- a/lib/rdoc/markup/verbatim.rb
+++ b/lib/rdoc/markup/verbatim.rb
@@ -4,6 +4,21 @@
 class RDoc::Markup::Verbatim < RDoc::Markup::Raw
 
   ##
+  # Format of this verbatim section
+
+  attr_accessor :format
+
+  def initialize *parts # :nodoc:
+    super
+
+    @format = nil
+  end
+
+  def == other # :nodoc:
+    super and @format == other.format
+  end
+
+  ##
   # Calls #accept_verbatim on +visitor+
 
   def accept visitor
@@ -34,6 +49,29 @@ class RDoc::Markup::Verbatim < RDoc::Markup::Raw
     @parts = parts
   end
 
+  def pretty_print q # :nodoc:
+    self.class.name =~ /.*::(\w{1,4})/i
+
+    q.group 2, "[#{$1.downcase}: ", ']' do
+      if @format then
+        q.text "format: #{@format}"
+        q.breakable
+      end
+
+      q.seplist @parts do |part|
+        q.pp part
+      end
+    end
+  end
+
+  ##
+  # Is this verbatim section ruby code?
+
+  def ruby?
+    @format ||= nil # TODO for older ri data, switch the tree to marshal_dump
+    @format == :ruby
+  end
+
   ##
   # The text of the section