Import RDoc 3.5.1

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@30760 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

2 files changed, 227 insertions, 57 deletions

diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
index db37985b6d..bb015690bd 100644
--- a/lib/rdoc/parser/c.rb
+++ b/lib/rdoc/parser/c.rb
@@ -71,7 +71,7 @@ require 'rdoc/known_classes'
 #   Documentation for the named attribute.
 #
 # [call-seq:  <i>text up to an empty line</i>]
-#   Because C source doesn't give descripive names to Ruby-level parameters,
+#   Because C source doesn't give descriptive names to Ruby-level parameters,
 #   you need to document the calling sequence explicitly
 #
 # In addition, RDoc assumes by default that the C method implementing a
@@ -156,6 +156,7 @@ class RDoc::Parser::C < RDoc::Parser
       comment = strip_stars comment
       al.comment = comment
 
+      al.record_location @top_level
       class_obj.add_alias al
       @stats.add_alias al
     end
@@ -284,7 +285,7 @@ class RDoc::Parser::C < RDoc::Parser
                      \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
                      \s*(-?\w+)\s*\)
                    (?:;\s*/[*/]\s+in\s+(\w+?\.[cy]))?
-                 %xm) do |type, var_name, meth_name, meth_body, param_count, source_file|
+                 %xm) do |type, var_name, meth_name, function, param_count, source_file|
 
       # Ignore top-object and weird struct.c dynamic stuff
       next if var_name == "ruby_top_self"
@@ -293,7 +294,7 @@ class RDoc::Parser::C < RDoc::Parser
       next if var_name == "argf"   # it'd be nice to handle this one
 
       var_name = "rb_cObject" if var_name == "rb_mKernel"
-      handle_method(type, var_name, meth_name, meth_body, param_count,
+      handle_method(type, var_name, meth_name, function, param_count,
                     source_file)
     end
 
@@ -302,18 +303,19 @@ class RDoc::Parser::C < RDoc::Parser
                              \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
                              \s*(-?\w+)\s*\)
                 (?:;\s*/[*/]\s+in\s+(\w+?\.[cy]))?
-                %xm) do |meth_name, meth_body, param_count, source_file|
-      handle_method("method", "rb_mKernel", meth_name,
-                    meth_body, param_count, source_file)
+                %xm) do |meth_name, function, param_count, source_file|
+      handle_method("method", "rb_mKernel", meth_name, function, param_count,
+                    source_file)
     end
 
     @content.scan(/define_filetest_function\s*\(
                      \s*"([^"]+)",
                      \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
-                     \s*(-?\w+)\s*\)/xm) do |meth_name, meth_body, param_count|
+                     \s*(-?\w+)\s*\)/xm) do |meth_name, function, param_count|
 
-      handle_method("method", "rb_mFileTest", meth_name, meth_body, param_count)
-      handle_method("singleton_method", "rb_cFile", meth_name, meth_body, param_count)
+      handle_method("method", "rb_mFileTest", meth_name, function, param_count)
+      handle_method("singleton_method", "rb_cFile", meth_name, function,
+                    param_count)
     end
   end
 
@@ -369,21 +371,20 @@ class RDoc::Parser::C < RDoc::Parser
   ##
   # Find the C code corresponding to a Ruby method
 
-  def find_body(class_name, meth_name, meth_obj, body, quiet = false)
-    case body
+  def find_body class_name, meth_name, meth_obj, file_content, quiet = false
+    case file_content
     when %r%((?>/\*.*?\*/\s*)?)
             ((?:(?:static|SWIGINTERN)\s+)?
              (?:intern\s+)?VALUE\s+#{meth_name}
-             \s*(\([^)]*\))([^;]|$))%xm then
+             \s*(\(.*?\))([^;]|$))%xm then
       comment = $1
-      body_text = $2
+      body = $2
+      offset = $~.offset(2).first
 
       remove_private_comments comment if comment
 
-      # see if we can find the whole body
-
-      re = Regexp.escape(body_text) + '[^(]*^\{.*?^\}'
-      body_text = $& if /#{re}/m =~ body
+      # try to find the whole body
+      body = $& if /#{Regexp.escape body}[^(]*?\{.*?^\}/m =~ file_content
 
       # The comment block may have been overridden with a 'Document-method'
       # block. This happens in the interpreter when multiple methods are
@@ -399,38 +400,53 @@ class RDoc::Parser::C < RDoc::Parser
       #meth_obj.params = params
       meth_obj.start_collecting_tokens
       tk = RDoc::RubyToken::Token.new nil, 1, 1
-      tk.set_text body_text
+      tk.set_text body
       meth_obj.add_token tk
       meth_obj.comment = strip_stars comment
-    when %r%((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+#{meth_name}\s+(\w+))%m
+      meth_obj.offset  = offset
+      meth_obj.line    = file_content[0, offset].count("\n") + 1
+
+      body
+    when %r%((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+#{meth_name}\s+(\w+))%m then
       comment = $1
-      body_text = $2
-      find_body class_name, $3, meth_obj, body, true
+      body = $2
+      offset = $~.offset(2).first
+
+      find_body class_name, $3, meth_obj, file_content, true
       find_modifiers comment, meth_obj
 
       meth_obj.start_collecting_tokens
       tk = RDoc::RubyToken::Token.new nil, 1, 1
-      tk.set_text body_text
+      tk.set_text body
       meth_obj.add_token tk
       meth_obj.comment = strip_stars(comment) + meth_obj.comment.to_s
-    when %r%^\s*\#\s*define\s+#{meth_name}\s+(\w+)%m
-      unless find_body(class_name, $1, meth_obj, body, true)
-        warn "No definition for #{meth_name}" if @options.verbosity > 1
-        return false
-      end
+      meth_obj.offset  = offset
+      meth_obj.line    = file_content[0, offset].count("\n") + 1
+
+      body
+    when %r%^\s*\#\s*define\s+#{meth_name}\s+(\w+)%m then
+      # with no comment we hope the aliased definition has it and use it's
+      # definition
+
+      body = find_body(class_name, $1, meth_obj, file_content, true)
+
+      return body if body
+
+      warn "No definition for #{meth_name}" if @options.verbosity > 1
+      false
     else # No body, but might still have an override comment
       comment = find_override_comment class_name, meth_obj.name
 
-      if comment
+      if comment then
         find_modifiers comment, meth_obj
         meth_obj.comment = strip_stars comment
+
+        ''
       else
         warn "No definition for #{meth_name}" if @options.verbosity > 1
-        return false
+        false
       end
     end
-
-    true
   end
 
   ##
@@ -618,6 +634,7 @@ class RDoc::Parser::C < RDoc::Parser
 
     attr = RDoc::Attr.new '', name, rw, comment
 
+    attr.record_location @top_level
     class_obj.add_attribute attr
     @stats.add_attribute attr
   end
@@ -732,6 +749,7 @@ class RDoc::Parser::C < RDoc::Parser
       con = RDoc::Constant.new const_name, definition, comment
     end
 
+    con.record_location @top_level
     @stats.add_constant con
     class_obj.add_constant con
   end
@@ -748,7 +766,7 @@ class RDoc::Parser::C < RDoc::Parser
   # to +var_name+.  +type+ is the type of method definition function used.
   # +singleton_method+ and +module_function+ create a singleton method.
 
-  def handle_method(type, var_name, meth_name, meth_body, param_count,
+  def handle_method(type, var_name, meth_name, function, param_count,
                     source_file = nil)
     singleton = false
     class_name = @known_classes[var_name]
@@ -770,33 +788,37 @@ class RDoc::Parser::C < RDoc::Parser
       end
 
       meth_obj = RDoc::AnyMethod.new '', meth_name
+      meth_obj.c_function = function
       meth_obj.singleton =
         singleton || %w[singleton_method module_function].include?(type)
 
       p_count = Integer(param_count) rescue -1
 
-      if p_count < 0 then
-        meth_obj.params = "(...)"
-      elsif p_count == 0
-        meth_obj.params = "()"
-      else
-        meth_obj.params = "(" + (1..p_count).map{|i| "p#{i}"}.join(", ") + ")"
-      end
-
       if source_file then
         file_name = File.join @file_dir, source_file
 
         if File.exist? file_name then
-          body = (@@known_bodies[file_name] ||= File.read(file_name))
+          file_content = (@@known_bodies[file_name] ||= File.read(file_name))
         else
           warn "unknown source #{source_file} for #{meth_name} in #{@file_name}"
         end
       else
-        body = @content
+        file_content = @content
       end
 
-      if find_body(class_name, meth_body, meth_obj, body) and
-         meth_obj.document_self then
+      body = find_body class_name, function, meth_obj, file_content
+
+      if body and meth_obj.document_self then
+        meth_obj.params = if p_count < -1 then # -2 is Array
+                            '(*args)'
+                          elsif p_count == -1 then # argc, argv
+                            rb_scan_args body
+                          else
+                            "(#{(1..p_count).map { |i| "p#{i}" }.join ', '})"
+                          end
+
+
+        meth_obj.record_location @top_level
         class_obj.add_method meth_obj
         @stats.add_method meth_obj
         meth_obj.visibility = :private if 'private_method' == type
@@ -857,6 +879,90 @@ class RDoc::Parser::C < RDoc::Parser
   end
 
   ##
+  # Extracts parameters from the +method_body+ and returns a method
+  # parameter string.  Follows 1.9.3dev's scan-arg-spec, see README.EXT
+
+  def rb_scan_args method_body
+    method_body =~ /rb_scan_args\((.*?)\)/m
+    return '(*args)' unless $1
+
+    $1.split(/,/)[2] =~ /"(.*?)"/ # format argument
+    format = $1.split(//)
+
+    lead = opt = trail = 0
+
+    if format.first =~ /\d/ then
+      lead = $&.to_i
+      format.shift
+      if format.first =~ /\d/ then
+        opt = $&.to_i
+        format.shift
+        if format.first =~ /\d/ then
+          trail = $&.to_i
+          format.shift
+          block_arg = true
+        end
+      end
+    end
+
+    if format.first == '*' and not block_arg then
+      var = true
+      format.shift
+      if format.first =~ /\d/ then
+        trail = $&.to_i
+        format.shift
+      end
+    end
+
+    if format.first == ':' then
+      hash = true
+      format.shift
+    end
+
+    if format.first == '&' then
+      block = true
+      format.shift
+    end
+
+    # if the format string is not empty there's a bug in the C code, ignore it
+
+    args = []
+    position = 1
+
+    (1...(position + lead)).each do |index|
+      args << "p#{index}"
+    end
+
+    position += lead
+
+    (position...(position + opt)).each do |index|
+      args << "p#{index} = v#{index}"
+    end
+
+    position += opt
+
+    if var then
+      args << '*args'
+      position += 1
+    end
+
+    (position...(position + trail)).each do |index|
+      args << "p#{index}"
+    end
+
+    position += trail
+
+    if hash then
+      args << "p#{position} = {}"
+      position += 1
+    end
+
+    args << '&block' if block
+
+    "(#{args.join ', '})"
+  end
+
+  ##
   # Removes lines that are commented out that might otherwise get picked up
   # when scanning for classes and methods
 
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
index bec88bc52d..bcf0ac1150 100644
--- a/lib/rdoc/parser/ruby.rb
+++ b/lib/rdoc/parser/ruby.rb
@@ -174,6 +174,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     @scanner.exception_on_syntax_error = false
     @prev_seek = nil
 
+    @encoding = nil
+    @encoding = @options.encoding if Object.const_defined? :Encoding
+
     reset
   end
 
@@ -183,6 +186,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def collect_first_comment
     skip_tkspace
     comment = ''
+    comment.force_encoding @encoding if @encoding
     first_line = true
 
     tk = get_tk
@@ -449,6 +453,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # +comment+.
 
   def parse_attr(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
     args = parse_symbol_arg 1
     if args.size > 0 then
       name = args[0]
@@ -464,6 +471,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
       att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
       att.record_location @top_level
+      att.offset = offset
+      att.line   = line_no
 
       read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
 
@@ -480,6 +489,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # comment for each to +comment+.
 
   def parse_attr_accessor(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
     args = parse_symbol_arg
     rw = "?"
 
@@ -498,6 +510,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
     for name in args
       att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
       att.record_location @top_level
+      att.offset = offset
+      att.line   = line_no
 
       context.add_attribute att
       @stats.add_attribute att
@@ -508,6 +522,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses an +alias+ in +context+ with +comment+
 
   def parse_alias(context, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
     skip_tkspace
 
     if TkLPAREN === peek_tk then
@@ -534,6 +551,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
     al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
                          single == SINGLE)
     al.record_location @top_level
+    al.offset = offset
+    al.line   = line_no
 
     read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
     context.add_alias al if al.document_self
@@ -586,6 +605,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a class in +context+ with +comment+
 
   def parse_class(container, single, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
     declaration_context = container
     container, name_t, given_name = get_class_or_module container
 
@@ -606,6 +628,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
       read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
       cls.record_location @top_level
+      cls.offset = offset
+      cls.line   = line_no
+
       cls.comment = comment if cls.document_self
 
       @top_level.add_to_classes_or_modules cls
@@ -622,6 +647,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
         unless other then
           other = container.add_module RDoc::NormalModule, name
           other.record_location @top_level
+          other.offset  = offset
+          other.line    = line_no
+
           other.comment = comment
         end
 
@@ -639,7 +667,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
         read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
         parse_statements(other, SINGLE)
       end
-
     else
       warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
     end
@@ -649,6 +676,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a constant in +context+ with +comment+
 
   def parse_constant(container, tk, comment)
+    offset  = tk.seek
+    line_no = tk.line_no
+
     name = tk.name
     skip_tkspace false
 
@@ -658,7 +688,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     unless TkASSIGN === eq_tk then
       unget_tk eq_tk
-      return
+      return false
     end
 
     nest = 0
@@ -669,7 +699,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     if TkGT === tk then
       unget_tk tk
       unget_tk eq_tk
-      return
+      return false
     end
 
     rhs_name = ''
@@ -698,7 +728,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
                   container.find_module_named rhs_name
                 end
 
-          container.add_module_alias mod, name if mod
+          container.add_module_alias mod, name, @top_level if mod
           get_tk # TkNL
           break
         end
@@ -721,10 +751,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     con = RDoc::Constant.new name, res, comment
     con.record_location @top_level
+    con.offset = offset
+    con.line   = line_no
     read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
 
     @stats.add_constant con
     container.add_constant con if con.document_self
+    true
   end
 
   ##
@@ -732,8 +765,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # :method: or :attr: directives in +comment+.
 
   def parse_comment(container, tk, comment)
-    line_no = tk.line_no
     column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
 
     singleton = !!comment.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3')
 
@@ -744,6 +778,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
       meth = RDoc::GhostMethod.new get_tkread, name
       meth.record_location @top_level
       meth.singleton = singleton
+      meth.offset    = offset
+      meth.line      = line_no
 
       meth.start_collecting_tokens
       indent = TkSPACE.new nil, 1, 1
@@ -764,7 +800,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       meth.comment = comment
 
       @stats.add_method meth
-    elsif comment.sub!(/# +:?(attr(_reader|_writer|_accessor)?:) *(\S*).*?\n/i, '') then
+    elsif comment.sub!(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then
       rw = case $1
            when 'attr_reader' then 'R'
            when 'attr_writer' then 'W'
@@ -776,6 +812,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
       # TODO authorize 'singleton-attr...'?
       att = RDoc::Attr.new get_tkread, name, rw, comment
       att.record_location @top_level
+      att.offset    = offset
+      att.line      = line_no
 
       container.add_attribute att
 
@@ -870,8 +908,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a meta-programmed method
 
   def parse_meta_method(container, single, tk, comment)
-    line_no = tk.line_no
     column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
 
     start_collecting_tokens
     add_token tk
@@ -903,6 +942,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     meth = RDoc::MetaMethod.new get_tkread, name
     meth.record_location @top_level
+    meth.offset = offset
+    meth.line   = line_no
     meth.singleton = singleton
 
     remove_token_listener self
@@ -933,6 +974,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
           break unless last_tk and TkCOMMA === last_tk
         when TkSPACE then
           # expression continues
+        when TkDO then
+          unget_tk tk
+          parse_statements container, single, meth
+          break
         else
           last_tk = tk
         end
@@ -951,8 +996,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
     added_container = nil
     meth = nil
     name = nil
-    line_no = tk.line_no
     column  = tk.char_no
+    offset  = tk.seek
+    line_no = tk.line_no
 
     start_collecting_tokens
     add_token tk
@@ -1044,6 +1090,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
     end
 
     meth.record_location @top_level
+    meth.offset = offset
+    meth.line   = line_no
 
     meth.start_collecting_tokens
     indent = TkSPACE.new nil, 1, 1
@@ -1216,6 +1264,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_statements(container, single = NORMAL, current_method = nil,
                        comment = '')
+    comment.force_encoding @encoding if @encoding
+
     nest = 1
     save_visibility = container.visibility
 
@@ -1223,6 +1273,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     while tk = get_tk do
       keep_comment = false
+      try_parse_comment = false
 
       non_comment_seen = true unless TkCOMMENT === tk
 
@@ -1238,6 +1289,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
               comment.empty?
 
             comment = ''
+            comment.force_encoding @encoding if @encoding
           end
 
           while TkCOMMENT === tk do
@@ -1287,7 +1339,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
       when TkCONSTANT then
         if container.document_self then
-          parse_constant container, tk, comment
+          if not parse_constant container, tk, comment then
+            try_parse_comment = true
+          end
         end
 
       when TkALIAS then
@@ -1365,13 +1419,20 @@ class RDoc::Parser::Ruby < RDoc::Parser
           return
         end
       else
+        try_parse_comment = nest == 1
+      end
+
+      if try_parse_comment then
         non_comment_seen = parse_comment container, tk, comment unless
           comment.empty?
 
-        comment = ''
+        keep_comment = false
       end
 
-      comment = '' unless keep_comment
+      unless keep_comment then
+        comment = ''
+        comment.force_encoding @encoding if @encoding
+      end
 
       begin
         get_tkread
@@ -1584,8 +1645,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Removes private comments from +comment+
 
   def remove_private_comments(comment)
-    comment.gsub!(/^#--\n.*?^#\+\+\n?/m, '')
-    comment.sub!(/^#--\n.*\n?/m, '')
+    empty = ''
+    empty.force_encoding comment.encoding if Object.const_defined? :Encoding
+
+    comment.gsub!(/^#--.*?^#\+\+\n?/m, empty)
+    comment.sub!(/^#--.*/m, '')
   end
 
   ##