summaryrefslogtreecommitdiff
path: root/lib/rdoc/parser
diff options
context:
space:
mode:
Diffstat (limited to 'lib/rdoc/parser')
-rw-r--r--lib/rdoc/parser/c.rb1268
-rw-r--r--lib/rdoc/parser/changelog.rb204
-rw-r--r--lib/rdoc/parser/markdown.rb24
-rw-r--r--lib/rdoc/parser/rd.rb23
-rw-r--r--lib/rdoc/parser/ripper_state_lex.rb605
-rw-r--r--lib/rdoc/parser/ruby.rb2232
-rw-r--r--lib/rdoc/parser/ruby_tools.rb159
-rw-r--r--lib/rdoc/parser/simple.rb61
-rw-r--r--lib/rdoc/parser/text.rb12
9 files changed, 4588 insertions, 0 deletions
diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
new file mode 100644
index 0000000000..183538d54b
--- /dev/null
+++ b/lib/rdoc/parser/c.rb
@@ -0,0 +1,1268 @@
+# frozen_string_literal: true
+require 'tsort'
+
+##
+# RDoc::Parser::C attempts to parse C extension files. It looks for
+# the standard patterns that you find in extensions: <tt>rb_define_class,
+# rb_define_method</tt> and so on. It tries to find the corresponding
+# C source for the methods and extract comments, but if we fail
+# we don't worry too much.
+#
+# The comments associated with a Ruby method are extracted from the C
+# comment block associated with the routine that _implements_ that
+# method, that is to say the method whose name is given in the
+# <tt>rb_define_method</tt> call. For example, you might write:
+#
+# /*
+# * Returns a new array that is a one-dimensional flattening of this
+# * array (recursively). That is, for every element that is an array,
+# * extract its elements into the new array.
+# *
+# * s = [ 1, 2, 3 ] #=> [1, 2, 3]
+# * t = [ 4, 5, 6, [7, 8] ] #=> [4, 5, 6, [7, 8]]
+# * a = [ s, t, 9, 10 ] #=> [[1, 2, 3], [4, 5, 6, [7, 8]], 9, 10]
+# * a.flatten #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
+# */
+# static VALUE
+# rb_ary_flatten(ary)
+# VALUE ary;
+# {
+# ary = rb_obj_dup(ary);
+# rb_ary_flatten_bang(ary);
+# return ary;
+# }
+#
+# ...
+#
+# void
+# Init_Array()
+# {
+# ...
+# rb_define_method(rb_cArray, "flatten", rb_ary_flatten, 0);
+#
+# Here RDoc will determine from the rb_define_method line that there's a
+# method called "flatten" in class Array, and will look for the implementation
+# in the method rb_ary_flatten. It will then use the comment from that
+# method in the HTML output. This method must be in the same source file
+# as the rb_define_method.
+#
+# The comment blocks may include special directives:
+#
+# [Document-class: +name+]
+# Documentation for the named class.
+#
+# [Document-module: +name+]
+# Documentation for the named module.
+#
+# [Document-const: +name+]
+# Documentation for the named +rb_define_const+.
+#
+# Constant values can be supplied on the first line of the comment like so:
+#
+# /* 300: The highest possible score in bowling */
+# rb_define_const(cFoo, "PERFECT", INT2FIX(300));
+#
+# The value can contain internal colons so long as they are escaped with a \
+#
+# [Document-global: +name+]
+# Documentation for the named +rb_define_global_const+
+#
+# [Document-variable: +name+]
+# Documentation for the named +rb_define_variable+
+#
+# [Document-method: +method_name+]
+# Documentation for the named method. Use this when the method name is
+# unambiguous.
+#
+# [Document-method: <tt>ClassName::method_name<tt>]
+# Documentation for a singleton method in the given class. Use this when
+# the method name alone is ambiguous.
+#
+# [Document-method: <tt>ClassName#method_name<tt>]
+# Documentation for a instance method in the given class. Use this when the
+# method name alone is ambiguous.
+#
+# [Document-attr: +name+]
+# Documentation for the named attribute.
+#
+# [call-seq: <i>text up to an empty line</i>]
+# Because C source doesn't give descriptive names to Ruby-level parameters,
+# you need to document the calling sequence explicitly
+#
+# In addition, RDoc assumes by default that the C method implementing a
+# Ruby function is in the same source file as the rb_define_method call.
+# If this isn't the case, add the comment:
+#
+# rb_define_method(....); // in filename
+#
+# As an example, we might have an extension that defines multiple classes
+# in its Init_xxx method. We could document them using
+#
+# /*
+# * Document-class: MyClass
+# *
+# * Encapsulate the writing and reading of the configuration
+# * file. ...
+# */
+#
+# /*
+# * Document-method: read_value
+# *
+# * call-seq:
+# * cfg.read_value(key) -> value
+# * cfg.read_value(key} { |key| } -> value
+# *
+# * Return the value corresponding to +key+ from the configuration.
+# * In the second form, if the key isn't found, invoke the
+# * block and return its value.
+# */
+
+class RDoc::Parser::C < RDoc::Parser
+
+ parse_files_matching(/\.(?:([CcHh])\1?|c([+xp])\2|y)\z/)
+
+ include RDoc::Text
+
+ ##
+ # Maps C variable names to names of Ruby classes or modules
+
+ attr_reader :classes
+
+ ##
+ # C file the parser is parsing
+
+ attr_accessor :content
+
+ ##
+ # Dependencies from a missing enclosing class to the classes in
+ # missing_dependencies that depend upon it.
+
+ attr_reader :enclosure_dependencies
+
+ ##
+ # Maps C variable names to names of Ruby classes (and singleton classes)
+
+ attr_reader :known_classes
+
+ ##
+ # Classes found while parsing the C file that were not yet registered due to
+ # a missing enclosing class. These are processed by do_missing
+
+ attr_reader :missing_dependencies
+
+ ##
+ # Maps C variable names to names of Ruby singleton classes
+
+ attr_reader :singleton_classes
+
+ ##
+ # The TopLevel items in the parsed file belong to
+
+ attr_reader :top_level
+
+ ##
+ # Prepares for parsing a C file. See RDoc::Parser#initialize for details on
+ # the arguments.
+
+ def initialize top_level, file_name, content, options, stats
+ super
+
+ @known_classes = RDoc::KNOWN_CLASSES.dup
+ @content = handle_tab_width handle_ifdefs_in @content
+ @file_dir = File.dirname @file_name
+
+ @classes = load_variable_map :c_class_variables
+ @singleton_classes = load_variable_map :c_singleton_class_variables
+
+ # class_variable => { function => [method, ...] }
+ @methods = Hash.new { |h, f| h[f] = Hash.new { |i, m| i[m] = [] } }
+
+ # missing variable => [handle_class_module arguments]
+ @missing_dependencies = {}
+
+ # missing enclosure variable => [dependent handle_class_module arguments]
+ @enclosure_dependencies = Hash.new { |h, k| h[k] = [] }
+ @enclosure_dependencies.instance_variable_set :@missing_dependencies,
+ @missing_dependencies
+
+ @enclosure_dependencies.extend TSort
+
+ def @enclosure_dependencies.tsort_each_node &block
+ each_key(&block)
+ rescue TSort::Cyclic => e
+ cycle_vars = e.message.scan(/"(.*?)"/).flatten
+
+ cycle = cycle_vars.sort.map do |var_name|
+ delete var_name
+
+ var_name, type, mod_name, = @missing_dependencies[var_name]
+
+ "#{type} #{mod_name} (#{var_name})"
+ end.join ', '
+
+ warn "Unable to create #{cycle} due to a cyclic class or module creation"
+
+ retry
+ end
+
+ def @enclosure_dependencies.tsort_each_child node, &block
+ fetch(node, []).each(&block)
+ end
+ end
+
+ ##
+ # Removes duplicate call-seq entries for methods using the same
+ # implementation.
+
+ def deduplicate_call_seq
+ @methods.each do |var_name, functions|
+ class_name = @known_classes[var_name]
+ class_obj = find_class var_name, class_name
+
+ functions.each_value do |method_names|
+ next if method_names.length == 1
+
+ method_names.each do |method_name|
+ deduplicate_method_name class_obj, method_name
+ end
+ end
+ end
+ end
+
+ ##
+ # If two ruby methods share a C implementation (and comment) this
+ # deduplicates the examples in the call_seq for the method to reduce
+ # confusion in the output.
+
+ def deduplicate_method_name class_obj, method_name # :nodoc:
+ return unless
+ method = class_obj.method_list.find { |m| m.name == method_name }
+ return unless call_seq = method.call_seq
+
+ method_name = method_name[0, 1] if method_name =~ /\A\[/
+
+ entries = call_seq.split "\n"
+
+ matching = entries.select do |entry|
+ entry =~ /^\w*\.?#{Regexp.escape method_name}/ or
+ entry =~ /\s#{Regexp.escape method_name}\s/
+ end
+
+ method.call_seq = matching.join "\n"
+ end
+
+ ##
+ # Scans #content for rb_define_alias
+
+ def do_aliases
+ @content.scan(/rb_define_alias\s*\(
+ \s*(\w+),
+ \s*"(.+?)",
+ \s*"(.+?)"
+ \s*\)/xm) do |var_name, new_name, old_name|
+ class_name = @known_classes[var_name]
+
+ unless class_name then
+ @options.warn "Enclosing class or module %p for alias %s %s is not known" % [
+ var_name, new_name, old_name]
+ next
+ end
+
+ class_obj = find_class var_name, class_name
+
+ al = RDoc::Alias.new '', old_name, new_name, ''
+ al.singleton = @singleton_classes.key? var_name
+
+ comment = find_alias_comment var_name, new_name, old_name
+
+ comment.normalize
+
+ al.comment = comment
+
+ al.record_location @top_level
+
+ class_obj.add_alias al
+ @stats.add_alias al
+ end
+ end
+
+ ##
+ # Scans #content for rb_attr and rb_define_attr
+
+ def do_attrs
+ @content.scan(/rb_attr\s*\(
+ \s*(\w+),
+ \s*([\w"()]+),
+ \s*([01]),
+ \s*([01]),
+ \s*\w+\);/xm) do |var_name, attr_name, read, write|
+ handle_attr var_name, attr_name, read, write
+ end
+
+ @content.scan(%r%rb_define_attr\(
+ \s*([\w\.]+),
+ \s*"([^"]+)",
+ \s*(\d+),
+ \s*(\d+)\s*\);
+ %xm) do |var_name, attr_name, read, write|
+ handle_attr var_name, attr_name, read, write
+ end
+ end
+
+ ##
+ # Scans #content for boot_defclass
+
+ def do_boot_defclass
+ @content.scan(/(\w+)\s*=\s*boot_defclass\s*\(\s*"(\w+?)",\s*(\w+?)\s*\)/) do
+ |var_name, class_name, parent|
+ parent = nil if parent == "0"
+ handle_class_module(var_name, :class, class_name, parent, nil)
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_class, boot_defclass, rb_define_class_under
+ # and rb_singleton_class
+
+ def do_classes
+ do_boot_defclass
+ do_define_class
+ do_define_class_under
+ do_singleton_class
+ do_struct_define_without_accessor
+ end
+
+ ##
+ # Scans #content for rb_define_variable, rb_define_readonly_variable,
+ # rb_define_const and rb_define_global_const
+
+ def do_constants
+ @content.scan(%r%\Wrb_define_
+ ( variable |
+ readonly_variable |
+ const |
+ global_const )
+ \s*\(
+ (?:\s*(\w+),)?
+ \s*"(\w+)",
+ \s*(.*?)\s*\)\s*;
+ %xm) do |type, var_name, const_name, definition|
+ var_name = "rb_cObject" if !var_name or var_name == "rb_mKernel"
+ handle_constants type, var_name, const_name, definition
+ end
+
+ @content.scan(%r%
+ \Wrb_curses_define_const
+ \s*\(
+ \s*
+ (\w+)
+ \s*
+ \)
+ \s*;%xm) do |consts|
+ const = consts.first
+
+ handle_constants 'const', 'mCurses', const, "UINT2NUM(#{const})"
+ end
+
+ @content.scan(%r%
+ \Wrb_file_const
+ \s*\(
+ \s*
+ "([^"]+)",
+ \s*
+ (.*?)
+ \s*
+ \)
+ \s*;%xm) do |name, value|
+ handle_constants 'const', 'rb_mFConst', name, value
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_class
+
+ def do_define_class
+ # The '.' lets us handle SWIG-generated files
+ @content.scan(/([\w\.]+)\s* = \s*rb_define_class\s*
+ \(
+ \s*"(\w+)",
+ \s*(\w+)\s*
+ \)/mx) do |var_name, class_name, parent|
+ handle_class_module(var_name, :class, class_name, parent, nil)
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_class_under
+
+ def do_define_class_under
+ @content.scan(/([\w\.]+)\s* = # var_name
+ \s*rb_define_class_under\s*
+ \(
+ \s* (\w+), # under
+ \s* "(\w+)", # class_name
+ \s*
+ (?:
+ ([\w\*\s\(\)\.\->]+) | # parent_name
+ rb_path2class\("([\w:]+)"\) # path
+ )
+ \s*
+ \)
+ /mx) do |var_name, under, class_name, parent_name, path|
+ parent = path || parent_name
+
+ handle_class_module var_name, :class, class_name, parent, under
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_module
+
+ def do_define_module
+ @content.scan(/(\w+)\s* = \s*rb_define_module\s*\(\s*"(\w+)"\s*\)/mx) do
+ |var_name, class_name|
+ handle_class_module(var_name, :module, class_name, nil, nil)
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_module_under
+
+ def do_define_module_under
+ @content.scan(/(\w+)\s* = \s*rb_define_module_under\s*
+ \(
+ \s*(\w+),
+ \s*"(\w+)"
+ \s*\)/mx) do |var_name, in_module, class_name|
+ handle_class_module(var_name, :module, class_name, nil, in_module)
+ end
+ end
+
+ ##
+ # Scans #content for rb_include_module
+
+ def do_includes
+ @content.scan(/rb_include_module\s*\(\s*(\w+?),\s*(\w+?)\s*\)/) do |c,m|
+ next unless cls = @classes[c]
+ m = @known_classes[m] || m
+
+ comment = RDoc::Comment.new '', @top_level
+ incl = cls.add_include RDoc::Include.new(m, comment)
+ incl.record_location @top_level
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_method, rb_define_singleton_method,
+ # rb_define_module_function, rb_define_private_method,
+ # rb_define_global_function and define_filetest_function
+
+ def do_methods
+ @content.scan(%r%rb_define_
+ (
+ singleton_method |
+ method |
+ module_function |
+ private_method
+ )
+ \s*\(\s*([\w\.]+),
+ \s*"([^"]+)",
+ \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\(|\(METHOD\))?(\w+)\)?,
+ \s*(-?\w+)\s*\)
+ (?:;\s*/[*/]\s+in\s+(\w+?\.(?:cpp|c|y)))?
+ %xm) do |type, var_name, meth_name, function, param_count, source_file|
+
+ # Ignore top-object and weird struct.c dynamic stuff
+ next if var_name == "ruby_top_self"
+ next if var_name == "nstr"
+
+ var_name = "rb_cObject" if var_name == "rb_mKernel"
+ handle_method(type, var_name, meth_name, function, param_count,
+ source_file)
+ end
+
+ @content.scan(%r%rb_define_global_function\s*\(
+ \s*"([^"]+)",
+ \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
+ \s*(-?\w+)\s*\)
+ (?:;\s*/[*/]\s+in\s+(\w+?\.[cy]))?
+ %xm) do |meth_name, function, param_count, source_file|
+ handle_method("method", "rb_mKernel", meth_name, function, param_count,
+ source_file)
+ end
+
+ @content.scan(/define_filetest_function\s*\(
+ \s*"([^"]+)",
+ \s*(?:RUBY_METHOD_FUNC\(|VALUEFUNC\()?(\w+)\)?,
+ \s*(-?\w+)\s*\)/xm) do |meth_name, function, param_count|
+
+ handle_method("method", "rb_mFileTest", meth_name, function, param_count)
+ handle_method("singleton_method", "rb_cFile", meth_name, function,
+ param_count)
+ end
+ end
+
+ ##
+ # Creates classes and module that were missing were defined due to the file
+ # order being different than the declaration order.
+
+ def do_missing
+ return if @missing_dependencies.empty?
+
+ @enclosure_dependencies.tsort.each do |in_module|
+ arguments = @missing_dependencies.delete in_module
+
+ next unless arguments # dependency on existing class
+
+ handle_class_module(*arguments)
+ end
+ end
+
+ ##
+ # Scans #content for rb_define_module and rb_define_module_under
+
+ def do_modules
+ do_define_module
+ do_define_module_under
+ end
+
+ ##
+ # Scans #content for rb_singleton_class
+
+ def do_singleton_class
+ @content.scan(/([\w\.]+)\s* = \s*rb_singleton_class\s*
+ \(
+ \s*(\w+)
+ \s*\)/mx) do |sclass_var, class_var|
+ handle_singleton sclass_var, class_var
+ end
+ end
+
+ ##
+ # Scans #content for struct_define_without_accessor
+
+ def do_struct_define_without_accessor
+ @content.scan(/([\w\.]+)\s* = \s*rb_struct_define_without_accessor\s*
+ \(
+ \s*"(\w+)", # Class name
+ \s*(\w+), # Parent class
+ \s*\w+, # Allocation function
+ (\s*"\w+",)* # Attributes
+ \s*NULL
+ \)/mx) do |var_name, class_name, parent|
+ handle_class_module(var_name, :class, class_name, parent, nil)
+ end
+ end
+
+ ##
+ # Finds the comment for an alias on +class_name+ from +new_name+ to
+ # +old_name+
+
+ def find_alias_comment class_name, new_name, old_name
+ content =~ %r%((?>/\*.*?\*/\s+))
+ rb_define_alias\(\s*#{Regexp.escape class_name}\s*,
+ \s*"#{Regexp.escape new_name}"\s*,
+ \s*"#{Regexp.escape old_name}"\s*\);%xm
+
+ RDoc::Comment.new($1 || '', @top_level)
+ end
+
+ ##
+ # Finds a comment for rb_define_attr, rb_attr or Document-attr.
+ #
+ # +var_name+ is the C class variable the attribute is defined on.
+ # +attr_name+ is the attribute's name.
+ #
+ # +read+ and +write+ are the read/write flags ('1' or '0'). Either both or
+ # neither must be provided.
+
+ def find_attr_comment var_name, attr_name, read = nil, write = nil
+ attr_name = Regexp.escape attr_name
+
+ rw = if read and write then
+ /\s*#{read}\s*,\s*#{write}\s*/xm
+ else
+ /.*?/m
+ end
+
+ comment = if @content =~ %r%((?>/\*.*?\*/\s+))
+ rb_define_attr\((?:\s*#{var_name},)?\s*
+ "#{attr_name}"\s*,
+ #{rw}\)\s*;%xm then
+ $1
+ elsif @content =~ %r%((?>/\*.*?\*/\s+))
+ rb_attr\(\s*#{var_name}\s*,
+ \s*#{attr_name}\s*,
+ #{rw},.*?\)\s*;%xm then
+ $1
+ elsif @content =~ %r%(/\*.*?(?:\s*\*\s*)?)
+ Document-attr:\s#{attr_name}\s*?\n
+ ((?>(.|\n)*?\*/))%x then
+ "#{$1}\n#{$2}"
+ else
+ ''
+ end
+
+ RDoc::Comment.new comment, @top_level
+ end
+
+ ##
+ # Generate a Ruby-method table
+
+ def gen_body_table file_content
+ table = {}
+ file_content.scan(%r{
+ ((?>/\*.*?\*/\s*)?)
+ ((?:(?:\w+)\s+)?
+ (?:intern\s+)?VALUE\s+(\w+)
+ \s*(?:\([^)]*\))(?:[^;]|$))
+ | ((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+(\w+)\s+(\w+))
+ | ^\s*\#\s*define\s+(\w+)\s+(\w+)
+ }xm) do
+ case
+ when $1
+ table[$3] = [:func_def, $1, $2, $~.offset(2)] if !table[$3] || table[$3][0] != :func_def
+ when $4
+ table[$6] = [:macro_def, $4, $5, $~.offset(5), $7] if !table[$6] || table[$6][0] == :macro_alias
+ when $8
+ table[$8] ||= [:macro_alias, $9]
+ end
+ end
+ table
+ end
+
+ ##
+ # Find the C code corresponding to a Ruby method
+
+ def find_body class_name, meth_name, meth_obj, file_content, quiet = false
+ if file_content
+ @body_table ||= {}
+ @body_table[file_content] ||= gen_body_table file_content
+ type, *args = @body_table[file_content][meth_name]
+ end
+
+ case type
+ when :func_def
+ comment = RDoc::Comment.new args[0], @top_level
+ body = args[1]
+ offset, = args[2]
+
+ comment.remove_private if comment
+
+ # try to find the whole body
+ body = $& if /#{Regexp.escape body}[^(]*?\{.*?^\}/m =~ file_content
+
+ # The comment block may have been overridden with a 'Document-method'
+ # block. This happens in the interpreter when multiple methods are
+ # vectored through to the same C method but those methods are logically
+ # distinct (for example Kernel.hash and Kernel.object_id share the same
+ # implementation
+
+ override_comment = find_override_comment class_name, meth_obj
+ comment = override_comment if override_comment
+
+ comment.normalize
+ find_modifiers comment, meth_obj if comment
+
+ #meth_obj.params = params
+ meth_obj.start_collecting_tokens
+ tk = { :line_no => 1, :char_no => 1, :text => body }
+ meth_obj.add_token tk
+ meth_obj.comment = comment
+ meth_obj.line = file_content[0, offset].count("\n") + 1
+
+ body
+ when :macro_def
+ comment = RDoc::Comment.new args[0], @top_level
+ body = args[1]
+ offset, = args[2]
+
+ find_body class_name, args[3], meth_obj, file_content, true
+
+ comment.normalize
+ find_modifiers comment, meth_obj
+
+ meth_obj.start_collecting_tokens
+ tk = { :line_no => 1, :char_no => 1, :text => body }
+ meth_obj.add_token tk
+ meth_obj.comment = comment
+ meth_obj.line = file_content[0, offset].count("\n") + 1
+
+ body
+ when :macro_alias
+ # with no comment we hope the aliased definition has it and use it's
+ # definition
+
+ body = find_body(class_name, args[0], meth_obj, file_content, true)
+
+ return body if body
+
+ @options.warn "No definition for #{meth_name}"
+ false
+ else # No body, but might still have an override comment
+ comment = find_override_comment class_name, meth_obj
+
+ if comment then
+ comment.normalize
+ find_modifiers comment, meth_obj
+ meth_obj.comment = comment
+
+ ''
+ else
+ @options.warn "No definition for #{meth_name}"
+ false
+ end
+ end
+ end
+
+ ##
+ # Finds a RDoc::NormalClass or RDoc::NormalModule for +raw_name+
+
+ def find_class(raw_name, name)
+ unless @classes[raw_name]
+ if raw_name =~ /^rb_m/
+ container = @top_level.add_module RDoc::NormalModule, name
+ else
+ container = @top_level.add_class RDoc::NormalClass, name
+ end
+
+ container.record_location @top_level
+ @classes[raw_name] = container
+ end
+ @classes[raw_name]
+ end
+
+ ##
+ # Look for class or module documentation above Init_+class_name+(void),
+ # in a Document-class +class_name+ (or module) comment or above an
+ # rb_define_class (or module). If a comment is supplied above a matching
+ # Init_ and a rb_define_class the Init_ comment is used.
+ #
+ # /*
+ # * This is a comment for Foo
+ # */
+ # Init_Foo(void) {
+ # VALUE cFoo = rb_define_class("Foo", rb_cObject);
+ # }
+ #
+ # /*
+ # * Document-class: Foo
+ # * This is a comment for Foo
+ # */
+ # Init_foo(void) {
+ # VALUE cFoo = rb_define_class("Foo", rb_cObject);
+ # }
+ #
+ # /*
+ # * This is a comment for Foo
+ # */
+ # VALUE cFoo = rb_define_class("Foo", rb_cObject);
+
+ def find_class_comment class_name, class_mod
+ comment = nil
+
+ if @content =~ %r%
+ ((?>/\*.*?\*/\s+))
+ (static\s+)?
+ void\s+
+ Init_#{class_name}\s*(?:_\(\s*)?\(\s*(?:void\s*)?\)%xmi then
+ comment = $1.sub(%r%Document-(?:class|module):\s+#{class_name}%, '')
+ elsif @content =~ %r%Document-(?:class|module):\s+#{class_name}\s*?
+ (?:<\s+[:,\w]+)?\n((?>.*?\*/))%xm then
+ comment = "/*\n#{$1}"
+ elsif @content =~ %r%((?>/\*.*?\*/\s+))
+ ([\w\.\s]+\s* = \s+)?rb_define_(class|module)[\t (]*?"(#{class_name})"%xm then
+ comment = $1
+ elsif @content =~ %r%((?>/\*.*?\*/\s+))
+ ([\w\. \t]+ = \s+)?rb_define_(class|module)_under[\t\w, (]*?"(#{class_name.split('::').last})"%xm then
+ comment = $1
+ else
+ comment = ''
+ end
+
+ comment = RDoc::Comment.new comment, @top_level
+ comment.normalize
+
+ look_for_directives_in class_mod, comment
+
+ class_mod.add_comment comment, @top_level
+ end
+
+ ##
+ # Generate a const table
+
+ def gen_const_table file_content
+ table = {}
+ @content.scan(%r{
+ ((?>^\s*/\*.*?\*/\s+))
+ rb_define_(\w+)\((?:\s*(?:\w+),)?\s*
+ "(\w+)"\s*,
+ .*?\)\s*;
+ | Document-(?:const|global|variable):\s
+ ((?:\w+::)*\w+)
+ \s*?\n((?>.*?\*/))
+ }mxi) do
+ case
+ when $1 then table[[$2, $3]] = $1
+ when $4 then table[$4] = "/*\n" + $5
+ end
+ end
+ table
+ end
+
+ ##
+ # Finds a comment matching +type+ and +const_name+ either above the
+ # comment or in the matching Document- section.
+
+ def find_const_comment(type, const_name, class_name = nil)
+ @const_table ||= {}
+ @const_table[@content] ||= gen_const_table @content
+ table = @const_table[@content]
+
+ comment =
+ table[[type, const_name]] ||
+ (class_name && table[class_name + "::" + const_name]) ||
+ table[const_name] ||
+ ''
+
+ RDoc::Comment.new comment, @top_level
+ end
+
+ ##
+ # Handles modifiers in +comment+ and updates +meth_obj+ as appropriate.
+
+ def find_modifiers comment, meth_obj
+ comment.normalize
+ comment.extract_call_seq meth_obj
+
+ look_for_directives_in meth_obj, comment
+ end
+
+ ##
+ # Finds a <tt>Document-method</tt> override for +meth_obj+ on +class_name+
+
+ def find_override_comment class_name, meth_obj
+ name = Regexp.escape meth_obj.name
+ prefix = Regexp.escape meth_obj.name_prefix
+
+ comment = if @content =~ %r%Document-method:
+ \s+#{class_name}#{prefix}#{name}
+ \s*?\n((?>.*?\*/))%xm then
+ "/*#{$1}"
+ elsif @content =~ %r%Document-method:
+ \s#{name}\s*?\n((?>.*?\*/))%xm then
+ "/*#{$1}"
+ end
+
+ return unless comment
+
+ RDoc::Comment.new comment, @top_level
+ end
+
+ ##
+ # Creates a new RDoc::Attr +attr_name+ on class +var_name+ that is either
+ # +read+, +write+ or both
+
+ def handle_attr(var_name, attr_name, read, write)
+ rw = ''
+ rw += 'R' if '1' == read
+ rw += 'W' if '1' == write
+
+ class_name = @known_classes[var_name]
+
+ return unless class_name
+
+ class_obj = find_class var_name, class_name
+
+ return unless class_obj
+
+ comment = find_attr_comment var_name, attr_name
+ comment.normalize
+
+ name = attr_name.gsub(/rb_intern(?:_const)?\("([^"]+)"\)/, '\1')
+
+ attr = RDoc::Attr.new '', name, rw, comment
+
+ attr.record_location @top_level
+ class_obj.add_attribute attr
+ @stats.add_attribute attr
+ end
+
+ ##
+ # Creates a new RDoc::NormalClass or RDoc::NormalModule based on +type+
+ # named +class_name+ in +parent+ which was assigned to the C +var_name+.
+
+ def handle_class_module(var_name, type, class_name, parent, in_module)
+ parent_name = @known_classes[parent] || parent
+
+ if in_module then
+ enclosure = @classes[in_module] || @store.find_c_enclosure(in_module)
+
+ if enclosure.nil? and enclosure = @known_classes[in_module] then
+ enc_type = /^rb_m/ =~ in_module ? :module : :class
+ handle_class_module in_module, enc_type, enclosure, nil, nil
+ enclosure = @classes[in_module]
+ end
+
+ unless enclosure then
+ @enclosure_dependencies[in_module] << var_name
+ @missing_dependencies[var_name] =
+ [var_name, type, class_name, parent, in_module]
+
+ return
+ end
+ else
+ enclosure = @top_level
+ end
+
+ if type == :class then
+ full_name = if RDoc::ClassModule === enclosure then
+ enclosure.full_name + "::#{class_name}"
+ else
+ class_name
+ end
+
+ if @content =~ %r%Document-class:\s+#{full_name}\s*<\s+([:,\w]+)% then
+ parent_name = $1
+ end
+
+ cm = enclosure.add_class RDoc::NormalClass, class_name, parent_name
+ else
+ cm = enclosure.add_module RDoc::NormalModule, class_name
+ end
+
+ cm.record_location enclosure.top_level
+
+ find_class_comment cm.full_name, cm
+
+ case cm
+ when RDoc::NormalClass
+ @stats.add_class cm
+ when RDoc::NormalModule
+ @stats.add_module cm
+ end
+
+ @classes[var_name] = cm
+ @known_classes[var_name] = cm.full_name
+ @store.add_c_enclosure var_name, cm
+ end
+
+ ##
+ # Adds constants. By providing some_value: at the start of the comment you
+ # can override the C value of the comment to give a friendly definition.
+ #
+ # /* 300: The perfect score in bowling */
+ # rb_define_const(cFoo, "PERFECT", INT2FIX(300);
+ #
+ # Will override <tt>INT2FIX(300)</tt> with the value +300+ in the output
+ # RDoc. Values may include quotes and escaped colons (\:).
+
+ def handle_constants(type, var_name, const_name, definition)
+ class_name = @known_classes[var_name]
+
+ return unless class_name
+
+ class_obj = find_class var_name, class_name
+
+ unless class_obj then
+ @options.warn 'Enclosing class or module %p is not known' % [const_name]
+ return
+ end
+
+ comment = find_const_comment type, const_name, class_name
+ comment.normalize
+
+ # In the case of rb_define_const, the definition and comment are in
+ # "/* definition: comment */" form. The literal ':' and '\' characters
+ # can be escaped with a backslash.
+ if type.downcase == 'const' then
+ no_match, new_definition, new_comment = comment.text.split(/(\A.*):/)
+
+ if no_match and no_match.empty? then
+ if new_definition.empty? then # Default to literal C definition
+ new_definition = definition
+ else
+ new_definition = new_definition.gsub("\:", ":")
+ new_definition = new_definition.gsub("\\", '\\')
+ end
+
+ new_definition.sub!(/\A(\s+)/, '')
+
+ new_comment = "#{$1}#{new_comment.lstrip}"
+
+ new_comment = RDoc::Comment.new new_comment, @top_level
+
+ con = RDoc::Constant.new const_name, new_definition, new_comment
+ else
+ con = RDoc::Constant.new const_name, definition, comment
+ end
+ else
+ con = RDoc::Constant.new const_name, definition, comment
+ end
+
+ con.record_location @top_level
+ @stats.add_constant con
+ class_obj.add_constant con
+ end
+
+ ##
+ # Removes #ifdefs that would otherwise confuse us
+
+ def handle_ifdefs_in(body)
+ body.gsub(/^#ifdef HAVE_PROTOTYPES.*?#else.*?\n(.*?)#endif.*?\n/m, '\1')
+ end
+
+ ##
+ # Adds an RDoc::AnyMethod +meth_name+ defined on a class or module assigned
+ # to +var_name+. +type+ is the type of method definition function used.
+ # +singleton_method+ and +module_function+ create a singleton method.
+
+ def handle_method(type, var_name, meth_name, function, param_count,
+ source_file = nil)
+ class_name = @known_classes[var_name]
+ singleton = @singleton_classes.key? var_name
+
+ @methods[var_name][function] << meth_name
+
+ return unless class_name
+
+ class_obj = find_class var_name, class_name
+
+ if class_obj then
+ if meth_name == 'initialize' then
+ meth_name = 'new'
+ singleton = true
+ type = 'method' # force public
+ end
+
+ meth_obj = RDoc::AnyMethod.new '', meth_name
+ meth_obj.c_function = function
+ meth_obj.singleton =
+ singleton || %w[singleton_method module_function].include?(type)
+
+ p_count = Integer(param_count) rescue -1
+
+ if source_file then
+ file_name = File.join @file_dir, source_file
+
+ if File.exist? file_name then
+ file_content = File.read file_name
+ else
+ @options.warn "unknown source #{source_file} for #{meth_name} in #{@file_name}"
+ end
+ else
+ file_content = @content
+ end
+
+ body = find_body class_name, function, meth_obj, file_content
+
+ if body and meth_obj.document_self then
+ meth_obj.params = if p_count < -1 then # -2 is Array
+ '(*args)'
+ elsif p_count == -1 then # argc, argv
+ rb_scan_args body
+ else
+ "(#{(1..p_count).map { |i| "p#{i}" }.join ', '})"
+ end
+
+
+ meth_obj.record_location @top_level
+ class_obj.add_method meth_obj
+ @stats.add_method meth_obj
+ meth_obj.visibility = :private if 'private_method' == type
+ end
+ end
+ end
+
+ ##
+ # Registers a singleton class +sclass_var+ as a singleton of +class_var+
+
+ def handle_singleton sclass_var, class_var
+ class_name = @known_classes[class_var]
+
+ @known_classes[sclass_var] = class_name
+ @singleton_classes[sclass_var] = class_name
+ end
+
+ ##
+ # Normalizes tabs in +body+
+
+ def handle_tab_width(body)
+ if /\t/ =~ body
+ tab_width = @options.tab_width
+ body.split(/\n/).map do |line|
+ 1 while line.gsub!(/\t+/) do
+ ' ' * (tab_width * $&.length - $`.length % tab_width)
+ end && $~
+ line
+ end.join "\n"
+ else
+ body
+ end
+ end
+
+ ##
+ # Loads the variable map with the given +name+ from the RDoc::Store, if
+ # present.
+
+ def load_variable_map map_name
+ return {} unless files = @store.cache[map_name]
+ return {} unless name_map = files[@file_name]
+
+ class_map = {}
+
+ name_map.each do |variable, name|
+ next unless mod = @store.find_class_or_module(name)
+
+ class_map[variable] = if map_name == :c_class_variables then
+ mod
+ else
+ name
+ end
+ @known_classes[variable] = name
+ end
+
+ class_map
+ end
+
+ ##
+ # Look for directives in a normal comment block:
+ #
+ # /*
+ # * :title: My Awesome Project
+ # */
+ #
+ # This method modifies the +comment+
+
+ def look_for_directives_in context, comment
+ @preprocess.handle comment, context do |directive, param|
+ case directive
+ when 'main' then
+ @options.main_page = param
+ ''
+ when 'title' then
+ @options.default_title = param if @options.respond_to? :default_title=
+ ''
+ end
+ end
+
+ comment
+ end
+
+ ##
+ # Extracts parameters from the +method_body+ and returns a method
+ # parameter string. Follows 1.9.3dev's scan-arg-spec, see README.EXT
+
+ def rb_scan_args method_body
+ method_body =~ /rb_scan_args\((.*?)\)/m
+ return '(*args)' unless $1
+
+ $1.split(/,/)[2] =~ /"(.*?)"/ # format argument
+ format = $1.split(//)
+
+ lead = opt = trail = 0
+
+ if format.first =~ /\d/ then
+ lead = $&.to_i
+ format.shift
+ if format.first =~ /\d/ then
+ opt = $&.to_i
+ format.shift
+ if format.first =~ /\d/ then
+ trail = $&.to_i
+ format.shift
+ block_arg = true
+ end
+ end
+ end
+
+ if format.first == '*' and not block_arg then
+ var = true
+ format.shift
+ if format.first =~ /\d/ then
+ trail = $&.to_i
+ format.shift
+ end
+ end
+
+ if format.first == ':' then
+ hash = true
+ format.shift
+ end
+
+ if format.first == '&' then
+ block = true
+ format.shift
+ end
+
+ # if the format string is not empty there's a bug in the C code, ignore it
+
+ args = []
+ position = 1
+
+ (1...(position + lead)).each do |index|
+ args << "p#{index}"
+ end
+
+ position += lead
+
+ (position...(position + opt)).each do |index|
+ args << "p#{index} = v#{index}"
+ end
+
+ position += opt
+
+ if var then
+ args << '*args'
+ position += 1
+ end
+
+ (position...(position + trail)).each do |index|
+ args << "p#{index}"
+ end
+
+ position += trail
+
+ if hash then
+ args << "p#{position} = {}"
+ end
+
+ args << '&block' if block
+
+ "(#{args.join ', '})"
+ end
+
+ ##
+ # Removes lines that are commented out that might otherwise get picked up
+ # when scanning for classes and methods
+
+ def remove_commented_out_lines
+ @content = @content.gsub(%r%//.*rb_define_%, '//')
+ end
+
+ ##
+ # Extracts the classes, modules, methods, attributes, constants and aliases
+ # from a C file and returns an RDoc::TopLevel for this file
+
+ def scan
+ remove_commented_out_lines
+
+ do_modules
+ do_classes
+ do_missing
+
+ do_constants
+ do_methods
+ do_includes
+ do_aliases
+ do_attrs
+
+ deduplicate_call_seq
+
+ @store.add_c_variables self
+
+ @top_level
+ end
+
+end
+
diff --git a/lib/rdoc/parser/changelog.rb b/lib/rdoc/parser/changelog.rb
new file mode 100644
index 0000000000..167892f543
--- /dev/null
+++ b/lib/rdoc/parser/changelog.rb
@@ -0,0 +1,204 @@
+# frozen_string_literal: true
+require 'time'
+
+##
+# A ChangeLog file parser.
+#
+# This parser converts a ChangeLog into an RDoc::Markup::Document. When
+# viewed as HTML a ChangeLog page will have an entry for each day's entries in
+# the sidebar table of contents.
+#
+# This parser is meant to parse the MRI ChangeLog, but can be used to parse any
+# {GNU style Change
+# Log}[http://www.gnu.org/prep/standards/html_node/Style-of-Change-Logs.html].
+
+class RDoc::Parser::ChangeLog < RDoc::Parser
+
+ include RDoc::Parser::Text
+
+ parse_files_matching(/(\/|\\|\A)ChangeLog[^\/\\]*\z/)
+
+ ##
+ # Attaches the +continuation+ of the previous line to the +entry_body+.
+ #
+ # Continued function listings are joined together as a single entry.
+ # Continued descriptions are joined to make a single paragraph.
+
+ def continue_entry_body entry_body, continuation
+ return unless last = entry_body.last
+
+ if last =~ /\)\s*\z/ and continuation =~ /\A\(/ then
+ last.sub!(/\)\s*\z/, ',')
+ continuation = continuation.sub(/\A\(/, '')
+ end
+
+ if last =~ /\s\z/ then
+ last << continuation
+ else
+ last << ' ' + continuation
+ end
+ end
+
+ ##
+ # Creates an RDoc::Markup::Document given the +groups+ of ChangeLog entries.
+
+ def create_document groups
+ doc = RDoc::Markup::Document.new
+ doc.omit_headings_below = 2
+ doc.file = @top_level
+
+ doc << RDoc::Markup::Heading.new(1, File.basename(@file_name))
+ doc << RDoc::Markup::BlankLine.new
+
+ groups.sort_by do |day,| day end.reverse_each do |day, entries|
+ doc << RDoc::Markup::Heading.new(2, day.dup)
+ doc << RDoc::Markup::BlankLine.new
+
+ doc.concat create_entries entries
+ end
+
+ doc
+ end
+
+ ##
+ # Returns a list of ChangeLog entries an RDoc::Markup nodes for the given
+ # +entries+.
+
+ def create_entries entries
+ out = []
+
+ entries.each do |entry, items|
+ out << RDoc::Markup::Heading.new(3, entry)
+ out << RDoc::Markup::BlankLine.new
+
+ out << create_items(items)
+ end
+
+ out
+ end
+
+ ##
+ # Returns an RDoc::Markup::List containing the given +items+ in the
+ # ChangeLog
+
+ def create_items items
+ list = RDoc::Markup::List.new :NOTE
+
+ items.each do |item|
+ item =~ /\A(.*?(?:\([^)]+\))?):\s*/
+
+ title = $1
+ body = $'
+
+ paragraph = RDoc::Markup::Paragraph.new body
+ list_item = RDoc::Markup::ListItem.new title, paragraph
+ list << list_item
+ end
+
+ list
+ end
+
+ ##
+ # Groups +entries+ by date.
+
+ def group_entries entries
+ @time_cache ||= {}
+ entries.group_by do |title, _|
+ begin
+ time = @time_cache[title]
+ (time || Time.parse(title)).strftime '%Y-%m-%d'
+ rescue NoMethodError, ArgumentError
+ time, = title.split ' ', 2
+ Time.parse(time).strftime '%Y-%m-%d'
+ end
+ end
+ end
+
+ ##
+ # Parses the entries in the ChangeLog.
+ #
+ # Returns an Array of each ChangeLog entry in order of parsing.
+ #
+ # A ChangeLog entry is an Array containing the ChangeLog title (date and
+ # committer) and an Array of ChangeLog items (file and function changed with
+ # description).
+ #
+ # An example result would be:
+ #
+ # [ 'Tue Dec 4 08:33:46 2012 Eric Hodel <drbrain@segment7.net>',
+ # [ 'README.EXT: Converted to RDoc format',
+ # 'README.EXT.ja: ditto']]
+
+ def parse_entries
+ @time_cache ||= {}
+ entries = []
+ entry_name = nil
+ entry_body = []
+
+ @content.each_line do |line|
+ case line
+ when /^\s*$/ then
+ next
+ when /^\w.*/ then
+ entries << [entry_name, entry_body] if entry_name
+
+ entry_name = $&
+
+ begin
+ time = Time.parse entry_name
+ @time_cache[entry_name] = time
+ # HACK Ruby 1.8 does not raise ArgumentError for Time.parse "Other"
+ entry_name = nil unless entry_name =~ /#{time.year}/
+ rescue NoMethodError
+ # HACK Ruby 2.1.2 and earlier raises NoMethodError if time part is absent
+ entry_name.split ' ', 2
+ rescue ArgumentError
+ if /out of range/ =~ $!.message
+ Time.parse(entry_name.split(' ', 2)[0]) rescue entry_name = nil
+ else
+ entry_name = nil
+ end
+ end
+
+ entry_body = []
+ when /^(\t| {8})?\*\s*(.*)/ then # "\t* file.c (func): ..."
+ entry_body << $2.dup
+ when /^(\t| {8})?\s*(\(.*)/ then # "\t(func): ..."
+ entry = $2
+
+ if entry_body.last =~ /:/ then
+ entry_body << entry.dup
+ else
+ continue_entry_body entry_body, entry
+ end
+ when /^(\t| {8})?\s*(.*)/ then
+ continue_entry_body entry_body, $2
+ end
+ end
+
+ entries << [entry_name, entry_body] if entry_name
+
+ entries.reject! do |(entry,_)|
+ entry == nil
+ end
+
+ entries
+ end
+
+ ##
+ # Converts the ChangeLog into an RDoc::Markup::Document
+
+ def scan
+ @time_cache = {}
+ entries = parse_entries
+ grouped_entries = group_entries entries
+
+ doc = create_document grouped_entries
+
+ @top_level.comment = doc
+
+ @top_level
+ end
+
+end
+
diff --git a/lib/rdoc/parser/markdown.rb b/lib/rdoc/parser/markdown.rb
new file mode 100644
index 0000000000..9ff478f872
--- /dev/null
+++ b/lib/rdoc/parser/markdown.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+##
+# Parse a Markdown format file. The parsed RDoc::Markup::Document is attached
+# as a file comment.
+
+class RDoc::Parser::Markdown < RDoc::Parser
+
+ include RDoc::Parser::Text
+
+ parse_files_matching(/\.(md|markdown)(?:\.[^.]+)?$/)
+
+ ##
+ # Creates an Markdown-format TopLevel for the given file.
+
+ def scan
+ comment = RDoc::Comment.new @content, @top_level
+ comment.format = 'markdown'
+
+ @top_level.comment = comment
+ end
+
+end
+
+
diff --git a/lib/rdoc/parser/rd.rb b/lib/rdoc/parser/rd.rb
new file mode 100644
index 0000000000..25f5711731
--- /dev/null
+++ b/lib/rdoc/parser/rd.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+##
+# Parse a RD format file. The parsed RDoc::Markup::Document is attached as a
+# file comment.
+
+class RDoc::Parser::RD < RDoc::Parser
+
+ include RDoc::Parser::Text
+
+ parse_files_matching(/\.rd(?:\.[^.]+)?$/)
+
+ ##
+ # Creates an rd-format TopLevel for the given file.
+
+ def scan
+ comment = RDoc::Comment.new @content, @top_level
+ comment.format = 'rd'
+
+ @top_level.comment = comment
+ end
+
+end
+
diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb
new file mode 100644
index 0000000000..b7cec84bfc
--- /dev/null
+++ b/lib/rdoc/parser/ripper_state_lex.rb
@@ -0,0 +1,605 @@
+require 'ripper'
+
+class RDoc::RipperStateLex
+ # TODO: Remove this constants after Ruby 2.4 EOL
+ RIPPER_HAS_LEX_STATE = Ripper::Filter.method_defined?(:state)
+
+ EXPR_NONE = 0
+ EXPR_BEG = 1
+ EXPR_END = 2
+ EXPR_ENDARG = 4
+ EXPR_ENDFN = 8
+ EXPR_ARG = 16
+ EXPR_CMDARG = 32
+ EXPR_MID = 64
+ EXPR_FNAME = 128
+ EXPR_DOT = 256
+ EXPR_CLASS = 512
+ EXPR_LABEL = 1024
+ EXPR_LABELED = 2048
+ EXPR_FITEM = 4096
+ EXPR_VALUE = EXPR_BEG
+ EXPR_BEG_ANY = (EXPR_BEG | EXPR_MID | EXPR_CLASS)
+ EXPR_ARG_ANY = (EXPR_ARG | EXPR_CMDARG)
+ EXPR_END_ANY = (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)
+
+ class InnerStateLex < Ripper::Filter
+ attr_accessor :lex_state
+
+ def initialize(code)
+ @lex_state = EXPR_BEG
+ @in_fname = false
+ @continue = false
+ reset
+ super(code)
+ end
+
+ def reset
+ @command_start = false
+ @cmd_state = @command_start
+ end
+
+ def on_nl(tok, data)
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @continue = true
+ else
+ @continue = false
+ @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_ignored_nl(tok, data)
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @continue = true
+ else
+ @continue = false
+ @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_op(tok, data)
+ case tok
+ when '&', '|', '!', '!=', '!~'
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_BEG
+ end
+ when '<<'
+ # TODO next token?
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_BEG
+ end
+ when '?'
+ @lex_state = EXPR_BEG
+ when '&&', '||', '+=', '-=', '*=', '**=',
+ '&=', '|=', '^=', '<<=', '>>=', '||=', '&&='
+ @lex_state = EXPR_BEG
+ else
+ case @lex_state
+ when EXPR_FNAME, EXPR_DOT
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_BEG
+ end
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_kw(tok, data)
+ case tok
+ when 'class'
+ @lex_state = EXPR_CLASS
+ @in_fname = true
+ when 'def'
+ @lex_state = EXPR_FNAME
+ @continue = true
+ @in_fname = true
+ when 'if', 'unless', 'while', 'until'
+ if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
+ @lex_state = EXPR_BEG | EXPR_LABEL
+ else
+ @lex_state = EXPR_BEG
+ end
+ when 'begin'
+ @lex_state = EXPR_BEG
+ else
+ if @lex_state == EXPR_FNAME
+ @lex_state = EXPR_END
+ else
+ @lex_state = EXPR_END
+ end
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_tstring_beg(tok, data)
+ @lex_state = EXPR_BEG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_tstring_end(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_CHAR(tok, data)
+ @lex_state = EXPR_END
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_period(tok, data)
+ @lex_state = EXPR_DOT
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_int(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_float(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_rational(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_imaginary(tok, data)
+ @lex_state = EXPR_END | EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_symbeg(tok, data)
+ @lex_state = EXPR_FNAME
+ @continue = true
+ @in_fname = true
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ private def on_variables(event, tok, data)
+ if @in_fname
+ @lex_state = EXPR_ENDFN
+ @in_fname = false
+ @continue = false
+ elsif @continue
+ case @lex_state
+ when EXPR_DOT
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_ENDFN
+ @continue = false
+ end
+ else
+ @lex_state = EXPR_CMDARG
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+ end
+
+ def on_ident(tok, data)
+ on_variables(__method__, tok, data)
+ end
+
+ def on_ivar(tok, data)
+ @lex_state = EXPR_END
+ on_variables(__method__, tok, data)
+ end
+
+ def on_cvar(tok, data)
+ @lex_state = EXPR_END
+ on_variables(__method__, tok, data)
+ end
+
+ def on_gvar(tok, data)
+ @lex_state = EXPR_END
+ on_variables(__method__, tok, data)
+ end
+
+ def on_backref(tok, data)
+ @lex_state = EXPR_END
+ on_variables(__method__, tok, data)
+ end
+
+ def on_lparen(tok, data)
+ @lex_state = EXPR_LABEL | EXPR_BEG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_rparen(tok, data)
+ @lex_state = EXPR_ENDFN
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_lbrace(tok, data)
+ @lex_state = EXPR_LABEL | EXPR_BEG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_rbrace(tok, data)
+ @lex_state = EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_lbracket(tok, data)
+ @lex_state = EXPR_LABEL | EXPR_BEG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_rbracket(tok, data)
+ @lex_state = EXPR_ENDARG
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_const(tok, data)
+ case @lex_state
+ when EXPR_FNAME
+ @lex_state = EXPR_ENDFN
+ when EXPR_CLASS
+ @lex_state = EXPR_ARG
+ else
+ @lex_state = EXPR_CMDARG
+ end
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_sp(tok, data)
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_comma(tok, data)
+ @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_comment(tok, data)
+ @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_ignored_sp(tok, data)
+ @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ end
+
+ def on_heredoc_end(tok, data)
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+ @lex_state = EXPR_BEG
+ end
+
+ def on_default(event, tok, data)
+ reset
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+ end
+
+ def each(&block)
+ @callback = block
+ parse
+ end
+ end unless RIPPER_HAS_LEX_STATE
+
+ class InnerStateLex < Ripper::Filter
+ def initialize(code)
+ super(code)
+ end
+
+ def on_default(event, tok, data)
+ @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => state})
+ end
+
+ def each(&block)
+ @callback = block
+ parse
+ end
+ end if RIPPER_HAS_LEX_STATE
+
+ def get_squashed_tk
+ if @buf.empty?
+ tk = @inner_lex_enumerator.next
+ else
+ tk = @buf.shift
+ end
+ case tk[:kind]
+ when :on_symbeg then
+ tk = get_symbol_tk(tk)
+ when :on_tstring_beg then
+ tk = get_string_tk(tk)
+ when :on_backtick then
+ if (tk[:state] & (EXPR_FNAME | EXPR_ENDFN)) != 0
+ @inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
+ tk[:kind] = :on_ident
+ tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
+ else
+ tk = get_string_tk(tk)
+ end
+ when :on_regexp_beg then
+ tk = get_regexp_tk(tk)
+ when :on_embdoc_beg then
+ tk = get_embdoc_tk(tk)
+ when :on_heredoc_beg then
+ @heredoc_queue << retrieve_heredoc_info(tk)
+ @inner_lex.lex_state = EXPR_END unless RIPPER_HAS_LEX_STATE
+ when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
+ unless @heredoc_queue.empty?
+ get_heredoc_tk(*@heredoc_queue.shift)
+ end
+ when :on_words_beg then
+ tk = get_words_tk(tk)
+ when :on_qwords_beg then
+ tk = get_words_tk(tk)
+ when :on_symbols_beg then
+ tk = get_words_tk(tk)
+ when :on_qsymbols_beg then
+ tk = get_words_tk(tk)
+ when :on_op then
+ if '&.' == tk[:text]
+ tk[:kind] = :on_period
+ else
+ tk = get_op_tk(tk)
+ end
+ end
+ tk
+ end
+
+ private def get_symbol_tk(tk)
+ is_symbol = true
+ symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol }
+ if ":'" == tk[:text] or ':"' == tk[:text]
+ tk1 = get_string_tk(tk)
+ symbol_tk[:text] = tk1[:text]
+ symbol_tk[:state] = tk1[:state]
+ else
+ case (tk1 = get_squashed_tk)[:kind]
+ when :on_ident
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_tstring_content
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end
+ when :on_tstring_end
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_op
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_ivar
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_cvar
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_gvar
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_const
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ when :on_kw
+ symbol_tk[:text] = ":#{tk1[:text]}"
+ symbol_tk[:state] = tk1[:state]
+ else
+ is_symbol = false
+ tk = tk1
+ end
+ end
+ if is_symbol
+ tk = symbol_tk
+ end
+ tk
+ end
+
+ private def get_string_tk(tk)
+ string = tk[:text]
+ state = nil
+ kind = :on_tstring
+ loop do
+ inner_str_tk = get_squashed_tk
+ if inner_str_tk.nil?
+ break
+ elsif :on_tstring_end == inner_str_tk[:kind]
+ string = string + inner_str_tk[:text]
+ state = inner_str_tk[:state]
+ break
+ elsif :on_label_end == inner_str_tk[:kind]
+ string = string + inner_str_tk[:text]
+ state = inner_str_tk[:state]
+ kind = :on_symbol
+ break
+ else
+ string = string + inner_str_tk[:text]
+ if :on_embexpr_beg == inner_str_tk[:kind] then
+ kind = :on_dstring if :on_tstring == kind
+ end
+ end
+ end
+ {
+ :line_no => tk[:line_no],
+ :char_no => tk[:char_no],
+ :kind => kind,
+ :text => string,
+ :state => state
+ }
+ end
+
+ private def get_regexp_tk(tk)
+ string = tk[:text]
+ state = nil
+ loop do
+ inner_str_tk = get_squashed_tk
+ if inner_str_tk.nil?
+ break
+ elsif :on_regexp_end == inner_str_tk[:kind]
+ string = string + inner_str_tk[:text]
+ state = inner_str_tk[:state]
+ break
+ else
+ string = string + inner_str_tk[:text]
+ end
+ end
+ {
+ :line_no => tk[:line_no],
+ :char_no => tk[:char_no],
+ :kind => :on_regexp,
+ :text => string,
+ :state => state
+ }
+ end
+
+ private def get_embdoc_tk(tk)
+ string = tk[:text]
+ until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do
+ string = string + embdoc_tk[:text]
+ end
+ string = string + embdoc_tk[:text]
+ {
+ :line_no => tk[:line_no],
+ :char_no => tk[:char_no],
+ :kind => :on_embdoc,
+ :text => string,
+ :state => embdoc_tk[:state]
+ }
+ end
+
+ private def get_heredoc_tk(heredoc_name, indent)
+ string = ''
+ start_tk = nil
+ prev_tk = nil
+ until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
+ start_tk = tk unless start_tk
+ if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
+ string = string + (' ' * tk[:char_no])
+ end
+ string = string + tk[:text]
+ prev_tk = tk
+ end
+ start_tk = tk unless start_tk
+ prev_tk = tk unless prev_tk
+ @buf.unshift tk # closing heredoc
+ heredoc_tk = {
+ :line_no => start_tk[:line_no],
+ :char_no => start_tk[:char_no],
+ :kind => :on_heredoc,
+ :text => string,
+ :state => prev_tk[:state]
+ }
+ @buf.unshift heredoc_tk
+ end
+
+ private def retrieve_heredoc_info(tk)
+ name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2')
+ indent = tk[:text] =~ /\A<<[-~]/
+ [name, indent]
+ end
+
+ private def heredoc_end?(name, indent, tk)
+ result = false
+ if :on_heredoc_end == tk[:kind] then
+ tk_name = (indent ? tk[:text].gsub(/^ *(.+)\n?$/, '\1') : tk[:text].gsub(/\n\z/, ''))
+ if name == tk_name
+ result = true
+ end
+ end
+ result
+ end
+
+ private def get_words_tk(tk)
+ string = ''
+ start_token = tk[:text]
+ start_quote = tk[:text].rstrip[-1]
+ line_no = tk[:line_no]
+ char_no = tk[:char_no]
+ state = tk[:state]
+ end_quote =
+ case start_quote
+ when ?( then ?)
+ when ?[ then ?]
+ when ?{ then ?}
+ when ?< then ?>
+ else start_quote
+ end
+ end_token = nil
+ loop do
+ tk = get_squashed_tk
+ if tk.nil?
+ end_token = end_quote
+ break
+ elsif :on_tstring_content == tk[:kind] then
+ string += tk[:text]
+ elsif :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then
+ if end_quote == tk[:text].strip then
+ end_token = tk[:text]
+ break
+ else
+ string += tk[:text]
+ end
+ else
+ string += tk[:text]
+ end
+ end
+ text = "#{start_token}#{string}#{end_token}"
+ {
+ :line_no => line_no,
+ :char_no => char_no,
+ :kind => :on_dstring,
+ :text => text,
+ :state => state
+ }
+ end
+
+ private def get_op_tk(tk)
+ redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
+ if redefinable_operators.include?(tk[:text]) and tk[:state] == EXPR_ARG then
+ @inner_lex.lex_state = EXPR_ARG unless RIPPER_HAS_LEX_STATE
+ tk[:state] = Ripper::Lexer.const_defined?(:State) ? Ripper::Lexer::State.new(EXPR_ARG) : EXPR_ARG
+ tk[:kind] = :on_ident
+ elsif tk[:text] =~ /^[-+]$/ then
+ tk_ahead = get_squashed_tk
+ case tk_ahead[:kind]
+ when :on_int, :on_float, :on_rational, :on_imaginary then
+ tk[:text] += tk_ahead[:text]
+ tk[:kind] = tk_ahead[:kind]
+ tk[:state] = tk_ahead[:state]
+ else
+ @buf.unshift tk_ahead
+ end
+ end
+ tk
+ end
+
+ def initialize(code)
+ @buf = []
+ @heredoc_queue = []
+ @inner_lex = InnerStateLex.new(code)
+ @inner_lex_enumerator = Enumerator.new do |y|
+ @inner_lex.each do |tk|
+ y << tk
+ end
+ end
+ end
+
+ def self.parse(code)
+ lex = self.new(code)
+ tokens = []
+ begin
+ while tk = lex.get_squashed_tk
+ tokens.push tk
+ end
+ rescue StopIteration
+ end
+ tokens
+ end
+
+ def self.end?(token)
+ (token[:state] & EXPR_END)
+ end
+end
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
new file mode 100644
index 0000000000..8599f655ad
--- /dev/null
+++ b/lib/rdoc/parser/ruby.rb
@@ -0,0 +1,2232 @@
+# frozen_string_literal: true
+##
+# This file contains stuff stolen outright from:
+#
+# rtags.rb -
+# ruby-lex.rb - ruby lexcal analyzer
+# ruby-token.rb - ruby tokens
+# by Keiju ISHITSUKA (Nippon Rational Inc.)
+#
+
+$TOKEN_DEBUG ||= nil
+
+##
+# Extracts code elements from a source file returning a TopLevel object
+# containing the constituent file elements.
+#
+# This file is based on rtags
+#
+# RubyParser understands how to document:
+# * classes
+# * modules
+# * methods
+# * constants
+# * aliases
+# * private, public, protected
+# * private_class_function, public_class_function
+# * private_constant, public_constant
+# * module_function
+# * attr, attr_reader, attr_writer, attr_accessor
+# * extra accessors given on the command line
+# * metaprogrammed methods
+# * require
+# * include
+#
+# == Method Arguments
+#
+#--
+# NOTE: I don't think this works, needs tests, remove the paragraph following
+# this block when known to work
+#
+# The parser extracts the arguments from the method definition. You can
+# override this with a custom argument definition using the :args: directive:
+#
+# ##
+# # This method tries over and over until it is tired
+#
+# def go_go_go(thing_to_try, tries = 10) # :args: thing_to_try
+# puts thing_to_try
+# go_go_go thing_to_try, tries - 1
+# end
+#
+# If you have a more-complex set of overrides you can use the :call-seq:
+# directive:
+#++
+#
+# The parser extracts the arguments from the method definition. You can
+# override this with a custom argument definition using the :call-seq:
+# directive:
+#
+# ##
+# # This method can be called with a range or an offset and length
+# #
+# # :call-seq:
+# # my_method(Range)
+# # my_method(offset, length)
+#
+# def my_method(*args)
+# end
+#
+# The parser extracts +yield+ expressions from method bodies to gather the
+# yielded argument names. If your method manually calls a block instead of
+# yielding or you want to override the discovered argument names use
+# the :yields: directive:
+#
+# ##
+# # My method is awesome
+#
+# def my_method(&block) # :yields: happy, times
+# block.call 1, 2
+# end
+#
+# == Metaprogrammed Methods
+#
+# To pick up a metaprogrammed method, the parser looks for a comment starting
+# with '##' before an identifier:
+#
+# ##
+# # This is a meta-programmed method!
+#
+# add_my_method :meta_method, :arg1, :arg2
+#
+# The parser looks at the token after the identifier to determine the name, in
+# this example, :meta_method. If a name cannot be found, a warning is printed
+# and 'unknown is used.
+#
+# You can force the name of a method using the :method: directive:
+#
+# ##
+# # :method: some_method!
+#
+# By default, meta-methods are instance methods. To indicate that a method is
+# a singleton method instead use the :singleton-method: directive:
+#
+# ##
+# # :singleton-method:
+#
+# You can also use the :singleton-method: directive with a name:
+#
+# ##
+# # :singleton-method: some_method!
+#
+# You can define arguments for metaprogrammed methods via either the
+# :call-seq:, :arg: or :args: directives.
+#
+# Additionally you can mark a method as an attribute by
+# using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:. Just like
+# for :method:, the name is optional.
+#
+# ##
+# # :attr_reader: my_attr_name
+#
+# == Hidden methods and attributes
+#
+# You can provide documentation for methods that don't appear using
+# the :method:, :singleton-method: and :attr: directives:
+#
+# ##
+# # :attr_writer: ghost_writer
+# # There is an attribute here, but you can't see it!
+#
+# ##
+# # :method: ghost_method
+# # There is a method here, but you can't see it!
+#
+# ##
+# # this is a comment for a regular method
+#
+# def regular_method() end
+#
+# Note that by default, the :method: directive will be ignored if there is a
+# standard rdocable item following it.
+
+require 'ripper'
+
+class RDoc::Parser::Ruby < RDoc::Parser
+
+ parse_files_matching(/\.rbw?$/)
+
+ include RDoc::TokenStream
+ include RDoc::Parser::RubyTools
+
+ ##
+ # RDoc::NormalClass type
+
+ NORMAL = "::"
+
+ ##
+ # RDoc::SingleClass type
+
+ SINGLE = "<<"
+
+ ##
+ # Creates a new Ruby parser.
+
+ def initialize(top_level, file_name, content, options, stats)
+ super
+
+ if /\t/ =~ content then
+ tab_width = @options.tab_width
+ content = content.split(/\n/).map do |line|
+ 1 while line.gsub!(/\t+/) {
+ ' ' * (tab_width*$&.length - $`.length % tab_width)
+ } && $~
+ line
+ end.join("\n")
+ end
+
+ @size = 0
+ @token_listeners = nil
+ @scanner = RDoc::RipperStateLex.parse(content)
+ @content = content
+ @scanner_point = 0
+ @prev_seek = nil
+ @markup = @options.markup
+ @track_visibility = :nodoc != @options.visibility
+ @encoding = @options.encoding
+
+ reset
+ end
+
+ def tk_nl?(tk)
+ :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+ end
+
+ ##
+ # Retrieves the read token stream and replaces +pattern+ with +replacement+
+ # using gsub. If the result is only a ";" returns an empty string.
+
+ def get_tkread_clean pattern, replacement # :nodoc:
+ read = get_tkread.gsub(pattern, replacement).strip
+ return '' if read == ';'
+ read
+ end
+
+ ##
+ # Extracts the visibility information for the visibility token +tk+
+ # and +single+ class type identifier.
+ #
+ # Returns the visibility type (a string), the visibility (a symbol) and
+ # +singleton+ if the methods following should be converted to singleton
+ # methods.
+
+ def get_visibility_information tk, single # :nodoc:
+ vis_type = tk[:text]
+ singleton = single == SINGLE
+
+ vis =
+ case vis_type
+ when 'private' then :private
+ when 'protected' then :protected
+ when 'public' then :public
+ when 'private_class_method' then
+ singleton = true
+ :private
+ when 'public_class_method' then
+ singleton = true
+ :public
+ when 'module_function' then
+ singleton = true
+ :public
+ else
+ raise RDoc::Error, "Invalid visibility: #{tk.name}"
+ end
+
+ return vis_type, vis, singleton
+ end
+
+ ##
+ # Look for the first comment in a file that isn't a shebang line.
+
+ def collect_first_comment
+ skip_tkspace
+ comment = ''.dup
+ comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
+ first_line = true
+ first_comment_tk_kind = nil
+
+ tk = get_tk
+
+ while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
+ if first_line and tk[:text] =~ /\A#!/ then
+ skip_tkspace
+ tk = get_tk
+ elsif first_line and tk[:text] =~ /\A#\s*-\*-/ then
+ first_line = false
+ skip_tkspace
+ tk = get_tk
+ else
+ break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind]
+ first_comment_tk_kind = tk[:kind]
+
+ first_line = false
+ comment << tk[:text]
+ tk = get_tk
+
+ if :on_nl === tk then
+ skip_tkspace false
+ tk = get_tk
+ end
+ end
+ end
+
+ unget_tk tk
+
+ new_comment comment
+ end
+
+ ##
+ # Consumes trailing whitespace from the token stream
+
+ def consume_trailing_spaces # :nodoc:
+ skip_tkspace false
+ end
+
+ ##
+ # Creates a new attribute in +container+ with +name+.
+
+ def create_attr container, single, name, rw, comment # :nodoc:
+ att = RDoc::Attr.new get_tkread, name, rw, comment, single == SINGLE
+ record_location att
+
+ container.add_attribute att
+ @stats.add_attribute att
+
+ att
+ end
+
+ ##
+ # Creates a module alias in +container+ at +rhs_name+ (or at the top-level
+ # for "::") with the name from +constant+.
+
+ def create_module_alias container, constant, rhs_name # :nodoc:
+ mod = if rhs_name =~ /^::/ then
+ @store.find_class_or_module rhs_name
+ else
+ container.find_module_named rhs_name
+ end
+
+ container.add_module_alias mod, constant.name, @top_level if mod
+ end
+
+ ##
+ # Aborts with +msg+
+
+ def error(msg)
+ msg = make_message msg
+
+ abort msg
+ end
+
+ ##
+ # Looks for a true or false token.
+
+ def get_bool
+ skip_tkspace
+ tk = get_tk
+ if :on_kw == tk[:kind] && 'true' == tk[:text]
+ true
+ elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text])
+ false
+ else
+ unget_tk tk
+ true
+ end
+ end
+
+ ##
+ # Look for the name of a class of module (optionally with a leading :: or
+ # with :: separated named) and return the ultimate name, the associated
+ # container, and the given name (with the ::).
+
+ def get_class_or_module container, ignore_constants = false
+ skip_tkspace
+ name_t = get_tk
+ given_name = ''.dup
+
+ # class ::A -> A is in the top level
+ if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug
+ name_t = get_tk
+ container = @top_level
+ given_name << '::'
+ end
+
+ skip_tkspace false
+ given_name << name_t[:text]
+
+ is_self = name_t[:kind] == :on_op && name_t[:text] == '<<'
+ while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do
+ prev_container = container
+ container = container.find_module_named name_t[:text]
+ container ||=
+ if ignore_constants then
+ RDoc::Context.new
+ else
+ c = prev_container.add_module RDoc::NormalModule, name_t[:text]
+ c.ignore unless prev_container.document_children
+ @top_level.add_to_classes_or_modules c
+ c
+ end
+
+ record_location container
+
+ get_tk
+ skip_tkspace false
+ name_t = get_tk
+ unless :on_const == name_t[:kind] || :on_ident == name_t[:kind]
+ raise RDoc::Error, "Invalid class or module definition: #{given_name}"
+ end
+ if prev_container == container and !ignore_constants
+ given_name = name_t[:text]
+ else
+ given_name << '::' + name_t[:text]
+ end
+ end
+
+ skip_tkspace false
+
+ return [container, name_t, given_name]
+ end
+
+ ##
+ # Return a superclass, which can be either a constant of an expression
+
+ def get_class_specification
+ tk = peek_tk
+ if tk.nil?
+ return ''
+ elsif :on_kw == tk[:kind] && 'self' == tk[:text]
+ return 'self'
+ elsif :on_gvar == tk[:kind]
+ return ''
+ end
+
+ res = get_constant
+
+ skip_tkspace false
+
+ get_tkread # empty out read buffer
+
+ tk = get_tk
+ return res unless tk
+
+ case tk[:kind]
+ when :on_nl, :on_comment, :on_embdoc, :on_semicolon then
+ unget_tk(tk)
+ return res
+ end
+
+ res += parse_call_parameters(tk)
+ res
+ end
+
+ ##
+ # Parse a constant, which might be qualified by one or more class or module
+ # names
+
+ def get_constant
+ res = ""
+ skip_tkspace false
+ tk = get_tk
+
+ while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do
+ res += tk[:text]
+ tk = get_tk
+ end
+
+ unget_tk(tk)
+ res
+ end
+
+ ##
+ # Get a constant that may be surrounded by parens
+
+ def get_constant_with_optional_parens
+ skip_tkspace false
+
+ nest = 0
+
+ while :on_lparen == (tk = peek_tk)[:kind] do
+ get_tk
+ skip_tkspace
+ nest += 1
+ end
+
+ name = get_constant
+
+ while nest > 0
+ skip_tkspace
+ tk = get_tk
+ nest -= 1 if :on_rparen == tk[:kind]
+ end
+
+ name
+ end
+
+ ##
+ # Little hack going on here. In the statement:
+ #
+ # f = 2*(1+yield)
+ #
+ # We see the RPAREN as the next token, so we need to exit early. This still
+ # won't catch all cases (such as "a = yield + 1"
+
+ def get_end_token tk # :nodoc:
+ case tk[:kind]
+ when :on_lparen
+ {
+ :kind => :on_rparen,
+ :text => ')'
+ }
+ when :on_rparen
+ nil
+ else
+ {
+ :kind => :on_nl,
+ :text => "\n"
+ }
+ end
+ end
+
+ ##
+ # Retrieves the method container for a singleton method.
+
+ def get_method_container container, name_t # :nodoc:
+ prev_container = container
+ container = container.find_module_named(name_t[:text])
+
+ unless container then
+ constant = prev_container.constants.find do |const|
+ const.name == name_t[:text]
+ end
+
+ if constant then
+ parse_method_dummy prev_container
+ return
+ end
+ end
+
+ unless container then
+ # TODO seems broken, should starting at Object in @store
+ obj = name_t[:text].split("::").inject(Object) do |state, item|
+ state.const_get(item)
+ end rescue nil
+
+ type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
+
+ unless [Class, Module].include?(obj.class) then
+ warn("Couldn't find #{name_t[:text]}. Assuming it's a module")
+ end
+
+ if type == RDoc::NormalClass then
+ sclass = obj.superclass ? obj.superclass.name : nil
+ container = prev_container.add_class type, name_t[:text], sclass
+ else
+ container = prev_container.add_module type, name_t[:text]
+ end
+
+ record_location container
+ end
+
+ container
+ end
+
+ ##
+ # Extracts a name or symbol from the token stream.
+
+ def get_symbol_or_name
+ tk = get_tk
+ case tk[:kind]
+ when :on_symbol then
+ text = tk[:text].sub(/^:/, '')
+
+ next_tk = peek_tk
+ if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then
+ get_tk
+ text << '='
+ end
+
+ text
+ when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then
+ tk[:text]
+ when :on_tstring, :on_dstring then
+ tk[:text][1..-2]
+ else
+ raise RDoc::Error, "Name or symbol expected (got #{tk})"
+ end
+ end
+
+ ##
+ # Marks containers between +container+ and +ancestor+ as ignored
+
+ def suppress_parents container, ancestor # :nodoc:
+ while container and container != ancestor do
+ container.suppress unless container.documented?
+ container = container.parent
+ end
+ end
+
+ ##
+ # Look for directives in a normal comment block:
+ #
+ # # :stopdoc:
+ # # Don't display comment from this point forward
+ #
+ # This routine modifies its +comment+ parameter.
+
+ def look_for_directives_in container, comment
+ @preprocess.handle comment, container do |directive, param|
+ case directive
+ when 'method', 'singleton-method',
+ 'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then
+ false # handled elsewhere
+ when 'section' then
+ break unless container.kind_of?(RDoc::Context)
+ container.set_current_section param, comment.dup
+ comment.text = ''
+ break
+ end
+ end
+
+ comment.remove_private
+ end
+
+ ##
+ # Adds useful info about the parser to +message+
+
+ def make_message message
+ prefix = "#{@file_name}:".dup
+
+ tk = peek_tk
+ prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk
+
+ "#{prefix} #{message}"
+ end
+
+ ##
+ # Creates a comment with the correct format
+
+ def new_comment comment
+ c = RDoc::Comment.new comment, @top_level
+ c.format = @markup
+ c
+ end
+
+ ##
+ # Creates an RDoc::Attr for the name following +tk+, setting the comment to
+ # +comment+.
+
+ def parse_attr(context, single, tk, comment)
+ line_no = tk[:line_no]
+
+ args = parse_symbol_arg 1
+ if args.size > 0 then
+ name = args[0]
+ rw = "R"
+ skip_tkspace false
+ tk = get_tk
+
+ if :on_comma == tk[:kind] then
+ rw = "RW" if get_bool
+ else
+ unget_tk tk
+ end
+
+ att = create_attr context, single, name, rw, comment
+ att.line = line_no
+
+ read_documentation_modifiers att, RDoc::ATTR_MODIFIERS
+ else
+ warn "'attr' ignored - looks like a variable"
+ end
+ end
+
+ ##
+ # Creates an RDoc::Attr for each attribute listed after +tk+, setting the
+ # comment for each to +comment+.
+
+ def parse_attr_accessor(context, single, tk, comment)
+ line_no = tk[:line_no]
+
+ args = parse_symbol_arg
+ rw = "?"
+
+ tmp = RDoc::CodeObject.new
+ read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
+ # TODO In most other places we let the context keep track of document_self
+ # and add found items appropriately but here we do not. I'm not sure why.
+ return if @track_visibility and not tmp.document_self
+
+ case tk[:text]
+ when "attr_reader" then rw = "R"
+ when "attr_writer" then rw = "W"
+ when "attr_accessor" then rw = "RW"
+ else
+ rw = '?'
+ end
+
+ for name in args
+ att = create_attr context, single, name, rw, comment
+ att.line = line_no
+ end
+ end
+
+ ##
+ # Parses an +alias+ in +context+ with +comment+
+
+ def parse_alias(context, single, tk, comment)
+ line_no = tk[:line_no]
+
+ skip_tkspace
+
+ if :on_lparen === peek_tk[:kind] then
+ get_tk
+ skip_tkspace
+ end
+
+ new_name = get_symbol_or_name
+
+ skip_tkspace
+ if :on_comma === peek_tk[:kind] then
+ get_tk
+ skip_tkspace
+ end
+
+ begin
+ old_name = get_symbol_or_name
+ rescue RDoc::Error
+ return
+ end
+
+ al = RDoc::Alias.new(get_tkread, old_name, new_name, comment,
+ single == SINGLE)
+ record_location al
+ al.line = line_no
+
+ read_documentation_modifiers al, RDoc::ATTR_MODIFIERS
+ context.add_alias al
+ @stats.add_alias al
+
+ al
+ end
+
+ ##
+ # Extracts call parameters from the token stream.
+
+ def parse_call_parameters(tk)
+ end_token = case tk[:kind]
+ when :on_lparen
+ :on_rparen
+ when :on_rparen
+ return ""
+ else
+ :on_nl
+ end
+ nest = 0
+
+ loop do
+ break if tk.nil?
+ case tk[:kind]
+ when :on_semicolon
+ break
+ when :on_lparen
+ nest += 1
+ when end_token
+ if end_token == :on_rparen
+ nest -= 1
+ break if RDoc::RipperStateLex.end?(tk) and nest <= 0
+ else
+ break if RDoc::RipperStateLex.end?(tk)
+ end
+ when :on_comment, :on_embdoc
+ unget_tk(tk)
+ break
+ when :on_op
+ if tk[:text] =~ /^(.{1,2})?=$/
+ unget_tk(tk)
+ break
+ end
+ end
+ tk = get_tk
+ end
+
+ get_tkread_clean "\n", " "
+ end
+
+ ##
+ # Parses a class in +context+ with +comment+
+
+ def parse_class container, single, tk, comment
+ line_no = tk[:line_no]
+
+ declaration_context = container
+ container, name_t, given_name = get_class_or_module container
+
+ if name_t[:kind] == :on_const
+ cls = parse_class_regular container, declaration_context, single,
+ name_t, given_name, comment
+ elsif name_t[:kind] == :on_op && name_t[:text] == '<<'
+ case name = get_class_specification
+ when 'self', container.name
+ read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+ parse_statements container, SINGLE
+ return # don't update line
+ else
+ cls = parse_class_singleton container, name, comment
+ end
+ else
+ warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}"
+ return
+ end
+
+ cls.line = line_no
+
+ # after end modifiers
+ read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+
+ cls
+ end
+
+ ##
+ # Parses and creates a regular class
+
+ def parse_class_regular container, declaration_context, single, # :nodoc:
+ name_t, given_name, comment
+ superclass = '::Object'
+
+ if given_name =~ /^::/ then
+ declaration_context = @top_level
+ given_name = $'
+ end
+
+ tk = peek_tk
+ if tk[:kind] == :on_op && tk[:text] == '<' then
+ get_tk
+ skip_tkspace
+ superclass = get_class_specification
+ superclass = '(unknown)' if superclass.empty?
+ end
+
+ cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass
+ cls = declaration_context.add_class cls_type, given_name, superclass
+ cls.ignore unless container.document_children
+
+ read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+ record_location cls
+
+ cls.add_comment comment, @top_level
+
+ @top_level.add_to_classes_or_modules cls
+ @stats.add_class cls
+
+ suppress_parents container, declaration_context unless cls.document_self
+
+ parse_statements cls
+
+ cls
+ end
+
+ ##
+ # Parses a singleton class in +container+ with the given +name+ and
+ # +comment+.
+
+ def parse_class_singleton container, name, comment # :nodoc:
+ other = @store.find_class_named name
+
+ unless other then
+ if name =~ /^::/ then
+ name = $'
+ container = @top_level
+ end
+
+ other = container.add_module RDoc::NormalModule, name
+ record_location other
+
+ # class << $gvar
+ other.ignore if name.empty?
+
+ other.add_comment comment, @top_level
+ end
+
+ # notify :nodoc: all if not a constant-named class/module
+ # (and remove any comment)
+ unless name =~ /\A(::)?[A-Z]/ then
+ other.document_self = nil
+ other.document_children = false
+ other.clear_comment
+ end
+
+ @top_level.add_to_classes_or_modules other
+ @stats.add_class other
+
+ read_documentation_modifiers other, RDoc::CLASS_MODIFIERS
+ parse_statements(other, SINGLE)
+
+ other
+ end
+
+ ##
+ # Parses a constant in +context+ with +comment+. If +ignore_constants+ is
+ # true, no found constants will be added to RDoc.
+
+ def parse_constant container, tk, comment, ignore_constants = false
+ line_no = tk[:line_no]
+
+ name = tk[:text]
+ skip_tkspace false
+
+ return unless name =~ /^\w+$/
+
+ if :on_op == peek_tk[:kind] && '::' == peek_tk[:text] then
+ unget_tk tk
+
+ container, name_t, = get_class_or_module container, ignore_constants
+
+ name = name_t[:text]
+ end
+
+ is_array_or_hash = false
+ if peek_tk && :on_lbracket == peek_tk[:kind]
+ get_tk
+ nest = 1
+ while bracket_tk = get_tk
+ case bracket_tk[:kind]
+ when :on_lbracket
+ nest += 1
+ when :on_rbracket
+ nest -= 1
+ break if nest == 0
+ end
+ end
+ skip_tkspace false
+ is_array_or_hash = true
+ end
+
+ unless peek_tk && :on_op == peek_tk[:kind] && '=' == peek_tk[:text] then
+ return false
+ end
+ get_tk
+
+ value = ''
+ con = RDoc::Constant.new name, value, comment
+
+ body = parse_constant_body container, con, is_array_or_hash
+
+ return unless body
+
+ con.value = body
+ record_location con
+ con.line = line_no
+ read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
+
+ return if is_array_or_hash
+
+ @stats.add_constant con
+ container.add_constant con
+
+ true
+ end
+
+ def parse_constant_body container, constant, is_array_or_hash # :nodoc:
+ nest = 0
+ rhs_name = ''.dup
+
+ get_tkread
+
+ tk = get_tk
+
+ body = nil
+ loop do
+ break if tk.nil?
+ if :on_semicolon == tk[:kind] then
+ break if nest <= 0
+ elsif [:on_tlambeg, :on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then
+ nest += 1
+ elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then
+ nest += 1
+ elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then
+ if (tk[:state] & RDoc::RipperStateLex::EXPR_LABEL) == 0
+ nest += 1
+ end
+ elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) ||
+ (:on_kw == tk[:kind] && 'end' == tk[:text]) then
+ nest -= 1
+ elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
+ unget_tk tk
+ if nest <= 0 and RDoc::RipperStateLex.end?(tk) then
+ body = get_tkread_clean(/^[ \t]+/, '')
+ read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
+ break
+ else
+ read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
+ end
+ elsif :on_const == tk[:kind] then
+ rhs_name << tk[:text]
+
+ next_tk = peek_tk
+ if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then
+ create_module_alias container, constant, rhs_name unless is_array_or_hash
+ break
+ end
+ elsif :on_nl == tk[:kind] then
+ if nest <= 0 and RDoc::RipperStateLex.end?(tk) then
+ unget_tk tk
+ break
+ end
+ elsif :on_op == tk[:kind] && '::' == tk[:text]
+ rhs_name << '::'
+ end
+ tk = get_tk
+ end
+
+ body ? body : get_tkread_clean(/^[ \t]+/, '')
+ end
+
+ ##
+ # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for
+ # :method: or :attr: directives in +comment+.
+
+ def parse_comment container, tk, comment
+ return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
+ column = tk[:char_no]
+ line_no = tk[:line_no]
+
+ comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
+ singleton = !!$~
+
+ co =
+ if (comment.text = comment.text.sub(/^# +:?method: *(\S*).*?\n/i, '')) && !!$~ then
+ parse_comment_ghost container, comment.text, $1, column, line_no, comment
+ elsif (comment.text = comment.text.sub(/# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '')) && !!$~ then
+ parse_comment_attr container, $1, $3, comment
+ end
+
+ if co then
+ co.singleton = singleton
+ co.line = line_no
+ end
+
+ true
+ end
+
+ ##
+ # Parse a comment that is describing an attribute in +container+ with the
+ # given +name+ and +comment+.
+
+ def parse_comment_attr container, type, name, comment # :nodoc:
+ return if name.empty?
+
+ rw = case type
+ when 'attr_reader' then 'R'
+ when 'attr_writer' then 'W'
+ else 'RW'
+ end
+
+ create_attr container, NORMAL, name, rw, comment
+ end
+
+ def parse_comment_ghost container, text, name, column, line_no, # :nodoc:
+ comment
+ name = nil if name.empty?
+
+ meth = RDoc::GhostMethod.new get_tkread, name
+ record_location meth
+
+ meth.start_collecting_tokens
+ indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+ position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+ position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+ newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+ meth.add_tokens [position_comment, newline, indent]
+
+ meth.params =
+ if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then
+ $1
+ else
+ ''
+ end
+
+ comment.normalize
+ comment.extract_call_seq meth
+
+ return unless meth.name
+
+ container.add_method meth
+
+ meth.comment = comment
+
+ @stats.add_method meth
+
+ meth
+ end
+
+ ##
+ # Creates an RDoc::Method on +container+ from +comment+ if there is a
+ # Signature section in the comment
+
+ def parse_comment_tomdoc container, tk, comment
+ return unless signature = RDoc::TomDoc.signature(comment)
+ column = tk[:char_no]
+ line_no = tk[:line_no]
+
+ name, = signature.split %r%[ \(]%, 2
+
+ meth = RDoc::GhostMethod.new get_tkread, name
+ record_location meth
+ meth.line = line_no
+
+ meth.start_collecting_tokens
+ indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+ position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+ position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+ newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+ meth.add_tokens [position_comment, newline, indent]
+
+ meth.call_seq = signature
+
+ comment.normalize
+
+ return unless meth.name
+
+ container.add_method meth
+
+ meth.comment = comment
+
+ @stats.add_method meth
+ end
+
+ ##
+ # Parses an +include+ or +extend+, indicated by the +klass+ and adds it to
+ # +container+ # with +comment+
+
+ def parse_extend_or_include klass, container, comment # :nodoc:
+ loop do
+ skip_tkspace_comment
+
+ name = get_constant_with_optional_parens
+
+ unless name.empty? then
+ obj = container.add klass, name, comment
+ record_location obj
+ end
+
+ return if peek_tk.nil? || :on_comma != peek_tk[:kind]
+
+ get_tk
+ end
+ end
+
+ ##
+ # Parses identifiers that can create new methods or change visibility.
+ #
+ # Returns true if the comment was not consumed.
+
+ def parse_identifier container, single, tk, comment # :nodoc:
+ case tk[:text]
+ when 'private', 'protected', 'public', 'private_class_method',
+ 'public_class_method', 'module_function' then
+ parse_visibility container, single, tk
+ return true
+ when 'private_constant', 'public_constant'
+ parse_constant_visibility container, single, tk
+ return true
+ when 'attr' then
+ parse_attr container, single, tk, comment
+ when /^attr_(reader|writer|accessor)$/ then
+ parse_attr_accessor container, single, tk, comment
+ when 'alias_method' then
+ parse_alias container, single, tk, comment
+ when 'require', 'include' then
+ # ignore
+ else
+ if comment.text =~ /\A#\#$/ then
+ case comment.text
+ when /^# +:?attr(_reader|_writer|_accessor)?:/ then
+ parse_meta_attr container, single, tk, comment
+ else
+ method = parse_meta_method container, single, tk, comment
+ method.params = container.params if
+ container.params
+ method.block_params = container.block_params if
+ container.block_params
+ end
+ end
+ end
+
+ false
+ end
+
+ ##
+ # Parses a meta-programmed attribute and creates an RDoc::Attr.
+ #
+ # To create foo and bar attributes on class C with comment "My attributes":
+ #
+ # class C
+ #
+ # ##
+ # # :attr:
+ # #
+ # # My attributes
+ #
+ # my_attr :foo, :bar
+ #
+ # end
+ #
+ # To create a foo attribute on class C with comment "My attribute":
+ #
+ # class C
+ #
+ # ##
+ # # :attr: foo
+ # #
+ # # My attribute
+ #
+ # my_attr :foo, :bar
+ #
+ # end
+
+ def parse_meta_attr(context, single, tk, comment)
+ args = parse_symbol_arg
+ rw = "?"
+
+ # If nodoc is given, don't document any of them
+
+ tmp = RDoc::CodeObject.new
+ read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS
+
+ regexp = /^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i
+ if regexp =~ comment.text then
+ comment.text = comment.text.sub(regexp, '')
+ rw = case $1
+ when 'attr_reader' then 'R'
+ when 'attr_writer' then 'W'
+ else 'RW'
+ end
+ name = $3 unless $3.empty?
+ end
+
+ if name then
+ att = create_attr context, single, name, rw, comment
+ else
+ args.each do |attr_name|
+ att = create_attr context, single, attr_name, rw, comment
+ end
+ end
+
+ att
+ end
+
+ ##
+ # Parses a meta-programmed method
+
+ def parse_meta_method(container, single, tk, comment)
+ column = tk[:char_no]
+ line_no = tk[:line_no]
+
+ start_collecting_tokens
+ add_token tk
+ add_token_listener self
+
+ skip_tkspace false
+
+ comment.text = comment.text.sub(/(^# +:?)(singleton-)(method:)/, '\1\3')
+ singleton = !!$~
+
+ name = parse_meta_method_name comment, tk
+
+ return unless name
+
+ meth = RDoc::MetaMethod.new get_tkread, name
+ record_location meth
+ meth.line = line_no
+ meth.singleton = singleton
+
+ remove_token_listener self
+
+ meth.start_collecting_tokens
+ indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+ position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+ position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+ newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+ meth.add_tokens [position_comment, newline, indent]
+ meth.add_tokens @token_stream
+
+ parse_meta_method_params container, single, meth, tk, comment
+
+ meth.comment = comment
+
+ @stats.add_method meth
+
+ meth
+ end
+
+ ##
+ # Parses the name of a metaprogrammed method. +comment+ is used to
+ # determine the name while +tk+ is used in an error message if the name
+ # cannot be determined.
+
+ def parse_meta_method_name comment, tk # :nodoc:
+ if comment.text.sub!(/^# +:?method: *(\S*).*?\n/i, '') then
+ return $1 unless $1.empty?
+ end
+
+ name_t = get_tk
+
+ if :on_symbol == name_t[:kind] then
+ name_t[:text][1..-1]
+ elsif :on_tstring == name_t[:kind] then
+ name_t[:text][1..-2]
+ elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore
+ remove_token_listener self
+
+ nil
+ else
+ warn "unknown name token #{name_t.inspect} for meta-method '#{tk[:text]}'"
+ 'unknown'
+ end
+ end
+
+ ##
+ # Parses the parameters and block for a meta-programmed method.
+
+ def parse_meta_method_params container, single, meth, tk, comment # :nodoc:
+ token_listener meth do
+ meth.params = ''
+
+ look_for_directives_in meth, comment
+ comment.normalize
+ comment.extract_call_seq meth
+
+ container.add_method meth
+
+ last_tk = tk
+
+ while tk = get_tk do
+ if :on_semicolon == tk[:kind] then
+ break
+ elsif :on_nl == tk[:kind] then
+ break unless last_tk and :on_comma == last_tk[:kind]
+ elsif :on_sp == tk[:kind] then
+ # expression continues
+ elsif :on_kw == tk[:kind] && 'do' == tk[:text] then
+ parse_statements container, single, meth
+ break
+ else
+ last_tk = tk
+ end
+ end
+ end
+ end
+
+ ##
+ # Parses a normal method defined by +def+
+
+ def parse_method(container, single, tk, comment)
+ singleton = nil
+ added_container = false
+ name = nil
+ column = tk[:char_no]
+ line_no = tk[:line_no]
+
+ start_collecting_tokens
+ add_token tk
+
+ token_listener self do
+ prev_container = container
+ name, container, singleton = parse_method_name container
+ added_container = container != prev_container
+ end
+
+ return unless name
+
+ meth = RDoc::AnyMethod.new get_tkread, name
+ look_for_directives_in meth, comment
+ meth.singleton = single == SINGLE ? true : singleton
+
+ record_location meth
+ meth.line = line_no
+
+ meth.start_collecting_tokens
+ indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+ token = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+ token[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+ newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+ meth.add_tokens [token, newline, indent]
+ meth.add_tokens @token_stream
+
+ parse_method_params_and_body container, single, meth, added_container
+
+ comment.normalize
+ comment.extract_call_seq meth
+
+ meth.comment = comment
+
+ # after end modifiers
+ read_documentation_modifiers meth, RDoc::METHOD_MODIFIERS
+
+ @stats.add_method meth
+ end
+
+ ##
+ # Parses the parameters and body of +meth+
+
+ def parse_method_params_and_body container, single, meth, added_container
+ token_listener meth do
+ parse_method_parameters meth
+
+ if meth.document_self or not @track_visibility then
+ container.add_method meth
+ elsif added_container then
+ container.document_self = false
+ end
+
+ # Having now read the method parameters and documentation modifiers, we
+ # now know whether we have to rename #initialize to ::new
+
+ if meth.name == "initialize" && !meth.singleton then
+ if meth.dont_rename_initialize then
+ meth.visibility = :protected
+ else
+ meth.singleton = true
+ meth.name = "new"
+ meth.visibility = :public
+ end
+ end
+
+ parse_statements container, single, meth
+ end
+ end
+
+ ##
+ # Parses a method that needs to be ignored.
+
+ def parse_method_dummy container
+ dummy = RDoc::Context.new
+ dummy.parent = container
+ dummy.store = container.store
+ skip_method dummy
+ end
+
+ ##
+ # Parses the name of a method in +container+.
+ #
+ # Returns the method name, the container it is in (for def Foo.name) and if
+ # it is a singleton or regular method.
+
+ def parse_method_name container # :nodoc:
+ skip_tkspace
+ name_t = get_tk
+ back_tk = skip_tkspace(false)
+ singleton = false
+
+ dot = get_tk
+ if dot[:kind] == :on_period || (dot[:kind] == :on_op && dot[:text] == '::') then
+ singleton = true
+
+ name, container = parse_method_name_singleton container, name_t
+ else
+ unget_tk dot
+ back_tk.reverse_each do |token|
+ unget_tk token
+ end
+
+ name = parse_method_name_regular container, name_t
+ end
+
+ return name, container, singleton
+ end
+
+ ##
+ # For the given +container+ and initial name token +name_t+ the method name
+ # is parsed from the token stream for a regular method.
+
+ def parse_method_name_regular container, name_t # :nodoc:
+ if :on_op == name_t[:kind] && (%w{* & [] []= <<}.include?(name_t[:text])) then
+ name_t[:text]
+ else
+ unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then
+ warn "expected method name token, . or ::, got #{name_t.inspect}"
+ skip_method container
+ return
+ end
+ name_t[:text]
+ end
+ end
+
+ ##
+ # For the given +container+ and initial name token +name_t+ the method name
+ # and the new +container+ (if necessary) are parsed from the token stream
+ # for a singleton method.
+
+ def parse_method_name_singleton container, name_t # :nodoc:
+ skip_tkspace
+ name_t2 = get_tk
+
+ if (:on_kw == name_t[:kind] && 'self' == name_t[:text]) || (:on_op == name_t[:kind] && '%' == name_t[:text]) then
+ # NOTE: work around '[' being consumed early
+ if :on_lbracket == name_t2[:kind]
+ get_tk
+ name = '[]'
+ else
+ name = name_t2[:text]
+ end
+ elsif :on_const == name_t[:kind] then
+ name = name_t2[:text]
+
+ container = get_method_container container, name_t
+
+ return unless container
+
+ name
+ elsif :on_ident == name_t[:kind] || :on_ivar == name_t[:kind] || :on_gvar == name_t[:kind] then
+ parse_method_dummy container
+
+ name = nil
+ elsif (:on_kw == name_t[:kind]) && ('true' == name_t[:text] || 'false' == name_t[:text] || 'nil' == name_t[:text]) then
+ klass_name = "#{name_t[:text].capitalize}Class"
+ container = @store.find_class_named klass_name
+ container ||= @top_level.add_class RDoc::NormalClass, klass_name
+
+ name = name_t2[:text]
+ else
+ warn "unexpected method name token #{name_t.inspect}"
+ # break
+ skip_method container
+
+ name = nil
+ end
+
+ return name, container
+ end
+
+ ##
+ # Extracts +yield+ parameters from +method+
+
+ def parse_method_or_yield_parameters(method = nil,
+ modifiers = RDoc::METHOD_MODIFIERS)
+ skip_tkspace false
+ tk = get_tk
+ end_token = get_end_token tk
+ return '' unless end_token
+
+ nest = 0
+ continue = false
+
+ while tk != nil do
+ case tk[:kind]
+ when :on_semicolon then
+ break if nest == 0
+ when :on_lbrace then
+ nest += 1
+ when :on_rbrace then
+ nest -= 1
+ if nest <= 0
+ # we might have a.each { |i| yield i }
+ unget_tk(tk) if nest < 0
+ break
+ end
+ when :on_lparen then
+ nest += 1
+ when end_token[:kind] then
+ if end_token[:kind] == :on_rparen
+ nest -= 1
+ break if nest <= 0
+ else
+ break
+ end
+ when :on_rparen then
+ nest -= 1
+ when :on_comment, :on_embdoc then
+ @read.pop
+ if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
+ (!continue or (tk[:state] & RDoc::RipperStateLex::EXPR_LABEL) != 0) then
+ if method && method.block_params.nil? then
+ unget_tk tk
+ read_documentation_modifiers method, modifiers
+ end
+ break if !continue and nest <= 0
+ end
+ when :on_comma then
+ continue = true
+ when :on_ident then
+ continue = false if continue
+ end
+ tk = get_tk
+ end
+
+ get_tkread_clean(/\s+/, ' ')
+ end
+
+ ##
+ # Capture the method's parameters. Along the way, look for a comment
+ # containing:
+ #
+ # # yields: ....
+ #
+ # and add this as the block_params for the method
+
+ def parse_method_parameters method
+ res = parse_method_or_yield_parameters method
+
+ res = "(#{res})" unless res =~ /\A\(/
+ method.params = res unless method.params
+
+ return if method.block_params
+
+ skip_tkspace false
+ read_documentation_modifiers method, RDoc::METHOD_MODIFIERS
+ end
+
+ ##
+ # Parses an RDoc::NormalModule in +container+ with +comment+
+
+ def parse_module container, single, tk, comment
+ container, name_t, = get_class_or_module container
+
+ name = name_t[:text]
+
+ mod = container.add_module RDoc::NormalModule, name
+ mod.ignore unless container.document_children
+ record_location mod
+
+ read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+ mod.add_comment comment, @top_level
+ parse_statements mod
+
+ # after end modifiers
+ read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS
+
+ @stats.add_module mod
+ end
+
+ ##
+ # Parses an RDoc::Require in +context+ containing +comment+
+
+ def parse_require(context, comment)
+ skip_tkspace_comment
+ tk = get_tk
+
+ if :on_lparen == tk[:kind] then
+ skip_tkspace_comment
+ tk = get_tk
+ end
+
+ name = tk[:text][1..-2] if :on_tstring == tk[:kind]
+
+ if name then
+ @top_level.add_require RDoc::Require.new(name, comment)
+ else
+ unget_tk tk
+ end
+ end
+
+ ##
+ # Parses a rescue
+
+ def parse_rescue
+ skip_tkspace false
+
+ while tk = get_tk
+ case tk[:kind]
+ when :on_nl, :on_semicolon, :on_comment then
+ break
+ when :on_comma then
+ skip_tkspace false
+
+ get_tk if :on_nl == peek_tk[:kind]
+ end
+
+ skip_tkspace false
+ end
+ end
+
+ ##
+ # The core of the Ruby parser.
+
+ def parse_statements(container, single = NORMAL, current_method = nil,
+ comment = new_comment(''))
+ raise 'no' unless RDoc::Comment === comment
+ comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
+
+ nest = 1
+ save_visibility = container.visibility
+
+ non_comment_seen = true
+
+ while tk = get_tk do
+ keep_comment = false
+ try_parse_comment = false
+
+ non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
+
+ case tk[:kind]
+ when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then
+ if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+ skip_tkspace
+ tk = get_tk
+ else
+ past_tokens = @read.size > 1 ? @read[0..-2] : []
+ nl_position = 0
+ past_tokens.reverse.each_with_index do |read_tk, i|
+ if read_tk =~ /^\n$/ then
+ nl_position = (past_tokens.size - 1) - i
+ break
+ elsif read_tk =~ /^#.*\n$/ then
+ nl_position = ((past_tokens.size - 1) - i) + 1
+ break
+ end
+ end
+ comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ }
+ unless comment_only_line then
+ tk = get_tk
+ end
+ end
+
+ if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
+ if non_comment_seen then
+ # Look for RDoc in a comment about to be thrown away
+ non_comment_seen = parse_comment container, tk, comment unless
+ comment.empty?
+
+ comment = ''
+ comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
+ end
+
+ while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do
+ comment += tk[:text]
+ comment += "\n" unless "\n" == tk[:text].chars.to_a.last
+
+ if tk[:text].size > 1 && "\n" == tk[:text].chars.to_a.last then
+ skip_tkspace false # leading spaces
+ end
+ tk = get_tk
+ end
+
+ comment = new_comment comment
+
+ unless comment.empty? then
+ look_for_directives_in container, comment
+
+ if container.done_documenting then
+ throw :eof if RDoc::TopLevel === container
+ container.ongoing_visibility = save_visibility
+ end
+ end
+
+ keep_comment = true
+ else
+ non_comment_seen = true
+ end
+
+ unget_tk tk
+ keep_comment = true
+ container.current_line_visibility = nil
+
+ when :on_kw then
+ case tk[:text]
+ when 'class' then
+ parse_class container, single, tk, comment
+
+ when 'module' then
+ parse_module container, single, tk, comment
+
+ when 'def' then
+ parse_method container, single, tk, comment
+
+ when 'alias' then
+ parse_alias container, single, tk, comment unless current_method
+
+ when 'yield' then
+ if current_method.nil? then
+ warn "Warning: yield outside of method" if container.document_self
+ else
+ parse_yield container, single, tk, current_method
+ end
+
+ when 'until', 'while' then
+ if (tk[:state] & RDoc::RipperStateLex::EXPR_LABEL) == 0
+ nest += 1
+ skip_optional_do_after_expression
+ end
+
+ # Until and While can have a 'do', which shouldn't increase the nesting.
+ # We can't solve the general case, but we can handle most occurrences by
+ # ignoring a do at the end of a line.
+
+ # 'for' is trickier
+ when 'for' then
+ nest += 1
+ skip_for_variable
+ skip_optional_do_after_expression
+
+ when 'case', 'do', 'if', 'unless', 'begin' then
+ if (tk[:state] & RDoc::RipperStateLex::EXPR_LABEL) == 0
+ nest += 1
+ end
+
+ when 'super' then
+ current_method.calls_super = true if current_method
+
+ when 'rescue' then
+ parse_rescue
+
+ when 'end' then
+ nest -= 1
+ if nest == 0 then
+ container.ongoing_visibility = save_visibility
+
+ parse_comment container, tk, comment unless comment.empty?
+
+ return
+ end
+ end
+
+ when :on_const then
+ unless parse_constant container, tk, comment, current_method then
+ try_parse_comment = true
+ end
+
+ when :on_ident then
+ if nest == 1 and current_method.nil? then
+ keep_comment = parse_identifier container, single, tk, comment
+ end
+
+ case tk[:text]
+ when "require" then
+ parse_require container, comment
+ when "include" then
+ parse_extend_or_include RDoc::Include, container, comment
+ when "extend" then
+ parse_extend_or_include RDoc::Extend, container, comment
+ end
+
+ else
+ try_parse_comment = nest == 1
+ end
+
+ if try_parse_comment then
+ non_comment_seen = parse_comment container, tk, comment unless
+ comment.empty?
+
+ keep_comment = false
+ end
+
+ unless keep_comment then
+ comment = new_comment ''
+ comment = RDoc::Encoding.change_encoding comment, @encoding if @encoding
+ container.params = nil
+ container.block_params = nil
+ end
+
+ consume_trailing_spaces
+ end
+
+ container.params = nil
+ container.block_params = nil
+ end
+
+ ##
+ # Parse up to +no+ symbol arguments
+
+ def parse_symbol_arg(no = nil)
+ skip_tkspace_comment
+
+ tk = get_tk
+ if tk[:kind] == :on_lparen
+ parse_symbol_arg_paren no
+ else
+ parse_symbol_arg_space no, tk
+ end
+ end
+
+ ##
+ # Parses up to +no+ symbol arguments surrounded by () and places them in
+ # +args+.
+
+ def parse_symbol_arg_paren no # :nodoc:
+ args = []
+
+ loop do
+ skip_tkspace_comment
+ if tk1 = parse_symbol_in_arg
+ args.push tk1
+ break if no and args.size >= no
+ end
+
+ skip_tkspace_comment
+ case (tk2 = get_tk)[:kind]
+ when :on_rparen
+ break
+ when :on_comma
+ else
+ warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
+ break
+ end
+ end
+
+ args
+ end
+
+ ##
+ # Parses up to +no+ symbol arguments separated by spaces and places them in
+ # +args+.
+
+ def parse_symbol_arg_space no, tk # :nodoc:
+ args = []
+
+ unget_tk tk
+ if tk = parse_symbol_in_arg
+ args.push tk
+ return args if no and args.size >= no
+ end
+
+ loop do
+ skip_tkspace false
+
+ tk1 = get_tk
+ if tk1.nil? || :on_comma != tk1[:kind] then
+ unget_tk tk1
+ break
+ end
+
+ skip_tkspace_comment
+ if tk = parse_symbol_in_arg
+ args.push tk
+ break if no and args.size >= no
+ end
+ end
+
+ args
+ end
+
+ ##
+ # Returns symbol text from the next token
+
+ def parse_symbol_in_arg
+ tk = get_tk
+ if :on_symbol == tk[:kind] then
+ tk[:text].sub(/^:/, '')
+ elsif :on_tstring == tk[:kind] then
+ tk[:text][1..-2]
+ elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then
+ nil # ignore
+ else
+ warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
+ nil
+ end
+ end
+
+ ##
+ # Parses statements in the top-level +container+
+
+ def parse_top_level_statements container
+ comment = collect_first_comment
+
+ look_for_directives_in container, comment
+
+ throw :eof if container.done_documenting
+
+ @markup = comment.format
+
+ # HACK move if to RDoc::Context#comment=
+ container.comment = comment if container.document_self unless comment.empty?
+
+ parse_statements container, NORMAL, nil, comment
+ end
+
+ ##
+ # Determines the visibility in +container+ from +tk+
+
+ def parse_visibility(container, single, tk)
+ vis_type, vis, singleton = get_visibility_information tk, single
+
+ skip_tkspace_comment false
+
+ ptk = peek_tk
+ # Ryan Davis suggested the extension to ignore modifiers, because he
+ # often writes
+ #
+ # protected unless $TESTING
+ #
+ if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then
+ container.ongoing_visibility = vis
+ elsif :on_kw == ptk[:kind] && 'def' == ptk[:text]
+ container.current_line_visibility = vis
+ else
+ update_visibility container, vis_type, vis, singleton
+ end
+ end
+
+ ##
+ # Parses a Module#private_constant or Module#public_constant call from +tk+.
+
+ def parse_constant_visibility(container, single, tk)
+ args = parse_symbol_arg
+ case tk[:text]
+ when 'private_constant'
+ vis = :private
+ when 'public_constant'
+ vis = :public
+ else
+ raise RDoc::Error, 'Unreachable'
+ end
+ container.set_constant_visibility_for args, vis
+ end
+
+ ##
+ # Determines the block parameter for +context+
+
+ def parse_yield(context, single, tk, method)
+ return if method.block_params
+
+ get_tkread
+ method.block_params = parse_method_or_yield_parameters
+ end
+
+ ##
+ # Directives are modifier comments that can appear after class, module, or
+ # method names. For example:
+ #
+ # def fred # :yields: a, b
+ #
+ # or:
+ #
+ # class MyClass # :nodoc:
+ #
+ # We return the directive name and any parameters as a two element array if
+ # the name is in +allowed+. A directive can be found anywhere up to the end
+ # of the current line.
+
+ def read_directive allowed
+ tokens = []
+
+ while tk = get_tk do
+ tokens << tk
+
+ if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then
+ return
+ elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then
+ return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/
+
+ directive = $1.downcase
+
+ return [directive, $2] if allowed.include? directive
+
+ return
+ end
+ end
+ ensure
+ unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then
+ tokens.reverse_each do |token|
+ unget_tk token
+ end
+ end
+ end
+
+ ##
+ # Handles directives following the definition for +context+ (any
+ # RDoc::CodeObject) if the directives are +allowed+ at this point.
+ #
+ # See also RDoc::Markup::PreProcess#handle_directive
+
+ def read_documentation_modifiers context, allowed
+ skip_tkspace(false)
+ directive, value = read_directive allowed
+
+ return unless directive
+
+ @preprocess.handle_directive '', directive, value, context do |dir, param|
+ if %w[notnew not_new not-new].include? dir then
+ context.dont_rename_initialize = true
+
+ true
+ end
+ end
+ end
+
+ ##
+ # Records the location of this +container+ in the file for this parser and
+ # adds it to the list of classes and modules in the file.
+
+ def record_location container # :nodoc:
+ case container
+ when RDoc::ClassModule then
+ @top_level.add_to_classes_or_modules container
+ end
+
+ container.record_location @top_level
+ end
+
+ ##
+ # Scans this Ruby file for Ruby constructs
+
+ def scan
+ reset
+
+ catch :eof do
+ begin
+ parse_top_level_statements @top_level
+
+ rescue StandardError => e
+ if @content.include?('<%') and @content.include?('%>') then
+ # Maybe, this is ERB.
+ $stderr.puts "\033[2KRDoc detects ERB file. Skips it for compatibility:"
+ $stderr.puts @file_name
+ return
+ end
+ bytes = ''
+
+ if @scanner_point >= @scanner.size
+ now_line_no = @scanner[@scanner.size - 1][:line_no]
+ else
+ now_line_no = peek_tk[:line_no]
+ end
+
+ $stderr.puts <<-EOF
+
+#{self.class} failure around line #{now_line_no} of
+#{@file_name}
+
+ EOF
+
+ unless bytes.empty? then
+ $stderr.puts
+ now_line_no = peek_tk[:line_no]
+ start_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no }
+ end_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 } - 1
+ $stderr.puts @scanner[start_index..end_index].join
+ end
+
+ raise e
+ end
+ end
+
+ @top_level
+ end
+
+ ##
+ # while, until, and for have an optional do
+
+ def skip_optional_do_after_expression
+ skip_tkspace false
+ tk = get_tk
+
+ b_nest = 0
+ nest = 0
+
+ loop do
+ break unless tk
+ case tk[:kind]
+ when :on_semicolon, :on_nl, :on_ignored_nl then
+ break if b_nest.zero?
+ when :on_lparen then
+ nest += 1
+ when :on_rparen then
+ nest -= 1
+ when :on_kw then
+ case tk[:text]
+ when 'begin'
+ b_nest += 1
+ when 'end'
+ b_nest -= 1
+ when 'do'
+ break if nest.zero?
+ end
+ when :on_comment, :on_embdoc then
+ if b_nest.zero? and "\n" == tk[:text][-1] then
+ break
+ end
+ end
+ tk = get_tk
+ end
+
+ skip_tkspace false
+
+ get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text]
+ end
+
+ ##
+ # skip the var [in] part of a 'for' statement
+
+ def skip_for_variable
+ skip_tkspace false
+ get_tk
+ skip_tkspace false
+ tk = get_tk
+ unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text]
+ end
+
+ ##
+ # Skips the next method in +container+
+
+ def skip_method container
+ meth = RDoc::AnyMethod.new "", "anon"
+ parse_method_parameters meth
+ parse_statements container, false, meth
+ end
+
+ ##
+ # Skip spaces until a comment is found
+
+ def skip_tkspace_comment(skip_nl = true)
+ loop do
+ skip_tkspace skip_nl
+ next_tk = peek_tk
+ return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind])
+ get_tk
+ end
+ end
+
+ ##
+ # Updates visibility in +container+ from +vis_type+ and +vis+.
+
+ def update_visibility container, vis_type, vis, singleton # :nodoc:
+ new_methods = []
+
+ case vis_type
+ when 'module_function' then
+ args = parse_symbol_arg
+ container.set_visibility_for args, :private, false
+
+ container.methods_matching args do |m|
+ s_m = m.dup
+ record_location s_m
+ s_m.singleton = true
+ new_methods << s_m
+ end
+ when 'public_class_method', 'private_class_method' then
+ args = parse_symbol_arg
+
+ container.methods_matching args, true do |m|
+ if m.parent != container then
+ m = m.dup
+ record_location m
+ new_methods << m
+ end
+
+ m.visibility = vis
+ end
+ else
+ args = parse_symbol_arg
+ container.set_visibility_for args, vis, singleton
+ end
+
+ new_methods.each do |method|
+ case method
+ when RDoc::AnyMethod then
+ container.add_method method
+ when RDoc::Attr then
+ container.add_attribute method
+ end
+ method.visibility = vis
+ end
+ end
+
+ ##
+ # Prints +message+ to +$stderr+ unless we're being quiet
+
+ def warn message
+ @options.warn make_message message
+ end
+
+end
diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb
new file mode 100644
index 0000000000..1f621cd32e
--- /dev/null
+++ b/lib/rdoc/parser/ruby_tools.rb
@@ -0,0 +1,159 @@
+# frozen_string_literal: true
+##
+# Collection of methods for writing parsers
+
+module RDoc::Parser::RubyTools
+
+ ##
+ # Adds a token listener +obj+, but you should probably use token_listener
+
+ def add_token_listener(obj)
+ @token_listeners ||= []
+ @token_listeners << obj
+ end
+
+ ##
+ # Fetches the next token from the scanner
+
+ def get_tk
+ tk = nil
+
+ if @tokens.empty? then
+ if @scanner_point >= @scanner.size
+ return nil
+ else
+ tk = @scanner[@scanner_point]
+ @scanner_point += 1
+ @read.push tk[:text]
+ puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+ end
+ else
+ @read.push @unget_read.shift
+ tk = @tokens.shift
+ puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
+ end
+
+ if tk == nil || :on___end__ == tk[:kind]
+ tk = nil
+ end
+
+ return nil unless tk
+
+ # inform any listeners of our shiny new token
+ @token_listeners.each do |obj|
+ obj.add_token(tk)
+ end if @token_listeners
+
+ tk
+ end
+
+ ##
+ # Reads and returns all tokens up to one of +tokens+. Leaves the matched
+ # token in the token list.
+
+ def get_tk_until(*tokens)
+ read = []
+
+ loop do
+ tk = get_tk
+
+ case tk
+ when *tokens then
+ unget_tk tk
+ break
+ end
+
+ read << tk
+ end
+
+ read
+ end
+
+ ##
+ # Retrieves a String representation of the read tokens
+
+ def get_tkread
+ read = @read.join("")
+ @read = []
+ read
+ end
+
+ ##
+ # Peek equivalent for get_tkread
+
+ def peek_read
+ @read.join('')
+ end
+
+ ##
+ # Peek at the next token, but don't remove it from the stream
+
+ def peek_tk
+ unget_tk(tk = get_tk)
+ tk
+ end
+
+ ##
+ # Removes the token listener +obj+
+
+ def remove_token_listener(obj)
+ @token_listeners.delete(obj)
+ end
+
+ ##
+ # Resets the tools
+
+ def reset
+ @read = []
+ @tokens = []
+ @unget_read = []
+ @nest = 0
+ @scanner_point = 0
+ end
+
+ def tk_nl?(tk)
+ :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+ end
+
+ ##
+ # Skips whitespace tokens including newlines if +skip_nl+ is true
+
+ def skip_tkspace(skip_nl = true)
+ tokens = []
+
+ while (tk = get_tk) and (:on_sp == tk[:kind] or (skip_nl and tk_nl?(tk))) do
+ tokens.push(tk)
+ end
+
+ unget_tk(tk)
+ tokens
+ end
+
+ ##
+ # Has +obj+ listen to tokens
+
+ def token_listener(obj)
+ add_token_listener obj
+ yield
+ ensure
+ remove_token_listener obj
+ end
+
+ ##
+ # Returns +tk+ to the scanner
+
+ def unget_tk(tk)
+ @tokens.unshift tk
+ @unget_read.unshift @read.pop
+
+ # Remove this token from any listeners
+ @token_listeners.each do |obj|
+ obj.pop_token
+ end if @token_listeners
+
+ nil
+ end
+
+end
+
+
diff --git a/lib/rdoc/parser/simple.rb b/lib/rdoc/parser/simple.rb
new file mode 100644
index 0000000000..b1dabad0f8
--- /dev/null
+++ b/lib/rdoc/parser/simple.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+##
+# Parse a non-source file. We basically take the whole thing as one big
+# comment.
+
+class RDoc::Parser::Simple < RDoc::Parser
+
+ include RDoc::Parser::Text
+
+ parse_files_matching(//)
+
+ attr_reader :content # :nodoc:
+
+ ##
+ # Prepare to parse a plain file
+
+ def initialize(top_level, file_name, content, options, stats)
+ super
+
+ preprocess = RDoc::Markup::PreProcess.new @file_name, @options.rdoc_include
+
+ @content = preprocess.handle @content, @top_level
+ end
+
+ ##
+ # Extract the file contents and attach them to the TopLevel as a comment
+
+ def scan
+ comment = remove_coding_comment @content
+ comment = remove_private_comment comment
+
+ comment = RDoc::Comment.new comment, @top_level
+
+ @top_level.comment = comment
+ @top_level
+ end
+
+ ##
+ # Removes the encoding magic comment from +text+
+
+ def remove_coding_comment text
+ text.sub(/\A# .*coding[=:].*$/, '')
+ end
+
+ ##
+ # Removes private comments.
+ #
+ # Unlike RDoc::Comment#remove_private this implementation only looks for two
+ # dashes at the beginning of the line. Three or more dashes are considered
+ # to be a rule and ignored.
+
+ def remove_private_comment comment
+ # Workaround for gsub encoding for Ruby 1.9.2 and earlier
+ empty = ''
+ empty = RDoc::Encoding.change_encoding empty, comment.encoding
+
+ comment = comment.gsub(%r%^--\n.*?^\+\+\n?%m, empty)
+ comment.sub(%r%^--\n.*%m, empty)
+ end
+
+end
diff --git a/lib/rdoc/parser/text.rb b/lib/rdoc/parser/text.rb
new file mode 100644
index 0000000000..01de0cc595
--- /dev/null
+++ b/lib/rdoc/parser/text.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+##
+# Indicates this parser is text and doesn't contain code constructs.
+#
+# Include this module in a RDoc::Parser subclass to make it show up as a file,
+# not as part of a class or module.
+#--
+# This is not named File to avoid overriding ::File
+
+module RDoc::Parser::Text
+end
+
generated by cgit v1.2.3 (git 2.25.1) at 2026-06-07 00:04:01 +0000