diff options
Diffstat (limited to 'lib/rdoc/parser')
| -rw-r--r-- | lib/rdoc/parser/c.rb | 288 | ||||
| -rw-r--r-- | lib/rdoc/parser/f95.rb | 1835 | ||||
| -rw-r--r-- | lib/rdoc/parser/perl.rb | 16 | ||||
| -rw-r--r-- | lib/rdoc/parser/ruby.rb | 2240 | ||||
| -rw-r--r-- | lib/rdoc/parser/ruby_tools.rb | 157 | ||||
| -rw-r--r-- | lib/rdoc/parser/simple.rb | 5 |
6 files changed, 822 insertions, 3719 deletions
diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb index a2c3eefbde..f2d460b62f 100644 --- a/lib/rdoc/parser/c.rb +++ b/lib/rdoc/parser/c.rb @@ -14,32 +14,32 @@ require 'rdoc/known_classes' # method, that is to say the method whose name is given in the # <tt>rb_define_method</tt> call. For example, you might write: # -# /* -# * Returns a new array that is a one-dimensional flattening of this -# * array (recursively). That is, for every element that is an array, -# * extract its elements into the new array. -# * -# * s = [ 1, 2, 3 ] #=> [1, 2, 3] -# * t = [ 4, 5, 6, [7, 8] ] #=> [4, 5, 6, [7, 8]] -# * a = [ s, t, 9, 10 ] #=> [[1, 2, 3], [4, 5, 6, [7, 8]], 9, 10] -# * a.flatten #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] -# */ -# static VALUE -# rb_ary_flatten(ary) -# VALUE ary; -# { -# ary = rb_obj_dup(ary); -# rb_ary_flatten_bang(ary); -# return ary; -# } +# /* +# * Returns a new array that is a one-dimensional flattening of this +# * array (recursively). That is, for every element that is an array, +# * extract its elements into the new array. +# * +# * s = [ 1, 2, 3 ] #=> [1, 2, 3] +# * t = [ 4, 5, 6, [7, 8] ] #=> [4, 5, 6, [7, 8]] +# * a = [ s, t, 9, 10 ] #=> [[1, 2, 3], [4, 5, 6, [7, 8]], 9, 10] +# * a.flatten #=> [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] +# */ +# static VALUE +# rb_ary_flatten(ary) +# VALUE ary; +# { +# ary = rb_obj_dup(ary); +# rb_ary_flatten_bang(ary); +# return ary; +# } # -# ... +# ... # -# void -# Init_Array() -# { -# ... -# rb_define_method(rb_cArray, "flatten", rb_ary_flatten, 0); +# void +# Init_Array() +# { +# ... +# rb_define_method(rb_cArray, "flatten", rb_ary_flatten, 0); # # Here RDoc will determine from the rb_define_method line that there's a # method called "flatten" in class Array, and will look for the implementation @@ -47,9 +47,6 @@ require 'rdoc/known_classes' # method in the HTML output. This method must be in the same source file # as the rb_define_method. # -# C classes can be diagrammed (see /tc/dl/ruby/ruby/error.c), and RDoc -# integrates C and Ruby source into one tree -# # The comment blocks may include special directives: # # [Document-class: <i>name</i>] @@ -68,7 +65,7 @@ require 'rdoc/known_classes' # Ruby function is in the same source file as the rb_define_method call. # If this isn't the case, add the comment: # -# rb_define_method(....); // in filename +# rb_define_method(....); // in: filename # # As an example, we might have an extension that defines multiple classes # in its Init_xxx method. We could document them using @@ -79,7 +76,7 @@ require 'rdoc/known_classes' # * Encapsulate the writing and reading of the configuration # * file. ... # */ -# +# # /* # * Document-method: read_value # * @@ -96,8 +93,23 @@ class RDoc::Parser::C < RDoc::Parser parse_files_matching(/\.(?:([CcHh])\1?|c([+xp])\2|y)\z/) - @@enclosure_classes = {} - @@known_bodies = {} + include RDoc::Text + + ## + # C file the parser is parsing + + attr_accessor :content + + ## + # Resets cross-file state. Call when parsing different projects that need + # separate documentation. + + def self.reset + @@enclosure_classes = {} + @@known_bodies = {} + end + + reset ## # Prepare to parse a C file @@ -164,19 +176,17 @@ class RDoc::Parser::C < RDoc::Parser def do_constants @content.scan(%r{\Wrb_define_ - ( - variable | - readonly_variable | - const | - global_const | - ) + ( variable | + readonly_variable | + const | + global_const | ) \s*\( (?:\s*(\w+),)? \s*"(\w+)", \s*(.*?)\s*\)\s*; }xm) do |type, var_name, const_name, definition| var_name = "rb_cObject" if !var_name or var_name == "rb_mKernel" - handle_constants(type, var_name, const_name, definition) + handle_constants type, var_name, const_name, definition end end @@ -270,12 +280,13 @@ class RDoc::Parser::C < RDoc::Parser def find_body(class_name, meth_name, meth_obj, body, quiet = false) case body - when %r"((?>/\*.*?\*/\s*))(?:(?:static|SWIGINTERN)\s+)?(?:intern\s+)?VALUE\s+#{meth_name} - \s*(\([^)]*\))([^;]|$)"xm - comment, params = $1, $2 - body_text = $& + when %r"((?>/\*.*?\*/\s*))((?:(?:static|SWIGINTERN)\s+)?(?:intern\s+)?VALUE\s+#{meth_name} + \s*(\([^)]*\))([^;]|$))"xm + comment = $1 + body_text = $2 + params = $3 - remove_private_comments(comment) if comment + remove_private_comments comment if comment # see if we can find the whole body @@ -288,33 +299,40 @@ class RDoc::Parser::C < RDoc::Parser # distinct (for example Kernel.hash and Kernel.object_id share the same # implementation - override_comment = find_override_comment(class_name, meth_obj.name) + override_comment = find_override_comment class_name, meth_obj.name comment = override_comment if override_comment - find_modifiers(comment, meth_obj) if comment + find_modifiers comment, meth_obj if comment # meth_obj.params = params meth_obj.start_collecting_tokens - meth_obj.add_token(RDoc::RubyToken::Token.new(1,1).set_text(body_text)) - meth_obj.comment = mangle_comment(comment) - when %r{((?>/\*.*?\*/\s*))^\s*\#\s*define\s+#{meth_name}\s+(\w+)}m + tk = RDoc::RubyToken::Token.new nil, 1, 1 + tk.set_text body_text + meth_obj.add_token tk + meth_obj.comment = strip_stars comment + when %r{((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+#{meth_name}\s+(\w+))}m comment = $1 - find_body(class_name, $2, meth_obj, body, true) - find_modifiers(comment, meth_obj) - meth_obj.comment = mangle_comment(comment) + meth_obj.comment + body_text = $2 + find_body class_name, $3, meth_obj, body, true + find_modifiers comment, meth_obj + + meth_obj.start_collecting_tokens + tk = RDoc::RubyToken::Token.new nil, 1, 1 + tk.set_text body_text + meth_obj.add_token tk + meth_obj.comment = strip_stars(comment) + meth_obj.comment.to_s when %r{^\s*\#\s*define\s+#{meth_name}\s+(\w+)}m unless find_body(class_name, $1, meth_obj, body, true) warn "No definition for #{meth_name}" unless @options.quiet return false end else - # No body, but might still have an override comment comment = find_override_comment(class_name, meth_obj.name) if comment find_modifiers(comment, meth_obj) - meth_obj.comment = mangle_comment(comment) + meth_obj.comment = strip_stars comment else warn "No definition for #{meth_name}" unless @options.quiet return false @@ -328,7 +346,7 @@ class RDoc::Parser::C < RDoc::Parser if raw_name =~ /^rb_m/ container = @top_level.add_module RDoc::NormalModule, name else - container = @top_level.add_class RDoc::NormalClass, name, nil + container = @top_level.add_class RDoc::NormalClass, name end container.record_location @top_level @@ -363,27 +381,23 @@ class RDoc::Parser::C < RDoc::Parser # */ # VALUE cFoo = rb_define_class("Foo", rb_cObject); - def find_class_comment(class_name, class_meth) + def find_class_comment(class_name, class_mod) comment = nil - if @content =~ %r{((?>/\*.*?\*/\s+)) - (static\s+)?void\s+Init_#{class_name}\s*(?:_\(\s*)?\(\s*(?:void\s*)\)}xmi then + + if @content =~ %r{ + ((?>/\*.*?\*/\s+)) + (static\s+)? + void\s+ + Init_#{class_name}\s*(?:_\(\s*)?\(\s*(?:void\s*)?\)}xmi then # ) comment = $1 - elsif @content =~ %r{Document-(?:class|module):\s#{class_name}\s*?(?:<\s+[:,\w]+)?\n((?>.*?\*/))}m + elsif @content =~ %r{Document-(?:class|module):\s+#{class_name}\s*?(?:<\s+[:,\w]+)?\n((?>.*?\*/))}m then + comment = $1 + elsif @content =~ %r{((?>/\*.*?\*/\s+)) + ([\w\.\s]+\s* = \s+)?rb_define_(class|module).*?"(#{class_name})"}xm then comment = $1 - else - if @content =~ /rb_define_(class|module)/m then - class_name = class_name.split("::").last - comments = [] - @content.split(/(\/\*.*?\*\/)\s*?\n/m).each_with_index do |chunk, index| - comments[index] = chunk - if chunk =~ /rb_define_(class|module).*?"(#{class_name})"/m then - comment = comments[index-1] - break - end - end - end end - class_meth.comment = mangle_comment(comment) if comment + + class_mod.comment = strip_stars comment if comment end ## @@ -434,67 +448,67 @@ class RDoc::Parser::C < RDoc::Parser def handle_attr(var_name, attr_name, reader, writer) rw = '' - if reader - #@stats.num_methods += 1 - rw << 'R' - end - if writer - #@stats.num_methods += 1 - rw << 'W' - end + rw << 'R' if reader + rw << 'W' if writer class_name = @known_classes[var_name] return unless class_name - class_obj = find_class(var_name, class_name) + class_obj = find_class(var_name, class_name) if class_obj comment = find_attr_comment(attr_name) - unless comment.empty? - comment = mangle_comment(comment) - end + comment = strip_stars comment att = RDoc::Attr.new '', attr_name, rw, comment + @stats.add_method att class_obj.add_attribute(att) end end - def handle_class_module(var_name, class_mod, class_name, parent, in_module) + def handle_class_module(var_name, type, class_name, parent, in_module) parent_name = @known_classes[parent] || parent - if in_module + if in_module then enclosure = @classes[in_module] || @@enclosure_classes[in_module] - unless enclosure - if enclosure = @known_classes[in_module] - handle_class_module(in_module, (/^rb_m/ =~ in_module ? "module" : "class"), - enclosure, nil, nil) - enclosure = @classes[in_module] - end + + if enclosure.nil? and enclosure = @known_classes[in_module] then + type = /^rb_m/ =~ in_module ? "module" : "class" + handle_class_module in_module, type, enclosure, nil, nil + enclosure = @classes[in_module] end - unless enclosure - warn("Enclosing class/module '#{in_module}' for " + - "#{class_mod} #{class_name} not known") + + unless enclosure then + warn("Enclosing class/module '#{in_module}' for #{type} #{class_name} not known") return end else enclosure = @top_level end - if class_mod == "class" then - full_name = enclosure.full_name.to_s + "::#{class_name}" + if type == "class" then + full_name = if RDoc::ClassModule === enclosure then + enclosure.full_name + "::#{class_name}" + else + class_name + end + if @content =~ %r{Document-class:\s+#{full_name}\s*<\s+([:,\w]+)} then parent_name = $1 end + cm = enclosure.add_class RDoc::NormalClass, class_name, parent_name + @stats.add_class cm else cm = enclosure.add_module RDoc::NormalModule, class_name @stats.add_module cm end - cm.record_location(enclosure.toplevel) + cm.record_location enclosure.top_level + + find_class_comment cm.full_name, cm - find_class_comment(cm.full_name, cm) @classes[var_name] = cm @@enclosure_classes[var_name] = cm @known_classes[var_name] = cm.full_name @@ -512,46 +526,55 @@ class RDoc::Parser::C < RDoc::Parser # Values may include quotes and escaped colons (\:). def handle_constants(type, var_name, const_name, definition) - #@stats.num_constants += 1 class_name = @known_classes[var_name] return unless class_name - class_obj = find_class(var_name, class_name) + class_obj = find_class var_name, class_name - unless class_obj - warn("Enclosing class/module '#{const_name}' for not known") + unless class_obj then + warn "Enclosing class/module #{const_name.inspect} not known" return end - comment = find_const_comment(type, const_name) + comment = find_const_comment type, const_name + comment = strip_stars comment + comment = normalize_comment comment # In the case of rb_define_const, the definition and comment are in # "/* definition: comment */" form. The literal ':' and '\' characters # can be escaped with a backslash. if type.downcase == 'const' then - elements = mangle_comment(comment).split(':') - if elements.nil? or elements.empty? then - con = RDoc::Constant.new(const_name, definition, - mangle_comment(comment)) - else - new_definition = elements[0..-2].join(':') - if new_definition.empty? then # Default to literal C definition - new_definition = definition - else - new_definition.gsub!("\:", ":") - new_definition.gsub!("\\", '\\') - end - new_definition.sub!(/\A(\s+)/, '') - new_comment = $1.nil? ? elements.last : "#{$1}#{elements.last.lstrip}" - con = RDoc::Constant.new(const_name, new_definition, - mangle_comment(new_comment)) - end + elements = comment.split ':' + + if elements.nil? or elements.empty? then + con = RDoc::Constant.new const_name, definition, comment + else + new_definition = elements[0..-2].join(':') + + if new_definition.empty? then # Default to literal C definition + new_definition = definition + else + new_definition.gsub!("\:", ":") + new_definition.gsub!("\\", '\\') + end + + new_definition.sub!(/\A(\s+)/, '') + + new_comment = if $1.nil? then + elements.last.lstrip + else + "#{$1}#{elements.last.lstrip}" + end + + con = RDoc::Constant.new const_name, new_definition, new_comment + end else - con = RDoc::Constant.new const_name, definition, mangle_comment(comment) + con = RDoc::Constant.new const_name, definition, comment end - class_obj.add_constant(con) + @stats.add_constant con + class_obj.add_constant con end ## @@ -588,9 +611,11 @@ class RDoc::Parser::C < RDoc::Parser meth_obj.params = "(" + (1..p_count).map{|i| "p#{i}"}.join(", ") + ")" end - if source_file then + if source_file and File.exist? source_file then file_name = File.join(@file_dir, source_file) body = (@@known_bodies[source_file] ||= File.read(file_name)) + elsif source_file then + warn "unknown source file #{source_file}" else body = @content end @@ -598,6 +623,7 @@ class RDoc::Parser::C < RDoc::Parser if find_body(class_name, meth_body, meth_obj, body) and meth_obj.document_self then class_obj.add_method meth_obj @stats.add_method meth_obj + meth_obj.visibility = :private if 'private_method' == type end end end @@ -615,16 +641,6 @@ class RDoc::Parser::C < RDoc::Parser end ## - # Remove the /*'s and leading asterisks from C comments - - def mangle_comment(comment) - comment.sub!(%r{/\*+}) { " " * $&.length } - comment.sub!(%r{\*+/}) { " " * $&.length } - comment.gsub!(/^[ \t]*\*/m) { " " * $&.length } - comment - end - - ## # Removes lines that are commented out that might otherwise get picked up # when scanning for classes and methods @@ -651,11 +667,5 @@ class RDoc::Parser::C < RDoc::Parser @top_level end - def warn(msg) - $stderr.puts - $stderr.puts msg - $stderr.flush - end - end diff --git a/lib/rdoc/parser/f95.rb b/lib/rdoc/parser/f95.rb deleted file mode 100644 index 0959db4936..0000000000 --- a/lib/rdoc/parser/f95.rb +++ /dev/null @@ -1,1835 +0,0 @@ -require 'rdoc/parser' - -## -# = Fortran95 RDoc Parser -# -# == Overview -# -# This parser parses Fortran95 files with suffixes "f90", "F90", "f95" and -# "F95". Fortran95 files are expected to be conformed to Fortran95 standards. -# -# == Rules -# -# Fundamental rules are same as that of the Ruby parser. But comment markers -# are '!' not '#'. -# -# === Correspondence between RDoc documentation and Fortran95 programs -# -# F95 parses main programs, modules, subroutines, functions, derived-types, -# public variables, public constants, defined operators and defined -# assignments. These components are described in items of RDoc documentation, -# as follows. -# -# Files :: Files (same as Ruby) -# Classes:: Modules -# Methods:: Subroutines, functions, variables, constants, derived-types, -# defined operators, defined assignments -# Required files:: Files in which imported modules, external subroutines and -# external functions are defined. -# Included Modules:: List of imported modules -# Attributes:: List of derived-types, List of imported modules all of whose -# components are published again -# -# Components listed in 'Methods' (subroutines, functions, ...) defined in -# modules are described in the item of 'Classes'. On the other hand, -# components defined in main programs or as external procedures are described -# in the item of 'Files'. -# -# === Components parsed by default -# -# By default, documentation on public components (subroutines, functions, -# variables, constants, derived-types, defined operators, defined assignments) -# are generated. -# -# With "--all" option, documentation on all components are generated (almost -# same as the Ruby parser). -# -# === Information parsed automatically -# -# The following information is automatically parsed. -# -# * Types of arguments -# * Types of variables and constants -# * Types of variables in the derived types, and initial values -# * NAMELISTs and types of variables in them, and initial values -# -# Aliases by interface statement are described in the item of 'Methods'. -# -# Components which are imported from other modules and published again are -# described in the item of 'Methods'. -# -# === Format of comment blocks -# -# Comment blocks should be written as follows. -# -# Comment blocks are considered to be ended when the line without '!' appears. -# -# The indentation is not necessary. -# -# ! (Top of file) -# ! -# ! Comment blocks for the files. -# ! -# !-- -# ! The comment described in the part enclosed by -# ! "!--" and "!++" is ignored. -# !++ -# ! -# module hogehoge -# ! -# ! Comment blocks for the modules (or the programs). -# ! -# -# private -# -# logical :: a ! a private variable -# real, public :: b ! a public variable -# integer, parameter :: c = 0 ! a public constant -# -# public :: c -# public :: MULTI_ARRAY -# public :: hoge, foo -# -# type MULTI_ARRAY -# ! -# ! Comment blocks for the derived-types. -# ! -# real, pointer :: var(:) =>null() ! Comments block for the variables. -# integer :: num = 0 -# end type MULTI_ARRAY -# -# contains -# -# subroutine hoge( in, & ! Comment blocks between continuation lines are ignored. -# & out ) -# ! -# ! Comment blocks for the subroutines or functions -# ! -# character(*),intent(in):: in ! Comment blocks for the arguments. -# character(*),intent(out),allocatable,target :: in -# ! Comment blocks can be -# ! written under Fortran statements. -# -# character(32) :: file ! This comment parsed as a variable in below NAMELIST. -# integer :: id -# -# namelist /varinfo_nml/ file, id -# ! -# ! Comment blocks for the NAMELISTs. -# ! Information about variables are described above. -# ! -# -# .... -# -# end subroutine hoge -# -# integer function foo( in ) -# ! -# ! This part is considered as comment block. -# -# ! Comment blocks under blank lines are ignored. -# ! -# integer, intent(in):: inA ! This part is considered as comment block. -# -# ! This part is ignored. -# -# end function foo -# -# subroutine hide( in, & -# & out ) !:nodoc: -# ! -# ! If "!:nodoc:" is described at end-of-line in subroutine -# ! statement as above, the subroutine is ignored. -# ! This assignment can be used to modules, subroutines, -# ! functions, variables, constants, derived-types, -# ! defined operators, defined assignments, -# ! list of imported modules ("use" statement). -# ! -# -# .... -# -# end subroutine hide -# -# end module hogehoge - -class RDoc::Parser::F95 < RDoc::Parser - - parse_files_matching(/\.((f|F)9(0|5)|F)$/) - - class Token - - NO_TEXT = "??".freeze - - def initialize(line_no, char_no) - @line_no = line_no - @char_no = char_no - @text = NO_TEXT - end - # Because we're used in contexts that expect to return a token, - # we set the text string and then return ourselves - def set_text(text) - @text = text - self - end - - attr_reader :line_no, :char_no, :text - - end - - @@external_aliases = [] - @@public_methods = [] - - ## - # "false":: Comments are below source code - # "true" :: Comments are upper source code - - COMMENTS_ARE_UPPER = false - - ## - # Internal alias message - - INTERNAL_ALIAS_MES = "Alias for" - - ## - # External alias message - - EXTERNAL_ALIAS_MES = "The entity is" - - ## - # Define code constructs - - def scan - # remove private comment - remaining_code = remove_private_comments(@content) - - # continuation lines are united to one line - remaining_code = united_to_one_line(remaining_code) - - # semicolons are replaced to line feed - remaining_code = semicolon_to_linefeed(remaining_code) - - # collect comment for file entity - whole_comment, remaining_code = collect_first_comment(remaining_code) - @top_level.comment = whole_comment - - # String "remaining_code" is converted to Array "remaining_lines" - remaining_lines = remaining_code.split("\n") - - # "module" or "program" parts are parsed (new) - # - level_depth = 0 - block_searching_flag = nil - block_searching_lines = [] - pre_comment = [] - module_program_trailing = "" - module_program_name = "" - other_block_level_depth = 0 - other_block_searching_flag = nil - remaining_lines.collect!{|line| - if !block_searching_flag && !other_block_searching_flag - if line =~ /^\s*?module\s+(\w+)\s*?(!.*?)?$/i - block_searching_flag = :module - block_searching_lines << line - module_program_name = $1 - module_program_trailing = find_comments($2) - next false - elsif line =~ /^\s*?program\s+(\w+)\s*?(!.*?)?$/i || - line =~ /^\s*?\w/ && !block_start?(line) - block_searching_flag = :program - block_searching_lines << line - module_program_name = $1 || "" - module_program_trailing = find_comments($2) - next false - - elsif block_start?(line) - other_block_searching_flag = true - next line - - elsif line =~ /^\s*?!\s?(.*)/ - pre_comment << line - next line - else - pre_comment = [] - next line - end - elsif other_block_searching_flag - other_block_level_depth += 1 if block_start?(line) - other_block_level_depth -= 1 if block_end?(line) - if other_block_level_depth < 0 - other_block_level_depth = 0 - other_block_searching_flag = nil - end - next line - end - - block_searching_lines << line - level_depth += 1 if block_start?(line) - level_depth -= 1 if block_end?(line) - if level_depth >= 0 - next false - end - - # "module_program_code" is formatted. - # ":nodoc:" flag is checked. - # - module_program_code = block_searching_lines.join("\n") - module_program_code = remove_empty_head_lines(module_program_code) - if module_program_trailing =~ /^:nodoc:/ - # next loop to search next block - level_depth = 0 - block_searching_flag = false - block_searching_lines = [] - pre_comment = [] - next false - end - - # NormalClass is created, and added to @top_level - # - if block_searching_flag == :module - module_name = module_program_name - module_code = module_program_code - module_trailing = module_program_trailing - - f9x_module = @top_level.add_module NormalClass, module_name - f9x_module.record_location @top_level - - @stats.add_module f9x_module - - f9x_comment = COMMENTS_ARE_UPPER ? - find_comments(pre_comment.join("\n")) + "\n" + module_trailing : - module_trailing + "\n" + find_comments(module_code.sub(/^.*$\n/i, '')) - f9x_module.comment = f9x_comment - parse_program_or_module(f9x_module, module_code) - - TopLevel.all_files.each do |name, toplevel| - if toplevel.include_includes?(module_name, @options.ignore_case) - if !toplevel.include_requires?(@file_name, @options.ignore_case) - toplevel.add_require(Require.new(@file_name, "")) - end - end - toplevel.each_classmodule{|m| - if m.include_includes?(module_name, @options.ignore_case) - if !m.include_requires?(@file_name, @options.ignore_case) - m.add_require(Require.new(@file_name, "")) - end - end - } - end - elsif block_searching_flag == :program - program_name = module_program_name - program_code = module_program_code - program_trailing = module_program_trailing - # progress "p" # HACK what stats thingy does this correspond to? - program_comment = COMMENTS_ARE_UPPER ? - find_comments(pre_comment.join("\n")) + "\n" + program_trailing : - program_trailing + "\n" + find_comments(program_code.sub(/^.*$\n/i, '')) - program_comment = "\n\n= <i>Program</i> <tt>#{program_name}</tt>\n\n" \ - + program_comment - @top_level.comment << program_comment - parse_program_or_module(@top_level, program_code, :private) - end - - # next loop to search next block - level_depth = 0 - block_searching_flag = false - block_searching_lines = [] - pre_comment = [] - next false - } - - remaining_lines.delete_if{ |line| - line == false - } - - # External subprograms and functions are parsed - # - parse_program_or_module(@top_level, remaining_lines.join("\n"), - :public, true) - - @top_level - end # End of scan - - private - - def parse_program_or_module(container, code, - visibility=:public, external=nil) - return unless container - return unless code - remaining_lines = code.split("\n") - remaining_code = "#{code}" - - # - # Parse variables before "contains" in module - # - level_depth = 0 - before_contains_lines = [] - before_contains_code = nil - before_contains_flag = nil - remaining_lines.each{ |line| - if !before_contains_flag - if line =~ /^\s*?module\s+\w+\s*?(!.*?)?$/i - before_contains_flag = true - end - else - break if line =~ /^\s*?contains\s*?(!.*?)?$/i - level_depth += 1 if block_start?(line) - level_depth -= 1 if block_end?(line) - break if level_depth < 0 - before_contains_lines << line - end - } - before_contains_code = before_contains_lines.join("\n") - if before_contains_code - before_contains_code.gsub!(/^\s*?interface\s+.*?\s+end\s+interface.*?$/im, "") - before_contains_code.gsub!(/^\s*?type[\s\,]+.*?\s+end\s+type.*?$/im, "") - end - - # - # Parse global "use" - # - use_check_code = "#{before_contains_code}" - cascaded_modules_list = [] - while use_check_code =~ /^\s*?use\s+(\w+)(.*?)(!.*?)?$/i - use_check_code = $~.pre_match - use_check_code << $~.post_match - used_mod_name = $1.strip.chomp - used_list = $2 || "" - used_trailing = $3 || "" - next if used_trailing =~ /!:nodoc:/ - if !container.include_includes?(used_mod_name, @options.ignore_case) - # progress "." # HACK what stats thingy does this correspond to? - container.add_include Include.new(used_mod_name, "") - end - if ! (used_list =~ /\,\s*?only\s*?:/i ) - cascaded_modules_list << "\#" + used_mod_name - end - end - - # - # Parse public and private, and store information. - # This information is used when "add_method" and - # "set_visibility_for" are called. - # - visibility_default, visibility_info = - parse_visibility(remaining_lines.join("\n"), visibility, container) - @@public_methods.concat visibility_info - if visibility_default == :public - if !cascaded_modules_list.empty? - cascaded_modules = - Attr.new("Cascaded Modules", - "Imported modules all of whose components are published again", - "", - cascaded_modules_list.join(", ")) - container.add_attribute(cascaded_modules) - end - end - - # - # Check rename elements - # - use_check_code = "#{before_contains_code}" - while use_check_code =~ /^\s*?use\s+(\w+)\s*?\,(.+)$/i - use_check_code = $~.pre_match - use_check_code << $~.post_match - used_mod_name = $1.strip.chomp - used_elements = $2.sub(/\s*?only\s*?:\s*?/i, '') - used_elements.split(",").each{ |used| - if /\s*?(\w+)\s*?=>\s*?(\w+)\s*?/ =~ used - local = $1 - org = $2 - @@public_methods.collect!{ |pub_meth| - if local == pub_meth["name"] || - local.upcase == pub_meth["name"].upcase && - @options.ignore_case - pub_meth["name"] = org - pub_meth["local_name"] = local - end - pub_meth - } - end - } - end - - # - # Parse private "use" - # - use_check_code = remaining_lines.join("\n") - while use_check_code =~ /^\s*?use\s+(\w+)(.*?)(!.*?)?$/i - use_check_code = $~.pre_match - use_check_code << $~.post_match - used_mod_name = $1.strip.chomp - used_trailing = $3 || "" - next if used_trailing =~ /!:nodoc:/ - if !container.include_includes?(used_mod_name, @options.ignore_case) - # progress "." # HACK what stats thingy does this correspond to? - container.add_include Include.new(used_mod_name, "") - end - end - - container.each_includes{ |inc| - TopLevel.all_files.each do |name, toplevel| - indicated_mod = toplevel.find_symbol(inc.name, - nil, @options.ignore_case) - if indicated_mod - indicated_name = indicated_mod.parent.file_relative_name - if !container.include_requires?(indicated_name, @options.ignore_case) - container.add_require(Require.new(indicated_name, "")) - end - break - end - end - } - - # - # Parse derived-types definitions - # - derived_types_comment = "" - remaining_code = remaining_lines.join("\n") - while remaining_code =~ /^\s*? - type[\s\,]+(public|private)?\s*?(::)?\s*? - (\w+)\s*?(!.*?)?$ - (.*?) - ^\s*?end\s+type.*?$ - /imx - remaining_code = $~.pre_match - remaining_code << $~.post_match - typename = $3.chomp.strip - type_elements = $5 || "" - type_code = remove_empty_head_lines($&) - type_trailing = find_comments($4) - next if type_trailing =~ /^:nodoc:/ - type_visibility = $1 - type_comment = COMMENTS_ARE_UPPER ? - find_comments($~.pre_match) + "\n" + type_trailing : - type_trailing + "\n" + find_comments(type_code.sub(/^.*$\n/i, '')) - type_element_visibility_public = true - type_code.split("\n").each{ |line| - if /^\s*?private\s*?$/ =~ line - type_element_visibility_public = nil - break - end - } if type_code - - args_comment = "" - type_args_info = nil - - if @options.show_all - args_comment = find_arguments(nil, type_code, true) - else - type_public_args_list = [] - type_args_info = definition_info(type_code) - type_args_info.each{ |arg| - arg_is_public = type_element_visibility_public - arg_is_public = true if arg.include_attr?("public") - arg_is_public = nil if arg.include_attr?("private") - type_public_args_list << arg.varname if arg_is_public - } - args_comment = find_arguments(type_public_args_list, type_code) - end - - type = AnyMethod.new("type #{typename}", typename) - type.singleton = false - type.params = "" - type.comment = "<b><em> Derived Type </em></b> :: <tt></tt>\n" - type.comment << args_comment if args_comment - type.comment << type_comment if type_comment - - @stats.add_method type - - container.add_method type - - set_visibility(container, typename, visibility_default, @@public_methods) - - if type_visibility - type_visibility.gsub!(/\s/,'') - type_visibility.gsub!(/\,/,'') - type_visibility.gsub!(/:/,'') - type_visibility.downcase! - if type_visibility == "public" - container.set_visibility_for([typename], :public) - elsif type_visibility == "private" - container.set_visibility_for([typename], :private) - end - end - - check_public_methods(type, container.name) - - if @options.show_all - derived_types_comment << ", " unless derived_types_comment.empty? - derived_types_comment << typename - else - if type.visibility == :public - derived_types_comment << ", " unless derived_types_comment.empty? - derived_types_comment << typename - end - end - - end - - if !derived_types_comment.empty? - derived_types_table = - Attr.new("Derived Types", "Derived_Types", "", - derived_types_comment) - container.add_attribute(derived_types_table) - end - - # - # move interface scope - # - interface_code = "" - while remaining_code =~ /^\s*? - interface( - \s+\w+ | - \s+operator\s*?\(.*?\) | - \s+assignment\s*?\(\s*?=\s*?\) - )?\s*?$ - (.*?) - ^\s*?end\s+interface.*?$ - /imx - interface_code << remove_empty_head_lines($&) + "\n" - remaining_code = $~.pre_match - remaining_code << $~.post_match - end - - # - # Parse global constants or variables in modules - # - const_var_defs = definition_info(before_contains_code) - const_var_defs.each{|defitem| - next if defitem.nodoc - const_or_var_type = "Variable" - const_or_var_progress = "v" - if defitem.include_attr?("parameter") - const_or_var_type = "Constant" - const_or_var_progress = "c" - end - const_or_var = AnyMethod.new(const_or_var_type, defitem.varname) - const_or_var.singleton = false - const_or_var.params = "" - self_comment = find_arguments([defitem.varname], before_contains_code) - const_or_var.comment = "<b><em>" + const_or_var_type + "</em></b> :: <tt></tt>\n" - const_or_var.comment << self_comment if self_comment - - @stats.add_method const_or_var_progress - - container.add_method const_or_var - - set_visibility(container, defitem.varname, visibility_default, @@public_methods) - - if defitem.include_attr?("public") - container.set_visibility_for([defitem.varname], :public) - elsif defitem.include_attr?("private") - container.set_visibility_for([defitem.varname], :private) - end - - check_public_methods(const_or_var, container.name) - - } if const_var_defs - - remaining_lines = remaining_code.split("\n") - - # "subroutine" or "function" parts are parsed (new) - # - level_depth = 0 - block_searching_flag = nil - block_searching_lines = [] - pre_comment = [] - procedure_trailing = "" - procedure_name = "" - procedure_params = "" - procedure_prefix = "" - procedure_result_arg = "" - procedure_type = "" - contains_lines = [] - contains_flag = nil - remaining_lines.collect!{|line| - if !block_searching_flag - # subroutine - if line =~ /^\s*? - (recursive|pure|elemental)?\s*? - subroutine\s+(\w+)\s*?(\(.*?\))?\s*?(!.*?)?$ - /ix - block_searching_flag = :subroutine - block_searching_lines << line - - procedure_name = $2.chomp.strip - procedure_params = $3 || "" - procedure_prefix = $1 || "" - procedure_trailing = $4 || "!" - next false - - # function - elsif line =~ /^\s*? - (recursive|pure|elemental)?\s*? - ( - character\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | type\s*?\([\w\s]+?\)\s+ - | integer\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | real\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | double\s+precision\s+ - | logical\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | complex\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - )? - function\s+(\w+)\s*? - (\(.*?\))?(\s+result\((.*?)\))?\s*?(!.*?)?$ - /ix - block_searching_flag = :function - block_searching_lines << line - - procedure_prefix = $1 || "" - procedure_type = $2 ? $2.chomp.strip : nil - procedure_name = $8.chomp.strip - procedure_params = $9 || "" - procedure_result_arg = $11 ? $11.chomp.strip : procedure_name - procedure_trailing = $12 || "!" - next false - elsif line =~ /^\s*?!\s?(.*)/ - pre_comment << line - next line - else - pre_comment = [] - next line - end - end - contains_flag = true if line =~ /^\s*?contains\s*?(!.*?)?$/ - block_searching_lines << line - contains_lines << line if contains_flag - - level_depth += 1 if block_start?(line) - level_depth -= 1 if block_end?(line) - if level_depth >= 0 - next false - end - - # "procedure_code" is formatted. - # ":nodoc:" flag is checked. - # - procedure_code = block_searching_lines.join("\n") - procedure_code = remove_empty_head_lines(procedure_code) - if procedure_trailing =~ /^!:nodoc:/ - # next loop to search next block - level_depth = 0 - block_searching_flag = nil - block_searching_lines = [] - pre_comment = [] - procedure_trailing = "" - procedure_name = "" - procedure_params = "" - procedure_prefix = "" - procedure_result_arg = "" - procedure_type = "" - contains_lines = [] - contains_flag = nil - next false - end - - # AnyMethod is created, and added to container - # - subroutine_function = nil - if block_searching_flag == :subroutine - subroutine_prefix = procedure_prefix - subroutine_name = procedure_name - subroutine_params = procedure_params - subroutine_trailing = procedure_trailing - subroutine_code = procedure_code - - subroutine_comment = COMMENTS_ARE_UPPER ? - pre_comment.join("\n") + "\n" + subroutine_trailing : - subroutine_trailing + "\n" + subroutine_code.sub(/^.*$\n/i, '') - subroutine = AnyMethod.new("subroutine", subroutine_name) - parse_subprogram(subroutine, subroutine_params, - subroutine_comment, subroutine_code, - before_contains_code, nil, subroutine_prefix) - - @stats.add_method subroutine - - container.add_method subroutine - subroutine_function = subroutine - - elsif block_searching_flag == :function - function_prefix = procedure_prefix - function_type = procedure_type - function_name = procedure_name - function_params_org = procedure_params - function_result_arg = procedure_result_arg - function_trailing = procedure_trailing - function_code_org = procedure_code - - function_comment = COMMENTS_ARE_UPPER ? - pre_comment.join("\n") + "\n" + function_trailing : - function_trailing + "\n " + function_code_org.sub(/^.*$\n/i, '') - - function_code = "#{function_code_org}" - if function_type - function_code << "\n" + function_type + " :: " + function_result_arg - end - - function_params = - function_params_org.sub(/^\(/, "\(#{function_result_arg}, ") - - function = AnyMethod.new("function", function_name) - parse_subprogram(function, function_params, - function_comment, function_code, - before_contains_code, true, function_prefix) - - # Specific modification due to function - function.params.sub!(/\(\s*?#{function_result_arg}\s*?,\s*?/, "\( ") - function.params << " result(" + function_result_arg + ")" - function.start_collecting_tokens - function.add_token Token.new(1,1).set_text(function_code_org) - - @stats.add_method function - - container.add_method function - subroutine_function = function - - end - - # The visibility of procedure is specified - # - set_visibility(container, procedure_name, - visibility_default, @@public_methods) - - # The alias for this procedure from external modules - # - check_external_aliases(procedure_name, - subroutine_function.params, - subroutine_function.comment, subroutine_function) if external - check_public_methods(subroutine_function, container.name) - - - # contains_lines are parsed as private procedures - if contains_flag - parse_program_or_module(container, - contains_lines.join("\n"), :private) - end - - # next loop to search next block - level_depth = 0 - block_searching_flag = nil - block_searching_lines = [] - pre_comment = [] - procedure_trailing = "" - procedure_name = "" - procedure_params = "" - procedure_prefix = "" - procedure_result_arg = "" - contains_lines = [] - contains_flag = nil - next false - } # End of remaining_lines.collect!{|line| - - # Array remains_lines is converted to String remains_code again - # - remaining_code = remaining_lines.join("\n") - - # - # Parse interface - # - interface_scope = false - generic_name = "" - interface_code.split("\n").each{ |line| - if /^\s*? - interface( - \s+\w+| - \s+operator\s*?\(.*?\)| - \s+assignment\s*?\(\s*?=\s*?\) - )? - \s*?(!.*?)?$ - /ix =~ line - generic_name = $1 ? $1.strip.chomp : nil - interface_trailing = $2 || "!" - interface_scope = true - interface_scope = false if interface_trailing =~ /!:nodoc:/ -# if generic_name =~ /operator\s*?\((.*?)\)/i -# operator_name = $1 -# if operator_name && !operator_name.empty? -# generic_name = "#{operator_name}" -# end -# end -# if generic_name =~ /assignment\s*?\((.*?)\)/i -# assignment_name = $1 -# if assignment_name && !assignment_name.empty? -# generic_name = "#{assignment_name}" -# end -# end - end - if /^\s*?end\s+interface/i =~ line - interface_scope = false - generic_name = nil - end - # internal alias - if interface_scope && /^\s*?module\s+procedure\s+(.*?)(!.*?)?$/i =~ line - procedures = $1.strip.chomp - procedures_trailing = $2 || "!" - next if procedures_trailing =~ /!:nodoc:/ - procedures.split(",").each{ |proc| - proc.strip! - proc.chomp! - next if generic_name == proc || !generic_name - old_meth = container.find_symbol(proc, nil, @options.ignore_case) - next if !old_meth - nolink = old_meth.visibility == :private ? true : nil - nolink = nil if @options.show_all - new_meth = - initialize_external_method(generic_name, proc, - old_meth.params, nil, - old_meth.comment, - old_meth.clone.token_stream[0].text, - true, nolink) - new_meth.singleton = old_meth.singleton - - @stats.add_method new_meth - - container.add_method new_meth - - set_visibility(container, generic_name, visibility_default, @@public_methods) - - check_public_methods(new_meth, container.name) - - } - end - - # external aliases - if interface_scope - # subroutine - proc = nil - params = nil - procedures_trailing = nil - if line =~ /^\s*? - (recursive|pure|elemental)?\s*? - subroutine\s+(\w+)\s*?(\(.*?\))?\s*?(!.*?)?$ - /ix - proc = $2.chomp.strip - generic_name = proc unless generic_name - params = $3 || "" - procedures_trailing = $4 || "!" - - # function - elsif line =~ /^\s*? - (recursive|pure|elemental)?\s*? - ( - character\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | type\s*?\([\w\s]+?\)\s+ - | integer\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | real\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | double\s+precision\s+ - | logical\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | complex\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - )? - function\s+(\w+)\s*? - (\(.*?\))?(\s+result\((.*?)\))?\s*?(!.*?)?$ - /ix - proc = $8.chomp.strip - generic_name = proc unless generic_name - params = $9 || "" - procedures_trailing = $12 || "!" - else - next - end - next if procedures_trailing =~ /!:nodoc:/ - indicated_method = nil - indicated_file = nil - TopLevel.all_files.each do |name, toplevel| - indicated_method = toplevel.find_local_symbol(proc, @options.ignore_case) - indicated_file = name - break if indicated_method - end - - if indicated_method - external_method = - initialize_external_method(generic_name, proc, - indicated_method.params, - indicated_file, - indicated_method.comment) - - @stats.add_method external_method - - container.add_method external_method - set_visibility(container, generic_name, visibility_default, @@public_methods) - if !container.include_requires?(indicated_file, @options.ignore_case) - container.add_require(Require.new(indicated_file, "")) - end - check_public_methods(external_method, container.name) - - else - @@external_aliases << { - "new_name" => generic_name, - "old_name" => proc, - "file_or_module" => container, - "visibility" => find_visibility(container, generic_name, @@public_methods) || visibility_default - } - end - end - - } if interface_code # End of interface_code.split("\n").each ... - - # - # Already imported methods are removed from @@public_methods. - # Remainders are assumed to be imported from other modules. - # - @@public_methods.delete_if{ |method| method["entity_is_discovered"]} - - @@public_methods.each{ |pub_meth| - next unless pub_meth["file_or_module"].name == container.name - pub_meth["used_modules"].each{ |used_mod| - TopLevel.all_classes_and_modules.each{ |modules| - if modules.name == used_mod || - modules.name.upcase == used_mod.upcase && - @options.ignore_case - modules.method_list.each{ |meth| - if meth.name == pub_meth["name"] || - meth.name.upcase == pub_meth["name"].upcase && - @options.ignore_case - new_meth = initialize_public_method(meth, - modules.name) - if pub_meth["local_name"] - new_meth.name = pub_meth["local_name"] - end - - @stats.add_method new_meth - - container.add_method new_meth - end - } - end - } - } - } - - container - end # End of parse_program_or_module - - ## - # Parse arguments, comment, code of subroutine and function. Return - # AnyMethod object. - - def parse_subprogram(subprogram, params, comment, code, - before_contains=nil, function=nil, prefix=nil) - subprogram.singleton = false - prefix = "" if !prefix - arguments = params.sub(/\(/, "").sub(/\)/, "").split(",") if params - args_comment, params_opt = - find_arguments(arguments, code.sub(/^s*?contains\s*?(!.*?)?$.*/im, ""), - nil, nil, true) - params_opt = "( " + params_opt + " ) " if params_opt - subprogram.params = params_opt || "" - namelist_comment = find_namelists(code, before_contains) - - block_comment = find_comments comment - if function - subprogram.comment = "<b><em> Function </em></b> :: <em>#{prefix}</em>\n" - else - subprogram.comment = "<b><em> Subroutine </em></b> :: <em>#{prefix}</em>\n" - end - subprogram.comment << args_comment if args_comment - subprogram.comment << block_comment if block_comment - subprogram.comment << namelist_comment if namelist_comment - - # For output source code - subprogram.start_collecting_tokens - subprogram.add_token Token.new(1,1).set_text(code) - - subprogram - end - - ## - # Collect comment for file entity - - def collect_first_comment(body) - comment = "" - not_comment = "" - comment_start = false - comment_end = false - body.split("\n").each{ |line| - if comment_end - not_comment << line - not_comment << "\n" - elsif /^\s*?!\s?(.*)$/i =~ line - comment_start = true - comment << $1 - comment << "\n" - elsif /^\s*?$/i =~ line - comment_end = true if comment_start && COMMENTS_ARE_UPPER - else - comment_end = true - not_comment << line - not_comment << "\n" - end - } - return comment, not_comment - end - - - ## - # Return comments of definitions of arguments - # - # If "all" argument is true, information of all arguments are returned. - # - # If "modified_params" is true, list of arguments are decorated, for - # example, optional arguments are parenthetic as "[arg]". - - def find_arguments(args, text, all=nil, indent=nil, modified_params=nil) - return unless args || all - indent = "" unless indent - args = ["all"] if all - params = "" if modified_params - comma = "" - return unless text - args_rdocforms = "\n" - remaining_lines = "#{text}" - definitions = definition_info(remaining_lines) - args.each{ |arg| - arg.strip! - arg.chomp! - definitions.each { |defitem| - if arg == defitem.varname.strip.chomp || all - args_rdocforms << <<-"EOF" - -#{indent}<tt><b>#{defitem.varname.chomp.strip}#{defitem.arraysuffix}</b> #{defitem.inivalue}</tt> :: -#{indent} <tt>#{defitem.types.chomp.strip}</tt> -EOF - if !defitem.comment.chomp.strip.empty? - comment = "" - defitem.comment.split("\n").each{ |line| - comment << " " + line + "\n" - } - args_rdocforms << <<-"EOF" - -#{indent} <tt></tt> :: -#{indent} <tt></tt> -#{indent} #{comment.chomp.strip} -EOF - end - - if modified_params - if defitem.include_attr?("optional") - params << "#{comma}[#{arg}]" - else - params << "#{comma}#{arg}" - end - comma = ", " - end - end - } - } - if modified_params - return args_rdocforms, params - else - return args_rdocforms - end - end - - ## - # Return comments of definitions of namelists - - def find_namelists(text, before_contains=nil) - return nil if !text - result = "" - lines = "#{text}" - before_contains = "" if !before_contains - while lines =~ /^\s*?namelist\s+\/\s*?(\w+)\s*?\/([\s\w\,]+)$/i - lines = $~.post_match - nml_comment = COMMENTS_ARE_UPPER ? - find_comments($~.pre_match) : find_comments($~.post_match) - nml_name = $1 - nml_args = $2.split(",") - result << "\n\n=== NAMELIST <tt><b>" + nml_name + "</tt></b>\n\n" - result << nml_comment + "\n" if nml_comment - if lines.split("\n")[0] =~ /^\//i - lines = "namelist " + lines - end - result << find_arguments(nml_args, "#{text}" + "\n" + before_contains) - end - return result - end - - ## - # Comments just after module or subprogram, or arguments are returned. If - # "COMMENTS_ARE_UPPER" is true, comments just before modules or subprograms - # are returnd - - def find_comments text - return "" unless text - lines = text.split("\n") - lines.reverse! if COMMENTS_ARE_UPPER - comment_block = Array.new - lines.each do |line| - break if line =~ /^\s*?\w/ || line =~ /^\s*?$/ - if COMMENTS_ARE_UPPER - comment_block.unshift line.sub(/^\s*?!\s?/,"") - else - comment_block.push line.sub(/^\s*?!\s?/,"") - end - end - nice_lines = comment_block.join("\n").split "\n\s*?\n" - nice_lines[0] ||= "" - nice_lines.shift - end - - ## - # Create method for internal alias - - def initialize_public_method(method, parent) - return if !method || !parent - - new_meth = AnyMethod.new("External Alias for module", method.name) - new_meth.singleton = method.singleton - new_meth.params = method.params.clone - new_meth.comment = remove_trailing_alias(method.comment.clone) - new_meth.comment << "\n\n#{EXTERNAL_ALIAS_MES} #{parent.strip.chomp}\##{method.name}" - - return new_meth - end - - ## - # Create method for external alias - # - # If argument "internal" is true, file is ignored. - - def initialize_external_method(new, old, params, file, comment, token=nil, - internal=nil, nolink=nil) - return nil unless new || old - - if internal - external_alias_header = "#{INTERNAL_ALIAS_MES} " - external_alias_text = external_alias_header + old - elsif file - external_alias_header = "#{EXTERNAL_ALIAS_MES} " - external_alias_text = external_alias_header + file + "#" + old - else - return nil - end - external_meth = AnyMethod.new(external_alias_text, new) - external_meth.singleton = false - external_meth.params = params - external_comment = remove_trailing_alias(comment) + "\n\n" if comment - external_meth.comment = external_comment || "" - if nolink && token - external_meth.start_collecting_tokens - external_meth.add_token Token.new(1,1).set_text(token) - else - external_meth.comment << external_alias_text - end - - return external_meth - end - - ## - # Parse visibility - - def parse_visibility(code, default, container) - result = [] - visibility_default = default || :public - - used_modules = [] - container.includes.each{|i| used_modules << i.name} if container - - remaining_code = code.gsub(/^\s*?type[\s\,]+.*?\s+end\s+type.*?$/im, "") - remaining_code.split("\n").each{ |line| - if /^\s*?private\s*?$/ =~ line - visibility_default = :private - break - end - } if remaining_code - - remaining_code.split("\n").each{ |line| - if /^\s*?private\s*?(::)?\s+(.*)\s*?(!.*?)?/i =~ line - methods = $2.sub(/!.*$/, '') - methods.split(",").each{ |meth| - meth.sub!(/!.*$/, '') - meth.gsub!(/:/, '') - result << { - "name" => meth.chomp.strip, - "visibility" => :private, - "used_modules" => used_modules.clone, - "file_or_module" => container, - "entity_is_discovered" => nil, - "local_name" => nil - } - } - elsif /^\s*?public\s*?(::)?\s+(.*)\s*?(!.*?)?/i =~ line - methods = $2.sub(/!.*$/, '') - methods.split(",").each{ |meth| - meth.sub!(/!.*$/, '') - meth.gsub!(/:/, '') - result << { - "name" => meth.chomp.strip, - "visibility" => :public, - "used_modules" => used_modules.clone, - "file_or_module" => container, - "entity_is_discovered" => nil, - "local_name" => nil - } - } - end - } if remaining_code - - if container - result.each{ |vis_info| - vis_info["parent"] = container.name - } - end - - return visibility_default, result - end - - ## - # Set visibility - # - # "subname" element of "visibility_info" is deleted. - - def set_visibility(container, subname, visibility_default, visibility_info) - return unless container || subname || visibility_default || visibility_info - not_found = true - visibility_info.collect!{ |info| - if info["name"] == subname || - @options.ignore_case && info["name"].upcase == subname.upcase - if info["file_or_module"].name == container.name - container.set_visibility_for([subname], info["visibility"]) - info["entity_is_discovered"] = true - not_found = false - end - end - info - } - if not_found - return container.set_visibility_for([subname], visibility_default) - else - return container - end - end - - ## - # Find visibility - - def find_visibility(container, subname, visibility_info) - return nil if !subname || !visibility_info - visibility_info.each{ |info| - if info["name"] == subname || - @options.ignore_case && info["name"].upcase == subname.upcase - if info["parent"] == container.name - return info["visibility"] - end - end - } - return nil - end - - ## - # Check external aliases - - def check_external_aliases(subname, params, comment, test=nil) - @@external_aliases.each{ |alias_item| - if subname == alias_item["old_name"] || - subname.upcase == alias_item["old_name"].upcase && - @options.ignore_case - - new_meth = initialize_external_method(alias_item["new_name"], - subname, params, @file_name, - comment) - new_meth.visibility = alias_item["visibility"] - - @stats.add_method new_meth - - alias_item["file_or_module"].add_method(new_meth) - - if !alias_item["file_or_module"].include_requires?(@file_name, @options.ignore_case) - alias_item["file_or_module"].add_require(Require.new(@file_name, "")) - end - end - } - end - - ## - # Check public_methods - - def check_public_methods(method, parent) - return if !method || !parent - @@public_methods.each{ |alias_item| - parent_is_used_module = nil - alias_item["used_modules"].each{ |used_module| - if used_module == parent || - used_module.upcase == parent.upcase && - @options.ignore_case - parent_is_used_module = true - end - } - next if !parent_is_used_module - - if method.name == alias_item["name"] || - method.name.upcase == alias_item["name"].upcase && - @options.ignore_case - - new_meth = initialize_public_method(method, parent) - if alias_item["local_name"] - new_meth.name = alias_item["local_name"] - end - - @stats.add_method new_meth - - alias_item["file_or_module"].add_method new_meth - end - } - end - - ## - # Continuous lines are united. - # - # Comments in continuous lines are removed. - - def united_to_one_line(f90src) - return "" unless f90src - lines = f90src.split("\n") - previous_continuing = false - now_continuing = false - body = "" - lines.each{ |line| - words = line.split("") - next if words.empty? && previous_continuing - commentout = false - brank_flag = true ; brank_char = "" - squote = false ; dquote = false - ignore = false - words.collect! { |char| - if previous_continuing && brank_flag - now_continuing = true - ignore = true - case char - when "!" ; break - when " " ; brank_char << char ; next "" - when "&" - brank_flag = false - now_continuing = false - next "" - else - brank_flag = false - now_continuing = false - ignore = false - next brank_char + char - end - end - ignore = false - - if now_continuing - next "" - elsif !(squote) && !(dquote) && !(commentout) - case char - when "!" ; commentout = true ; next char - when "\""; dquote = true ; next char - when "\'"; squote = true ; next char - when "&" ; now_continuing = true ; next "" - else next char - end - elsif commentout - next char - elsif squote - case char - when "\'"; squote = false ; next char - else next char - end - elsif dquote - case char - when "\""; dquote = false ; next char - else next char - end - end - } - if !ignore && !previous_continuing || !brank_flag - if previous_continuing - body << words.join("") - else - body << "\n" + words.join("") - end - end - previous_continuing = now_continuing ? true : nil - now_continuing = nil - } - return body - end - - - ## - # Continuous line checker - - def continuous_line?(line) - continuous = false - if /&\s*?(!.*)?$/ =~ line - continuous = true - if comment_out?($~.pre_match) - continuous = false - end - end - return continuous - end - - ## - # Comment out checker - - def comment_out?(line) - return nil unless line - commentout = false - squote = false ; dquote = false - line.split("").each { |char| - if !(squote) && !(dquote) - case char - when "!" ; commentout = true ; break - when "\""; dquote = true - when "\'"; squote = true - else next - end - elsif squote - case char - when "\'"; squote = false - else next - end - elsif dquote - case char - when "\""; dquote = false - else next - end - end - } - return commentout - end - - ## - # Semicolons are replaced to line feed. - - def semicolon_to_linefeed(text) - return "" unless text - lines = text.split("\n") - lines.collect!{ |line| - words = line.split("") - commentout = false - squote = false ; dquote = false - words.collect! { |char| - if !(squote) && !(dquote) && !(commentout) - case char - when "!" ; commentout = true ; next char - when "\""; dquote = true ; next char - when "\'"; squote = true ; next char - when ";" ; "\n" - else next char - end - elsif commentout - next char - elsif squote - case char - when "\'"; squote = false ; next char - else next char - end - elsif dquote - case char - when "\""; dquote = false ; next char - else next char - end - end - } - words.join("") - } - return lines.join("\n") - end - - ## - # Which "line" is start of block (module, program, block data, subroutine, - # function) statement ? - - def block_start?(line) - return nil if !line - - if line =~ /^\s*?module\s+(\w+)\s*?(!.*?)?$/i || - line =~ /^\s*?program\s+(\w+)\s*?(!.*?)?$/i || - line =~ /^\s*?block\s+data(\s+\w+)?\s*?(!.*?)?$/i || - line =~ \ - /^\s*? - (recursive|pure|elemental)?\s*? - subroutine\s+(\w+)\s*?(\(.*?\))?\s*?(!.*?)?$ - /ix || - line =~ \ - /^\s*? - (recursive|pure|elemental)?\s*? - ( - character\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | type\s*?\([\w\s]+?\)\s+ - | integer\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | real\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | double\s+precision\s+ - | logical\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - | complex\s*?(\([\w\s\=\(\)\*]+?\))?\s+ - )? - function\s+(\w+)\s*? - (\(.*?\))?(\s+result\((.*?)\))?\s*?(!.*?)?$ - /ix - return true - end - - return nil - end - - ## - # Which "line" is end of block (module, program, block data, subroutine, - # function) statement ? - - def block_end?(line) - return nil if !line - - if line =~ /^\s*?end\s*?(!.*?)?$/i || - line =~ /^\s*?end\s+module(\s+\w+)?\s*?(!.*?)?$/i || - line =~ /^\s*?end\s+program(\s+\w+)?\s*?(!.*?)?$/i || - line =~ /^\s*?end\s+block\s+data(\s+\w+)?\s*?(!.*?)?$/i || - line =~ /^\s*?end\s+subroutine(\s+\w+)?\s*?(!.*?)?$/i || - line =~ /^\s*?end\s+function(\s+\w+)?\s*?(!.*?)?$/i - return true - end - - return nil - end - - ## - # Remove "Alias for" in end of comments - - def remove_trailing_alias(text) - return "" if !text - lines = text.split("\n").reverse - comment_block = Array.new - checked = false - lines.each do |line| - if !checked - if /^\s?#{INTERNAL_ALIAS_MES}/ =~ line || - /^\s?#{EXTERNAL_ALIAS_MES}/ =~ line - checked = true - next - end - end - comment_block.unshift line - end - nice_lines = comment_block.join("\n") - nice_lines ||= "" - return nice_lines - end - - ## - # Empty lines in header are removed - - def remove_empty_head_lines(text) - return "" unless text - lines = text.split("\n") - header = true - lines.delete_if{ |line| - header = false if /\S/ =~ line - header && /^\s*?$/ =~ line - } - lines.join("\n") - end - - ## - # header marker "=", "==", ... are removed - - def remove_header_marker(text) - return text.gsub(/^\s?(=+)/, '<tt></tt>\1') - end - - def remove_private_comments(body) - body.gsub!(/^\s*!--\s*?$.*?^\s*!\+\+\s*?$/m, '') - return body - end - - ## - # Information of arguments of subroutines and functions in Fortran95 - - class Fortran95Definition - - # Name of variable - # - attr_reader :varname - - # Types of variable - # - attr_reader :types - - # Initial Value - # - attr_reader :inivalue - - # Suffix of array - # - attr_reader :arraysuffix - - # Comments - # - attr_accessor :comment - - # Flag of non documentation - # - attr_accessor :nodoc - - def initialize(varname, types, inivalue, arraysuffix, comment, - nodoc=false) - @varname = varname - @types = types - @inivalue = inivalue - @arraysuffix = arraysuffix - @comment = comment - @nodoc = nodoc - end - - def to_s - return <<-EOF -<Fortran95Definition: -varname=#{@varname}, types=#{types}, -inivalue=#{@inivalue}, arraysuffix=#{@arraysuffix}, nodoc=#{@nodoc}, -comment= -#{@comment} -> -EOF - end - - # - # If attr is included, true is returned - # - def include_attr?(attr) - return if !attr - @types.split(",").each{ |type| - return true if type.strip.chomp.upcase == attr.strip.chomp.upcase - } - return nil - end - - end # End of Fortran95Definition - - ## - # Parse string argument "text", and Return Array of Fortran95Definition - # object - - def definition_info(text) - return nil unless text - lines = "#{text}" - defs = Array.new - comment = "" - trailing_comment = "" - under_comment_valid = false - lines.split("\n").each{ |line| - if /^\s*?!\s?(.*)/ =~ line - if COMMENTS_ARE_UPPER - comment << remove_header_marker($1) - comment << "\n" - elsif defs[-1] && under_comment_valid - defs[-1].comment << "\n" - defs[-1].comment << remove_header_marker($1) - end - next - elsif /^\s*?$/ =~ line - comment = "" - under_comment_valid = false - next - end - type = "" - characters = "" - if line =~ /^\s*? - ( - character\s*?(\([\w\s\=\(\)\*]+?\))?[\s\,]* - | type\s*?\([\w\s]+?\)[\s\,]* - | integer\s*?(\([\w\s\=\(\)\*]+?\))?[\s\,]* - | real\s*?(\([\w\s\=\(\)\*]+?\))?[\s\,]* - | double\s+precision[\s\,]* - | logical\s*?(\([\w\s\=\(\)\*]+?\))?[\s\,]* - | complex\s*?(\([\w\s\=\(\)\*]+?\))?[\s\,]* - ) - (.*?::)? - (.+)$ - /ix - characters = $8 - type = $1 - type << $7.gsub(/::/, '').gsub(/^\s*?\,/, '') if $7 - else - under_comment_valid = false - next - end - squote = false ; dquote = false ; bracket = 0 - iniflag = false; commentflag = false - varname = "" ; arraysuffix = "" ; inivalue = "" - start_pos = defs.size - characters.split("").each { |char| - if !(squote) && !(dquote) && bracket <= 0 && !(iniflag) && !(commentflag) - case char - when "!" ; commentflag = true - when "(" ; bracket += 1 ; arraysuffix = char - when "\""; dquote = true - when "\'"; squote = true - when "=" ; iniflag = true ; inivalue << char - when "," - defs << Fortran95Definition.new(varname, type, inivalue, arraysuffix, comment) - varname = "" ; arraysuffix = "" ; inivalue = "" - under_comment_valid = true - when " " ; next - else ; varname << char - end - elsif commentflag - comment << remove_header_marker(char) - trailing_comment << remove_header_marker(char) - elsif iniflag - if dquote - case char - when "\"" ; dquote = false ; inivalue << char - else ; inivalue << char - end - elsif squote - case char - when "\'" ; squote = false ; inivalue << char - else ; inivalue << char - end - elsif bracket > 0 - case char - when "(" ; bracket += 1 ; inivalue << char - when ")" ; bracket -= 1 ; inivalue << char - else ; inivalue << char - end - else - case char - when "," - defs << Fortran95Definition.new(varname, type, inivalue, arraysuffix, comment) - varname = "" ; arraysuffix = "" ; inivalue = "" - iniflag = false - under_comment_valid = true - when "(" ; bracket += 1 ; inivalue << char - when "\""; dquote = true ; inivalue << char - when "\'"; squote = true ; inivalue << char - when "!" ; commentflag = true - else ; inivalue << char - end - end - elsif !(squote) && !(dquote) && bracket > 0 - case char - when "(" ; bracket += 1 ; arraysuffix << char - when ")" ; bracket -= 1 ; arraysuffix << char - else ; arraysuffix << char - end - elsif squote - case char - when "\'"; squote = false ; inivalue << char - else ; inivalue << char - end - elsif dquote - case char - when "\""; dquote = false ; inivalue << char - else ; inivalue << char - end - end - } - defs << Fortran95Definition.new(varname, type, inivalue, arraysuffix, comment) - if trailing_comment =~ /^:nodoc:/ - defs[start_pos..-1].collect!{ |defitem| - defitem.nodoc = true - } - end - varname = "" ; arraysuffix = "" ; inivalue = "" - comment = "" - under_comment_valid = true - trailing_comment = "" - } - return defs - end - -end - diff --git a/lib/rdoc/parser/perl.rb b/lib/rdoc/parser/perl.rb index 0023a013a6..43d1e9ff69 100644 --- a/lib/rdoc/parser/perl.rb +++ b/lib/rdoc/parser/perl.rb @@ -15,7 +15,7 @@ require 'rdoc/parser' # # We would like to support all the markup the POD provides # so that it will convert happily to HTML. At the moment -# I don't think I can do that: time constraints. +# I don't think I can do that: time constraints. # class RDoc::Parser::PerlPOD < RDoc::Parser @@ -45,39 +45,39 @@ class RDoc::Parser::PerlPOD < RDoc::Parser # but I think it would obscure the intent, scatter the # code all over tha place. This machine is necessary # because POD requires that directives be preceded by - # blank lines, so reading line by line is necessary, + # blank lines, so reading line by line is necessary, # and preserving state about what is seen is necesary. def scan @top_level.comment ||= "" - state=:code_blank + state=:code_blank line_number = 0 line = nil # This started out as a really long nested case statement, # which also led to repetitive code. I'd like to avoid that # so I'm using a "table" instead. - + # Firstly we need some procs to do the transition and processing # work. Because these are procs they are closures, and they can # use variables in the local scope. # # First, the "nothing to see here" stuff. - code_noop = lambda do + code_noop = lambda do if line =~ /^\s+$/ state = :code_blank end end - pod_noop = lambda do + pod_noop = lambda do if line =~ /^\s+$/ state = :pod_blank end @top_level.comment += filter(line) end - begin_noop = lambda do + begin_noop = lambda do if line =~ /^\s+$/ state = :begin_blank end @@ -151,7 +151,7 @@ class RDoc::Parser::PerlPOD < RDoc::Parser def filter(comment) return '' if comment =~ /^=pod\s*$/ comment.gsub!(/^=pod/, '==') - comment.gsub!(/^=head(\d+)/) do + comment.gsub!(/^=head(\d+)/) do "=" * $1.to_i end comment.gsub!(/=item/, ''); diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 1697ab85aa..f78dcf07d6 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -7,1339 +7,16 @@ # by Keiju ISHITSUKA (Nippon Rational Inc.) # -require 'e2mmap' -require 'irb/slex' +require 'rdoc/ruby_token' +require 'rdoc/ruby_lex' require 'rdoc/code_objects' require 'rdoc/tokenstream' require 'rdoc/markup/preprocess' require 'rdoc/parser' +require 'rdoc/parser/ruby_tools' $TOKEN_DEBUG ||= nil -#$TOKEN_DEBUG = $DEBUG_RDOC - -## -# Definitions of all tokens involved in the lexical analysis - -module RDoc::RubyToken - - EXPR_BEG = :EXPR_BEG - EXPR_MID = :EXPR_MID - EXPR_END = :EXPR_END - EXPR_ARG = :EXPR_ARG - EXPR_FNAME = :EXPR_FNAME - EXPR_DOT = :EXPR_DOT - EXPR_CLASS = :EXPR_CLASS - - class Token - NO_TEXT = "??".freeze - - attr_accessor :text - attr_reader :line_no - attr_reader :char_no - - def initialize(line_no, char_no) - @line_no = line_no - @char_no = char_no - @text = NO_TEXT - end - - def ==(other) - self.class == other.class and - other.line_no == @line_no and - other.char_no == @char_no and - other.text == @text - end - - ## - # Because we're used in contexts that expect to return a token, we set the - # text string and then return ourselves - - def set_text(text) - @text = text - self - end - - end - - class TkNode < Token - attr :node - end - - class TkId < Token - def initialize(line_no, char_no, name) - super(line_no, char_no) - @name = name - end - attr :name - end - - class TkKW < TkId - end - - class TkVal < Token - def initialize(line_no, char_no, value = nil) - super(line_no, char_no) - set_text(value) - end - end - - class TkOp < Token - def name - self.class.op_name - end - end - - class TkOPASGN < TkOp - def initialize(line_no, char_no, op) - super(line_no, char_no) - op = TkReading2Token[op] unless Symbol === op - @op = op - end - attr :op - end - - class TkUnknownChar < Token - def initialize(line_no, char_no, id) - super(line_no, char_no) - @name = char_no.chr - end - attr :name - end - - class TkError < Token - end - - def set_token_position(line, char) - @prev_line_no = line - @prev_char_no = char - end - - def Token(token, value = nil) - tk = nil - case token - when String, Symbol - source = String === token ? TkReading2Token : TkSymbol2Token - raise TkReading2TokenNoKey, token if (tk = source[token]).nil? - tk = Token(tk[0], value) - else - tk = if (token.ancestors & [TkId, TkVal, TkOPASGN, TkUnknownChar]).empty? - token.new(@prev_line_no, @prev_char_no) - else - token.new(@prev_line_no, @prev_char_no, value) - end - end - tk - end - - TokenDefinitions = [ - [:TkCLASS, TkKW, "class", EXPR_CLASS], - [:TkMODULE, TkKW, "module", EXPR_CLASS], - [:TkDEF, TkKW, "def", EXPR_FNAME], - [:TkUNDEF, TkKW, "undef", EXPR_FNAME], - [:TkBEGIN, TkKW, "begin", EXPR_BEG], - [:TkRESCUE, TkKW, "rescue", EXPR_MID], - [:TkENSURE, TkKW, "ensure", EXPR_BEG], - [:TkEND, TkKW, "end", EXPR_END], - [:TkIF, TkKW, "if", EXPR_BEG, :TkIF_MOD], - [:TkUNLESS, TkKW, "unless", EXPR_BEG, :TkUNLESS_MOD], - [:TkTHEN, TkKW, "then", EXPR_BEG], - [:TkELSIF, TkKW, "elsif", EXPR_BEG], - [:TkELSE, TkKW, "else", EXPR_BEG], - [:TkCASE, TkKW, "case", EXPR_BEG], - [:TkWHEN, TkKW, "when", EXPR_BEG], - [:TkWHILE, TkKW, "while", EXPR_BEG, :TkWHILE_MOD], - [:TkUNTIL, TkKW, "until", EXPR_BEG, :TkUNTIL_MOD], - [:TkFOR, TkKW, "for", EXPR_BEG], - [:TkBREAK, TkKW, "break", EXPR_END], - [:TkNEXT, TkKW, "next", EXPR_END], - [:TkREDO, TkKW, "redo", EXPR_END], - [:TkRETRY, TkKW, "retry", EXPR_END], - [:TkIN, TkKW, "in", EXPR_BEG], - [:TkDO, TkKW, "do", EXPR_BEG], - [:TkRETURN, TkKW, "return", EXPR_MID], - [:TkYIELD, TkKW, "yield", EXPR_END], - [:TkSUPER, TkKW, "super", EXPR_END], - [:TkSELF, TkKW, "self", EXPR_END], - [:TkNIL, TkKW, "nil", EXPR_END], - [:TkTRUE, TkKW, "true", EXPR_END], - [:TkFALSE, TkKW, "false", EXPR_END], - [:TkAND, TkKW, "and", EXPR_BEG], - [:TkOR, TkKW, "or", EXPR_BEG], - [:TkNOT, TkKW, "not", EXPR_BEG], - [:TkIF_MOD, TkKW], - [:TkUNLESS_MOD, TkKW], - [:TkWHILE_MOD, TkKW], - [:TkUNTIL_MOD, TkKW], - [:TkALIAS, TkKW, "alias", EXPR_FNAME], - [:TkDEFINED, TkKW, "defined?", EXPR_END], - [:TklBEGIN, TkKW, "BEGIN", EXPR_END], - [:TklEND, TkKW, "END", EXPR_END], - [:Tk__LINE__, TkKW, "__LINE__", EXPR_END], - [:Tk__FILE__, TkKW, "__FILE__", EXPR_END], - - [:TkIDENTIFIER, TkId], - [:TkFID, TkId], - [:TkGVAR, TkId], - [:TkIVAR, TkId], - [:TkCONSTANT, TkId], - - [:TkINTEGER, TkVal], - [:TkFLOAT, TkVal], - [:TkSTRING, TkVal], - [:TkXSTRING, TkVal], - [:TkREGEXP, TkVal], - [:TkCOMMENT, TkVal], - - [:TkDSTRING, TkNode], - [:TkDXSTRING, TkNode], - [:TkDREGEXP, TkNode], - [:TkNTH_REF, TkId], - [:TkBACK_REF, TkId], - - [:TkUPLUS, TkOp, "+@"], - [:TkUMINUS, TkOp, "-@"], - [:TkPOW, TkOp, "**"], - [:TkCMP, TkOp, "<=>"], - [:TkEQ, TkOp, "=="], - [:TkEQQ, TkOp, "==="], - [:TkNEQ, TkOp, "!="], - [:TkGEQ, TkOp, ">="], - [:TkLEQ, TkOp, "<="], - [:TkANDOP, TkOp, "&&"], - [:TkOROP, TkOp, "||"], - [:TkMATCH, TkOp, "=~"], - [:TkNMATCH, TkOp, "!~"], - [:TkDOT2, TkOp, ".."], - [:TkDOT3, TkOp, "..."], - [:TkAREF, TkOp, "[]"], - [:TkASET, TkOp, "[]="], - [:TkLSHFT, TkOp, "<<"], - [:TkRSHFT, TkOp, ">>"], - [:TkCOLON2, TkOp], - [:TkCOLON3, TkOp], -# [:OPASGN, TkOp], # +=, -= etc. # - [:TkASSOC, TkOp, "=>"], - [:TkQUESTION, TkOp, "?"], #? - [:TkCOLON, TkOp, ":"], #: - - [:TkfLPAREN], # func( # - [:TkfLBRACK], # func[ # - [:TkfLBRACE], # func{ # - [:TkSTAR], # *arg - [:TkAMPER], # &arg # - [:TkSYMBOL, TkId], # :SYMBOL - [:TkSYMBEG, TkId], - [:TkGT, TkOp, ">"], - [:TkLT, TkOp, "<"], - [:TkPLUS, TkOp, "+"], - [:TkMINUS, TkOp, "-"], - [:TkMULT, TkOp, "*"], - [:TkDIV, TkOp, "/"], - [:TkMOD, TkOp, "%"], - [:TkBITOR, TkOp, "|"], - [:TkBITXOR, TkOp, "^"], - [:TkBITAND, TkOp, "&"], - [:TkBITNOT, TkOp, "~"], - [:TkNOTOP, TkOp, "!"], - - [:TkBACKQUOTE, TkOp, "`"], - - [:TkASSIGN, Token, "="], - [:TkDOT, Token, "."], - [:TkLPAREN, Token, "("], #(exp) - [:TkLBRACK, Token, "["], #[arry] - [:TkLBRACE, Token, "{"], #{hash} - [:TkRPAREN, Token, ")"], - [:TkRBRACK, Token, "]"], - [:TkRBRACE, Token, "}"], - [:TkCOMMA, Token, ","], - [:TkSEMICOLON, Token, ";"], - - [:TkRD_COMMENT], - [:TkSPACE], - [:TkNL], - [:TkEND_OF_SCRIPT], - - [:TkBACKSLASH, TkUnknownChar, "\\"], - [:TkAT, TkUnknownChar, "@"], - [:TkDOLLAR, TkUnknownChar, "\$"], #" - ] - - # {reading => token_class} - # {reading => [token_class, *opt]} - TkReading2Token = {} - TkSymbol2Token = {} - - def self.def_token(token_n, super_token = Token, reading = nil, *opts) - token_n = token_n.id2name unless String === token_n - - fail AlreadyDefinedToken, token_n if const_defined?(token_n) - - token_c = Class.new super_token - const_set token_n, token_c -# token_c.inspect - - if reading - if TkReading2Token[reading] - fail TkReading2TokenDuplicateError, token_n, reading - end - if opts.empty? - TkReading2Token[reading] = [token_c] - else - TkReading2Token[reading] = [token_c].concat(opts) - end - end - TkSymbol2Token[token_n.intern] = token_c - - if token_c <= TkOp - token_c.class_eval %{ - def self.op_name; "#{reading}"; end - } - end - end - - for defs in TokenDefinitions - def_token(*defs) - end - - NEWLINE_TOKEN = TkNL.new(0,0) - NEWLINE_TOKEN.set_text("\n") - -end - -## -# Lexical analyzer for Ruby source - -class RDoc::RubyLex - - ## - # Read an input stream character by character. We allow for unlimited - # ungetting of characters just read. - # - # We simplify the implementation greatly by reading the entire input - # into a buffer initially, and then simply traversing it using - # pointers. - # - # We also have to allow for the <i>here document diversion</i>. This - # little gem comes about when the lexer encounters a here - # document. At this point we effectively need to split the input - # stream into two parts: one to read the body of the here document, - # the other to read the rest of the input line where the here - # document was initially encountered. For example, we might have - # - # do_something(<<-A, <<-B) - # stuff - # for - # A - # stuff - # for - # B - # - # When the lexer encounters the <<A, it reads until the end of the - # line, and keeps it around for later. It then reads the body of the - # here document. Once complete, it needs to read the rest of the - # original line, but then skip the here document body. - # - - class BufferedReader - - attr_reader :line_num - - def initialize(content, options) - @options = options - - if /\t/ =~ content - tab_width = @options.tab_width - content = content.split(/\n/).map do |line| - 1 while line.gsub!(/\t+/) { ' ' * (tab_width*$&.length - $`.length % tab_width)} && $~ #` - line - end .join("\n") - end - @content = content - @content << "\n" unless @content[-1,1] == "\n" - @size = @content.size - @offset = 0 - @hwm = 0 - @line_num = 1 - @read_back_offset = 0 - @last_newline = 0 - @newline_pending = false - end - - def column - @offset - @last_newline - end - - def getc - return nil if @offset >= @size - ch = @content[@offset, 1] - - @offset += 1 - @hwm = @offset if @hwm < @offset - - if @newline_pending - @line_num += 1 - @last_newline = @offset - 1 - @newline_pending = false - end - - if ch == "\n" - @newline_pending = true - end - ch - end - - def getc_already_read - getc - end - - def ungetc(ch) - raise "unget past beginning of file" if @offset <= 0 - @offset -= 1 - if @content[@offset] == ?\n - @newline_pending = false - end - end - - def get_read - res = @content[@read_back_offset...@offset] - @read_back_offset = @offset - res - end - - def peek(at) - pos = @offset + at - if pos >= @size - nil - else - @content[pos, 1] - end - end - - def peek_equal(str) - @content[@offset, str.length] == str - end - - def divert_read_from(reserve) - @content[@offset, 0] = reserve - @size = @content.size - end - end - - # end of nested class BufferedReader - - extend Exception2MessageMapper - def_exception(:AlreadyDefinedToken, "Already defined token(%s)") - def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')") - def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')") - def_exception(:TkReading2TokenDuplicateError, - "key duplicate(token_n='%s', key='%s')") - def_exception(:SyntaxError, "%s") - - include RDoc::RubyToken - include IRB - - attr_reader :continue - attr_reader :lex_state - - def self.debug? - false - end - - def initialize(content, options) - lex_init - - @options = options - - @reader = BufferedReader.new content, @options - - @exp_line_no = @line_no = 1 - @base_char_no = 0 - @indent = 0 - - @ltype = nil - @quoted = nil - @lex_state = EXPR_BEG - @space_seen = false - - @continue = false - @line = "" - - @skip_space = false - @read_auto_clean_up = false - @exception_on_syntax_error = true - end - - attr_accessor :skip_space - attr_accessor :read_auto_clean_up - attr_accessor :exception_on_syntax_error - attr_reader :indent - - # io functions - def line_no - @reader.line_num - end - - def char_no - @reader.column - end - - def get_read - @reader.get_read - end - - def getc - @reader.getc - end - - def getc_of_rests - @reader.getc_already_read - end - - def gets - c = getc or return - l = "" - begin - l.concat c unless c == "\r" - break if c == "\n" - end while c = getc - l - end - - - def ungetc(c = nil) - @reader.ungetc(c) - end - - def peek_equal?(str) - @reader.peek_equal(str) - end - - def peek(i = 0) - @reader.peek(i) - end - - def lex - until (TkNL === (tk = token) or TkEND_OF_SCRIPT === tk) and - not @continue or tk.nil? - end - - line = get_read - - if line == "" and TkEND_OF_SCRIPT === tk or tk.nil? then - nil - else - line - end - end - - def token - set_token_position(line_no, char_no) - begin - begin - tk = @OP.match(self) - @space_seen = TkSPACE === tk - rescue SyntaxError => e - raise RDoc::Error, "syntax error: #{e.message}" if - @exception_on_syntax_error - - tk = TkError.new(line_no, char_no) - end - end while @skip_space and TkSPACE === tk - if @read_auto_clean_up - get_read - end -# throw :eof unless tk - tk - end - - ENINDENT_CLAUSE = [ - "case", "class", "def", "do", "for", "if", - "module", "unless", "until", "while", "begin" #, "when" - ] - DEINDENT_CLAUSE = ["end" #, "when" - ] - - PERCENT_LTYPE = { - "q" => "\'", - "Q" => "\"", - "x" => "\`", - "r" => "/", - "w" => "]" - } - - PERCENT_PAREN = { - "{" => "}", - "[" => "]", - "<" => ">", - "(" => ")" - } - - Ltype2Token = { - "\'" => TkSTRING, - "\"" => TkSTRING, - "\`" => TkXSTRING, - "/" => TkREGEXP, - "]" => TkDSTRING - } - Ltype2Token.default = TkSTRING - - DLtype2Token = { - "\"" => TkDSTRING, - "\`" => TkDXSTRING, - "/" => TkDREGEXP, - } - - def lex_init() - @OP = IRB::SLex.new - @OP.def_rules("\0", "\004", "\032") do |chars, io| - Token(TkEND_OF_SCRIPT).set_text(chars) - end - - @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |chars, io| - @space_seen = TRUE - while (ch = getc) =~ /[ \t\f\r\13]/ - chars << ch - end - ungetc - Token(TkSPACE).set_text(chars) - end - - @OP.def_rule("#") do - |op, io| - identify_comment - end - - @OP.def_rule("=begin", proc{@prev_char_no == 0 && peek(0) =~ /\s/}) do - |op, io| - str = op - @ltype = "=" - - - begin - line = "" - begin - ch = getc - line << ch - end until ch == "\n" - str << line - end until line =~ /^=end/ - - ungetc - - @ltype = nil - - if str =~ /\A=begin\s+rdoc/i - str.sub!(/\A=begin.*\n/, '') - str.sub!(/^=end.*/m, '') - Token(TkCOMMENT).set_text(str) - else - Token(TkRD_COMMENT)#.set_text(str) - end - end - - @OP.def_rule("\n") do - print "\\n\n" if RDoc::RubyLex.debug? - case @lex_state - when EXPR_BEG, EXPR_FNAME, EXPR_DOT - @continue = TRUE - else - @continue = FALSE - @lex_state = EXPR_BEG - end - Token(TkNL).set_text("\n") - end - - @OP.def_rules("*", "**", - "!", "!=", "!~", - "=", "==", "===", - "=~", "<=>", - "<", "<=", - ">", ">=", ">>") do - |op, io| - @lex_state = EXPR_BEG - Token(op).set_text(op) - end - - @OP.def_rules("<<") do - |op, io| - tk = nil - if @lex_state != EXPR_END && @lex_state != EXPR_CLASS && - (@lex_state != EXPR_ARG || @space_seen) - c = peek(0) - if /[-\w_\"\'\`]/ =~ c - tk = identify_here_document - end - end - if !tk - @lex_state = EXPR_BEG - tk = Token(op).set_text(op) - end - tk - end - - @OP.def_rules("'", '"') do - |op, io| - identify_string(op) - end - - @OP.def_rules("`") do - |op, io| - if @lex_state == EXPR_FNAME - Token(op).set_text(op) - else - identify_string(op) - end - end - - @OP.def_rules('?') do - |op, io| - if @lex_state == EXPR_END - @lex_state = EXPR_BEG - Token(TkQUESTION).set_text(op) - else - ch = getc - if @lex_state == EXPR_ARG && ch !~ /\s/ - ungetc - @lex_state = EXPR_BEG - Token(TkQUESTION).set_text(op) - else - str = op - str << ch - if (ch == '\\') #' - str << read_escape - end - @lex_state = EXPR_END - Token(TkINTEGER).set_text(str) - end - end - end - - @OP.def_rules("&", "&&", "|", "||") do - |op, io| - @lex_state = EXPR_BEG - Token(op).set_text(op) - end - - @OP.def_rules("+=", "-=", "*=", "**=", - "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do - |op, io| - @lex_state = EXPR_BEG - op =~ /^(.*)=$/ - Token(TkOPASGN, $1).set_text(op) - end - - @OP.def_rule("+@", proc{@lex_state == EXPR_FNAME}) do |op, io| - Token(TkUPLUS).set_text(op) - end - - @OP.def_rule("-@", proc{@lex_state == EXPR_FNAME}) do |op, io| - Token(TkUMINUS).set_text(op) - end - - @OP.def_rules("+", "-") do - |op, io| - catch(:RET) do - if @lex_state == EXPR_ARG - if @space_seen and peek(0) =~ /[0-9]/ - throw :RET, identify_number(op) - else - @lex_state = EXPR_BEG - end - elsif @lex_state != EXPR_END and peek(0) =~ /[0-9]/ - throw :RET, identify_number(op) - else - @lex_state = EXPR_BEG - end - Token(op).set_text(op) - end - end - - @OP.def_rule(".") do - @lex_state = EXPR_BEG - if peek(0) =~ /[0-9]/ - ungetc - identify_number("") - else - # for obj.if - @lex_state = EXPR_DOT - Token(TkDOT).set_text(".") - end - end - - @OP.def_rules("..", "...") do - |op, io| - @lex_state = EXPR_BEG - Token(op).set_text(op) - end - - lex_int2 - end - - def lex_int2 - @OP.def_rules("]", "}", ")") do - |op, io| - @lex_state = EXPR_END - @indent -= 1 - Token(op).set_text(op) - end - - @OP.def_rule(":") do - if @lex_state == EXPR_END || peek(0) =~ /\s/ - @lex_state = EXPR_BEG - tk = Token(TkCOLON) - else - @lex_state = EXPR_FNAME - tk = Token(TkSYMBEG) - end - tk.set_text(":") - end - - @OP.def_rule("::") do - if @lex_state == EXPR_BEG or @lex_state == EXPR_ARG && @space_seen - @lex_state = EXPR_BEG - tk = Token(TkCOLON3) - else - @lex_state = EXPR_DOT - tk = Token(TkCOLON2) - end - tk.set_text("::") - end - - @OP.def_rule("/") do - |op, io| - if @lex_state == EXPR_BEG || @lex_state == EXPR_MID - identify_string(op) - elsif peek(0) == '=' - getc - @lex_state = EXPR_BEG - Token(TkOPASGN, :/).set_text("/=") #") - elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/ - identify_string(op) - else - @lex_state = EXPR_BEG - Token("/").set_text(op) - end - end - - @OP.def_rules("^") do - @lex_state = EXPR_BEG - Token("^").set_text("^") - end - - @OP.def_rules(",", ";") do - |op, io| - @lex_state = EXPR_BEG - Token(op).set_text(op) - end - - @OP.def_rule("~") do - @lex_state = EXPR_BEG - Token("~").set_text("~") - end - - @OP.def_rule("~@", proc{@lex_state = EXPR_FNAME}) do - @lex_state = EXPR_BEG - Token("~").set_text("~@") - end - - @OP.def_rule("(") do - @indent += 1 - if @lex_state == EXPR_BEG || @lex_state == EXPR_MID - @lex_state = EXPR_BEG - tk = Token(TkfLPAREN) - else - @lex_state = EXPR_BEG - tk = Token(TkLPAREN) - end - tk.set_text("(") - end - - @OP.def_rule("[]", proc{@lex_state == EXPR_FNAME}) do - Token("[]").set_text("[]") - end - - @OP.def_rule("[]=", proc{@lex_state == EXPR_FNAME}) do - Token("[]=").set_text("[]=") - end - - @OP.def_rule("[") do - @indent += 1 - if @lex_state == EXPR_FNAME - t = Token(TkfLBRACK) - else - if @lex_state == EXPR_BEG || @lex_state == EXPR_MID - t = Token(TkLBRACK) - elsif @lex_state == EXPR_ARG && @space_seen - t = Token(TkLBRACK) - else - t = Token(TkfLBRACK) - end - @lex_state = EXPR_BEG - end - t.set_text("[") - end - - @OP.def_rule("{") do - @indent += 1 - if @lex_state != EXPR_END && @lex_state != EXPR_ARG - t = Token(TkLBRACE) - else - t = Token(TkfLBRACE) - end - @lex_state = EXPR_BEG - t.set_text("{") - end - - @OP.def_rule('\\') do #' - if getc == "\n" - @space_seen = true - @continue = true - Token(TkSPACE).set_text("\\\n") - else - ungetc - Token("\\").set_text("\\") #" - end - end - - @OP.def_rule('%') do - |op, io| - if @lex_state == EXPR_BEG || @lex_state == EXPR_MID - identify_quotation('%') - elsif peek(0) == '=' - getc - Token(TkOPASGN, "%").set_text("%=") - elsif @lex_state == EXPR_ARG and @space_seen and peek(0) !~ /\s/ - identify_quotation('%') - else - @lex_state = EXPR_BEG - Token("%").set_text("%") - end - end - - @OP.def_rule('$') do #' - identify_gvar - end - - @OP.def_rule('@') do - if peek(0) =~ /[@\w_]/ - ungetc - identify_identifier - else - Token("@").set_text("@") - end - end - - @OP.def_rule("__END__", proc{@prev_char_no == 0 && peek(0) =~ /[\r\n]/}) do - throw :eof - end - - @OP.def_rule("") do - |op, io| - printf "MATCH: start %s: %s\n", op, io.inspect if RDoc::RubyLex.debug? - if peek(0) =~ /[0-9]/ - t = identify_number("") - elsif peek(0) =~ /[\w_]/ - t = identify_identifier - end - printf "MATCH: end %s: %s\n", op, io.inspect if RDoc::RubyLex.debug? - t - end - end - - def identify_gvar - @lex_state = EXPR_END - str = "$" - - tk = case ch = getc - when /[~_*$?!@\/\\;,=:<>".]/ #" - str << ch - Token(TkGVAR, str) - - when "-" - str << "-" << getc - Token(TkGVAR, str) - - when "&", "`", "'", "+" - str << ch - Token(TkBACK_REF, str) - - when /[1-9]/ - str << ch - while (ch = getc) =~ /[0-9]/ - str << ch - end - ungetc - Token(TkNTH_REF) - when /\w/ - ungetc - ungetc - return identify_identifier - else - ungetc - Token("$") - end - tk.set_text(str) - end - - def identify_identifier - token = "" - token.concat getc if peek(0) =~ /[$@]/ - token.concat getc if peek(0) == "@" - - while (ch = getc) =~ /\w|_/ - print ":", ch, ":" if RDoc::RubyLex.debug? - token.concat ch - end - ungetc - - if ch == "!" or ch == "?" - token.concat getc - end - # fix token - - # $stderr.puts "identifier - #{token}, state = #@lex_state" - - case token - when /^\$/ - return Token(TkGVAR, token).set_text(token) - when /^\@/ - @lex_state = EXPR_END - return Token(TkIVAR, token).set_text(token) - end - - if @lex_state != EXPR_DOT - print token, "\n" if RDoc::RubyLex.debug? - - token_c, *trans = TkReading2Token[token] - if token_c - # reserved word? - - if (@lex_state != EXPR_BEG && - @lex_state != EXPR_FNAME && - trans[1]) - # modifiers - token_c = TkSymbol2Token[trans[1]] - @lex_state = trans[0] - else - if @lex_state != EXPR_FNAME - if ENINDENT_CLAUSE.include?(token) - @indent += 1 - elsif DEINDENT_CLAUSE.include?(token) - @indent -= 1 - end - @lex_state = trans[0] - else - @lex_state = EXPR_END - end - end - return Token(token_c, token).set_text(token) - end - end - - if @lex_state == EXPR_FNAME - @lex_state = EXPR_END - if peek(0) == '=' - token.concat getc - end - elsif @lex_state == EXPR_BEG || @lex_state == EXPR_DOT - @lex_state = EXPR_ARG - else - @lex_state = EXPR_END - end - - if token[0, 1] =~ /[A-Z]/ - return Token(TkCONSTANT, token).set_text(token) - elsif token[token.size - 1, 1] =~ /[!?]/ - return Token(TkFID, token).set_text(token) - else - return Token(TkIDENTIFIER, token).set_text(token) - end - end - - def identify_here_document - ch = getc - if ch == "-" - ch = getc - indent = true - end - if /['"`]/ =~ ch # ' - lt = ch - quoted = "" - while (c = getc) && c != lt - quoted.concat c - end - else - lt = '"' - quoted = ch.dup - while (c = getc) && c =~ /\w/ - quoted.concat c - end - ungetc - end - - ltback, @ltype = @ltype, lt - reserve = "" - - while ch = getc - reserve << ch - if ch == "\\" #" - ch = getc - reserve << ch - elsif ch == "\n" - break - end - end - - str = "" - while (l = gets) - l.chomp! - l.strip! if indent - break if l == quoted - str << l.chomp << "\n" - end - - @reader.divert_read_from(reserve) - - @ltype = ltback - @lex_state = EXPR_END - Token(Ltype2Token[lt], str).set_text(str.dump) - end - - def identify_quotation(initial_char) - ch = getc - if lt = PERCENT_LTYPE[ch] - initial_char += ch - ch = getc - elsif ch =~ /\W/ - lt = "\"" - else - fail SyntaxError, "unknown type of %string ('#{ch}')" - end -# if ch !~ /\W/ -# ungetc -# next -# end - #@ltype = lt - @quoted = ch unless @quoted = PERCENT_PAREN[ch] - identify_string(lt, @quoted, ch, initial_char) - end - - def identify_number(start) - str = start.dup - - if start == "+" or start == "-" or start == "" - start = getc - str << start - end - - @lex_state = EXPR_END - - if start == "0" - if peek(0) == "x" - ch = getc - str << ch - match = /[0-9a-f_]/ - else - match = /[0-7_]/ - end - while ch = getc - if ch !~ match - ungetc - break - else - str << ch - end - end - return Token(TkINTEGER).set_text(str) - end - - type = TkINTEGER - allow_point = TRUE - allow_e = TRUE - while ch = getc - case ch - when /[0-9_]/ - str << ch - - when allow_point && "." - type = TkFLOAT - if peek(0) !~ /[0-9]/ - ungetc - break - end - str << ch - allow_point = false - - when allow_e && "e", allow_e && "E" - str << ch - type = TkFLOAT - if peek(0) =~ /[+-]/ - str << getc - end - allow_e = false - allow_point = false - else - ungetc - break - end - end - Token(type).set_text(str) - end - - def identify_string(ltype, quoted = ltype, opener=nil, initial_char = nil) - @ltype = ltype - @quoted = quoted - subtype = nil - - str = "" - str << initial_char if initial_char - str << (opener||quoted) - - nest = 0 - begin - while ch = getc - str << ch - if @quoted == ch - if nest == 0 - break - else - nest -= 1 - end - elsif opener == ch - nest += 1 - elsif @ltype != "'" && @ltype != "]" and ch == "#" - ch = getc - if ch == "{" - subtype = true - str << ch << skip_inner_expression - else - ungetc(ch) - end - elsif ch == '\\' #' - str << read_escape - end - end - if @ltype == "/" - if peek(0) =~ /i|o|n|e|s/ - str << getc - end - end - if subtype - Token(DLtype2Token[ltype], str) - else - Token(Ltype2Token[ltype], str) - end.set_text(str) - ensure - @ltype = nil - @quoted = nil - @lex_state = EXPR_END - end - end - - def skip_inner_expression - res = "" - nest = 0 - while (ch = getc) - res << ch - if ch == '}' - break if nest.zero? - nest -= 1 - elsif ch == '{' - nest += 1 - end - end - res - end - - def identify_comment - @ltype = "#" - comment = "#" - while ch = getc - if ch == "\\" - ch = getc - if ch == "\n" - ch = " " - else - comment << "\\" - end - else - if ch == "\n" - @ltype = nil - ungetc - break - end - end - comment << ch - end - return Token(TkCOMMENT).set_text(comment) - end - - def read_escape - res = "" - case ch = getc - when /[0-7]/ - ungetc ch - 3.times do - case ch = getc - when /[0-7]/ - when nil - break - else - ungetc - break - end - res << ch - end - - when "x" - res << ch - 2.times do - case ch = getc - when /[0-9a-fA-F]/ - when nil - break - else - ungetc - break - end - res << ch - end - - when "M" - res << ch - if (ch = getc) != '-' - ungetc - else - res << ch - if (ch = getc) == "\\" #" - res << ch - res << read_escape - else - res << ch - end - end - - when "C", "c" #, "^" - res << ch - if ch == "C" and (ch = getc) != "-" - ungetc - else - res << ch - if (ch = getc) == "\\" #" - res << ch - res << read_escape - else - res << ch - end - end - else - res << ch - end - res - end -end ## # Extracts code elements from a source file returning a TopLevel object @@ -1373,7 +50,7 @@ end # # ## # # This method tries over and over until it is tired -# +# # def go_go_go(thing_to_try, tries = 10) # :args: thing_to_try # puts thing_to_try # go_go_go thing_to_try, tries - 1 @@ -1393,7 +70,7 @@ end # # :call-seq: # # my_method(Range) # # my_method(offset, length) -# +# # def my_method(*args) # end # @@ -1404,7 +81,7 @@ end # # ## # # My method is awesome -# +# # def my_method(&block) # :yields: happy, times # block.call 1, 2 # end @@ -1416,7 +93,7 @@ end # # ## # # This is a meta-programmed method! -# +# # add_my_method :meta_method, :arg1, :arg2 # # The parser looks at the token after the identifier to determine the name, in @@ -1439,18 +116,29 @@ end # ## # # :singleton-method: woo_hoo! # -# == Hidden methods +# Additionally you can mark a method as an attribute by +# using :attr:, :attr_reader:, :attr_writer: or :attr_accessor:. Just like +# for :method:, the name is optional. +# +# ## +# # :attr_reader: my_attr_name +# +# == Hidden methods and attributes # # You can provide documentation for methods that don't appear using -# the :method: and :singleton-method: directives: +# the :method:, :singleton-method: and :attr: directives: # # ## +# # :attr_writer: ghost_writer +# # There is an attribute here, but you can't see it! +# +# ## # # :method: ghost_method # # There is a method here, but you can't see it! -# +# # ## # # this is a comment for a regular method -# +# # def regular_method() end # # Note that by default, the :method: directive will be ignored if there is a @@ -1458,12 +146,20 @@ end class RDoc::Parser::Ruby < RDoc::Parser - parse_files_matching(/\.(?:rbw?|rdoc)\z/) + parse_files_matching(/\.rbw?$/) include RDoc::RubyToken include RDoc::TokenStream + include RDoc::Parser::RubyTools + + ## + # RDoc::NormalClass type NORMAL = "::" + + ## + # RDoc::SingleClass type + SINGLE = "<<" def initialize(top_level, file_name, content, options, stats) @@ -1473,21 +169,17 @@ class RDoc::Parser::Ruby < RDoc::Parser @token_listeners = nil @scanner = RDoc::RubyLex.new content, @options @scanner.exception_on_syntax_error = false + @prev_seek = nil reset end - def add_token_listener(obj) - @token_listeners ||= [] - @token_listeners << obj - end - ## # Look for the first comment in a file that isn't a shebang line. def collect_first_comment skip_tkspace - res = '' + comment = '' first_line = true tk = get_tk @@ -1502,7 +194,7 @@ class RDoc::Parser::Ruby < RDoc::Parser tk = get_tk else first_line = false - res << tk.text << "\n" + comment << tk.text << "\n" tk = get_tk if TkNL === tk then @@ -1514,7 +206,7 @@ class RDoc::Parser::Ruby < RDoc::Parser unget_tk tk - res + comment end def error(msg) @@ -1560,24 +252,24 @@ class RDoc::Parser::Ruby < RDoc::Parser name_t = get_tk # class ::A -> A is in the top level - if TkCOLON2 === name_t then + case name_t + when TkCOLON2, TkCOLON3 then # bug name_t = get_tk container = @top_level end - skip_tkspace(false) + skip_tkspace false while TkCOLON2 === peek_tk do prev_container = container - container = container.find_module_named(name_t.name) - if !container -# warn("Couldn't find module #{name_t.name}") + container = container.find_module_named name_t.name + unless container then container = prev_container.add_module RDoc::NormalModule, name_t.name end get_tk name_t = get_tk end - skip_tkspace(false) + skip_tkspace false return [container, name_t] end @@ -1590,12 +282,12 @@ class RDoc::Parser::Ruby < RDoc::Parser res = "" while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do - res += tk.text + res += tk.name tk = get_tk end unget_tk(tk) - skip_tkspace(false) + skip_tkspace false get_tkread # empty out read buffer @@ -1617,11 +309,11 @@ class RDoc::Parser::Ruby < RDoc::Parser def get_constant res = "" - skip_tkspace(false) + skip_tkspace false tk = get_tk while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do - res += tk.text + res += tk.name tk = get_tk end @@ -1636,88 +328,51 @@ class RDoc::Parser::Ruby < RDoc::Parser # Get a constant that may be surrounded by parens def get_constant_with_optional_parens - skip_tkspace(false) + skip_tkspace false nest = 0 while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do get_tk - skip_tkspace(true) + skip_tkspace nest += 1 end name = get_constant while nest > 0 - skip_tkspace(true) + skip_tkspace tk = get_tk nest -= 1 if TkRPAREN === tk end + name end def get_symbol_or_name tk = get_tk case tk - when TkSYMBOL - tk.text.sub(/^:/, '') - when TkId, TkOp + when TkSYMBOL then + text = tk.text.sub(/^:/, '') + + if TkASSIGN === peek_tk then + get_tk + text << '=' + end + + text + when TkId, TkOp then tk.name - when TkSTRING + when TkSTRING, TkDSTRING then tk.text else - raise "Name or symbol expected (got #{tk})" + raise RDoc::Error, "Name or symbol expected (got #{tk})" end end - def get_tk - tk = nil - if @tokens.empty? - tk = @scanner.token - @read.push @scanner.get_read - puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG - else - @read.push @unget_read.shift - tk = @tokens.shift - puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG - end - - if TkSYMBEG === tk then - set_token_position(tk.line_no, tk.char_no) - tk1 = get_tk - if TkId === tk1 or TkOp === tk1 or TkSTRING === tk1 then - if tk1.respond_to?(:name) - tk = Token(TkSYMBOL).set_text(":" + tk1.name) - else - tk = Token(TkSYMBOL).set_text(":" + tk1.text) - end - # remove the identifier we just read (we're about to - # replace it with a symbol) - @token_listeners.each do |obj| - obj.pop_token - end if @token_listeners - else - warn("':' not followed by identifier or operator") - tk = tk1 - end - end - - # inform any listeners of our shiny new token - @token_listeners.each do |obj| - obj.add_token(tk) - end if @token_listeners - - tk - end - - def get_tkread - read = @read.join("") - @read = [] - read - end - ## # Look for directives in a normal comment block: # - # #-- - don't display comment from this point forward + # # :stopdoc: + # # Don't display comment from this point forward # # This routine modifies it's parameter @@ -1732,8 +387,9 @@ class RDoc::Parser::Ruby < RDoc::Parser when 'main' then @options.main_page = param '' - when 'method', 'singleton-method' then - false # ignore + when 'method', 'singleton-method', + 'attr', 'attr_accessor', 'attr_reader', 'attr_writer' then + false # handled elsewhere when 'section' then context.set_current_section(param, comment) comment.replace '' @@ -1754,23 +410,30 @@ class RDoc::Parser::Ruby < RDoc::Parser end end - remove_private_comments(comment) + remove_private_comments comment end - def make_message(msg) - prefix = "\n" + @file_name + ":" - if @scanner - prefix << "#{@scanner.line_no}:#{@scanner.char_no}: " - end - return prefix + msg + ## + # Adds useful info about the parser to +message+ + + def make_message message + prefix = "\n#{@file_name}:" + + prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner + + "#{prefix} #{message}" end + ## + # Creates an RDoc::Attr for the name following +tk+, setting the comment to + # +comment+. + def parse_attr(context, single, tk, comment) - args = parse_symbol_arg(1) + args = parse_symbol_arg 1 if args.size > 0 name = args[0] rw = "R" - skip_tkspace(false) + skip_tkspace false tk = get_tk if TkCOMMA === tk then rw = "RW" if get_bool @@ -1787,12 +450,16 @@ class RDoc::Parser::Ruby < RDoc::Parser end end + ## + # Creates an RDoc::Attr for each attribute listed after +tk+, setting the + # comment for each to +comment+. + def parse_attr_accessor(context, single, tk, comment) args = parse_symbol_arg read = get_tkread rw = "?" - # If nodoc is given, don't document any of them + # TODO If nodoc is given, don't document any of them tmp = RDoc::CodeObject.new read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS @@ -1803,8 +470,7 @@ class RDoc::Parser::Ruby < RDoc::Parser when "attr_writer" then rw = "W" when "attr_accessor" then rw = "RW" else - rw = @options.extra_accessor_flags[tk.name] - rw = '?' if rw.nil? + rw = '?' end for name in args @@ -1820,19 +486,24 @@ class RDoc::Parser::Ruby < RDoc::Parser skip_tkspace end new_name = get_symbol_or_name - @scanner.instance_eval{@lex_state = EXPR_FNAME} + + @scanner.instance_eval { @lex_state = EXPR_FNAME } + skip_tkspace if TkCOMMA === peek_tk then get_tk skip_tkspace end - old_name = get_symbol_or_name + + begin + old_name = get_symbol_or_name + rescue RDoc::Error + return + end al = RDoc::Alias.new get_tkread, old_name, new_name, comment read_documentation_modifiers al, RDoc::ATTR_MODIFIERS - if al.document_self - context.add_alias(al) - end + context.add_alias al if al.document_self end def parse_call_parameters(tk) @@ -1847,23 +518,25 @@ class RDoc::Parser::Ruby < RDoc::Parser nest = 0 loop do - case tk - when TkSEMICOLON - break - when TkLPAREN, TkfLPAREN - nest += 1 - when end_token - if end_token == TkRPAREN - nest -= 1 - break if @scanner.lex_state == EXPR_END and nest <= 0 - else - break unless @scanner.continue - end - when TkCOMMENT - unget_tk(tk) - break + case tk + when TkSEMICOLON + break + when TkLPAREN, TkfLPAREN + nest += 1 + when end_token + if end_token == TkRPAREN + nest -= 1 + break if @scanner.lex_state == EXPR_END and nest <= 0 + else + break unless @scanner.continue end - tk = get_tk + when TkCOMMENT + unget_tk(tk) + break + when nil then + break + end + tk = get_tk end res = get_tkread.tr("\n", " ").strip res = "" if res == ";" @@ -1871,7 +544,7 @@ class RDoc::Parser::Ruby < RDoc::Parser end def parse_class(container, single, tk, comment) - container, name_t = get_class_or_module(container) + container, name_t = get_class_or_module container case name_t when TkCONSTANT @@ -1880,7 +553,7 @@ class RDoc::Parser::Ruby < RDoc::Parser if TkLT === peek_tk then get_tk - skip_tkspace(true) + skip_tkspace superclass = get_class_specification superclass = "<unknown>" if superclass.empty? end @@ -1888,25 +561,24 @@ class RDoc::Parser::Ruby < RDoc::Parser cls_type = single == SINGLE ? RDoc::SingleClass : RDoc::NormalClass cls = container.add_class cls_type, name, superclass - @stats.add_class cls - read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS cls.record_location @top_level - - parse_statements cls cls.comment = comment + @stats.add_class cls + + parse_statements cls when TkLSHFT case name = get_class_specification when "self", container.name - parse_statements(container, SINGLE) + parse_statements container, SINGLE else - other = RDoc::TopLevel.find_class_named(name) - unless other - # other = @top_level.add_class(NormalClass, name, nil) - # other.record_location(@top_level) - # other.comment = comment - other = RDoc::NormalClass.new "Dummy", nil + other = RDoc::TopLevel.find_class_named name + + unless other then + other = container.add_module RDoc::NormalModule, name + other.record_location @top_level + other.comment = comment end @stats.add_class other @@ -1920,47 +592,69 @@ class RDoc::Parser::Ruby < RDoc::Parser end end - def parse_constant(container, single, tk, comment) + def parse_constant(container, tk, comment) name = tk.name - skip_tkspace(false) + skip_tkspace false eq_tk = get_tk unless TkASSIGN === eq_tk then - unget_tk(eq_tk) + unget_tk eq_tk return end - nest = 0 get_tkread tk = get_tk + if TkGT === tk then - unget_tk(tk) - unget_tk(eq_tk) + unget_tk tk + unget_tk eq_tk return end + rhs_name = '' + loop do - case tk - when TkSEMICOLON + case tk + when TkSEMICOLON then + break + when TkLPAREN, TkfLPAREN, TkLBRACE, TkLBRACK, TkDO, TkIF, TkUNLESS, + TkCASE then + nest += 1 + when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then + nest -= 1 + when TkCOMMENT then + if nest <= 0 && @scanner.lex_state == EXPR_END + unget_tk tk break - when TkLPAREN, TkfLPAREN, TkLBRACE, TkLBRACK, TkDO - nest += 1 - when TkRPAREN, TkRBRACE, TkRBRACK, TkEND - nest -= 1 - when TkCOMMENT - if nest <= 0 && @scanner.lex_state == EXPR_END - unget_tk(tk) - break - end - when TkNL - if (nest <= 0) && ((@scanner.lex_state == EXPR_END) || (!@scanner.continue)) - unget_tk(tk) - break - end end - tk = get_tk + when TkCONSTANT then + rhs_name << tk.name + + if nest <= 0 and TkNL === peek_tk then + mod = if rhs_name =~ /^::/ then + RDoc::TopLevel.find_class_or_module rhs_name + else + container.find_module_named rhs_name + end + + container.add_module_alias mod, name if mod + get_tk # TkNL + break + end + when TkNL then + if nest <= 0 && + (@scanner.lex_state == EXPR_END || !@scanner.continue) then + unget_tk tk + break + end + when TkCOLON2, TkCOLON3 then + rhs_name << '::' + when nil then + break + end + tk = get_tk end res = get_tkread.tr("\n", " ").strip @@ -1969,42 +663,60 @@ class RDoc::Parser::Ruby < RDoc::Parser con = RDoc::Constant.new name, res, comment read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS - if con.document_self - container.add_constant(con) - end + @stats.add_constant con + container.add_constant con if con.document_self end + ## + # Generates an RDoc::Method or RDoc::Attr from +comment+ by looking for + # :method: or :attr: directives in +comment+. + def parse_comment(container, tk, comment) line_no = tk.line_no column = tk.char_no singleton = !!comment.sub!(/(^# +:?)(singleton-)(method:)/, '\1\3') + # REFACTOR if comment.sub!(/^# +:?method: *(\S*).*?\n/i, '') then name = $1 unless $1.empty? - else - return nil - end - meth = RDoc::GhostMethod.new get_tkread, name - meth.singleton = singleton + meth = RDoc::GhostMethod.new get_tkread, name + meth.singleton = singleton - @stats.add_method meth + meth.start_collecting_tokens + indent = TkSPACE.new nil, 1, 1 + indent.set_text " " * column - meth.start_collecting_tokens - indent = TkSPACE.new 1, 1 - indent.set_text " " * column + position_comment = TkCOMMENT.new nil, line_no, 1 + position_comment.set_text "# File #{@top_level.absolute_name}, line #{line_no}" + meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] - position_comment = TkCOMMENT.new(line_no, 1, "# File #{@top_level.file_absolute_name}, line #{line_no}") - meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] + meth.params = '' - meth.params = '' + extract_call_seq comment, meth - extract_call_seq comment, meth + return unless meth.name - container.add_method meth if meth.document_self + container.add_method meth if meth.document_self - meth.comment = comment + meth.comment = comment + + @stats.add_method meth + elsif comment.sub!(/# +:?(attr(_reader|_writer|_accessor)?:) *(\S*).*?\n/i, '') then + rw = case $1 + when 'attr_reader' then 'R' + when 'attr_writer' then 'W' + else 'RW' + end + + name = $3 unless $3.empty? + + att = RDoc::Attr.new get_tkread, name, rw, comment + container.add_attribute att + + @stats.add_method att + end end def parse_include(context, comment) @@ -2020,6 +732,66 @@ class RDoc::Parser::Ruby < RDoc::Parser end ## + # Parses a meta-programmed attribute and creates an RDoc::Attr. + # + # To create foo and bar attributes on class C with comment "My attributes": + # + # class C + # + # ## + # # :attr: + # # + # # My attributes + # + # my_attr :foo, :bar + # + # end + # + # To create a foo attribute on class C with comment "My attribute": + # + # class C + # + # ## + # # :attr: foo + # # + # # My attribute + # + # my_attr :foo, :bar + # + # end + + def parse_meta_attr(context, single, tk, comment) + args = parse_symbol_arg + read = get_tkread + rw = "?" + + # If nodoc is given, don't document any of them + + tmp = RDoc::CodeObject.new + read_documentation_modifiers tmp, RDoc::ATTR_MODIFIERS + return unless tmp.document_self + + if comment.sub!(/^# +:?(attr(_reader|_writer|_accessor)?): *(\S*).*?\n/i, '') then + rw = case $1 + when 'attr_reader' then 'R' + when 'attr_writer' then 'W' + else 'RW' + end + name = $3 unless $3.empty? + end + + if name then + att = RDoc::Attr.new get_tkread, name, rw, comment + context.add_attribute att + else + args.each do |attr_name| + att = RDoc::Attr.new get_tkread, attr_name, rw, comment + context.add_attribute att + end + end + end + + ## # Parses a meta-programmed method def parse_meta_method(container, single, tk, comment) @@ -2044,9 +816,12 @@ class RDoc::Parser::Ruby < RDoc::Parser when TkSYMBOL then name = name_t.text[1..-1] when TkSTRING then - name = name_t.text[1..-2] + name = name_t.value[1..-2] + when TkASSIGN then # ignore + remove_token_listener self + return else - warn "#{container.toplevel.file_relative_name}:#{name_t.line_no} unknown name token #{name_t.inspect} for meta-method" + warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'" name = 'unknown' end end @@ -2054,166 +829,183 @@ class RDoc::Parser::Ruby < RDoc::Parser meth = RDoc::MetaMethod.new get_tkread, name meth.singleton = singleton - @stats.add_method meth - remove_token_listener self meth.start_collecting_tokens - indent = TkSPACE.new 1, 1 + indent = TkSPACE.new nil, 1, 1 indent.set_text " " * column - position_comment = TkCOMMENT.new(line_no, 1, "# File #{@top_level.file_absolute_name}, line #{line_no}") + position_comment = TkCOMMENT.new nil, line_no, 1 + position_comment.value = "# File #{@top_level.absolute_name}, line #{line_no}" meth.add_tokens [position_comment, NEWLINE_TOKEN, indent] meth.add_tokens @token_stream - add_token_listener meth - - meth.params = '' + token_listener meth do + meth.params = '' - extract_call_seq comment, meth + extract_call_seq comment, meth - container.add_method meth if meth.document_self + container.add_method meth if meth.document_self - last_tk = tk + last_tk = tk - while tk = get_tk do - case tk - when TkSEMICOLON then - break - when TkNL then - break unless last_tk and TkCOMMA === last_tk - when TkSPACE then - # expression continues - else - last_tk = tk + while tk = get_tk do + case tk + when TkSEMICOLON then + break + when TkNL then + break unless last_tk and TkCOMMA === last_tk + when TkSPACE then + # expression continues + else + last_tk = tk + end end end - remove_token_listener meth - meth.comment = comment + + @stats.add_method meth end ## - # Parses a method + # Parses a normal method defined by +def+ def parse_method(container, single, tk, comment) + added_container = nil + meth = nil + name = nil line_no = tk.line_no column = tk.char_no start_collecting_tokens - add_token(tk) - add_token_listener(self) + add_token tk - @scanner.instance_eval do @lex_state = EXPR_FNAME end + token_listener self do + @scanner.instance_eval do @lex_state = EXPR_FNAME end - skip_tkspace(false) - name_t = get_tk - back_tk = skip_tkspace - meth = nil - added_container = false + skip_tkspace false + name_t = get_tk + back_tk = skip_tkspace + meth = nil + added_container = false - dot = get_tk - if TkDOT === dot or TkCOLON2 === dot then - @scanner.instance_eval do @lex_state = EXPR_FNAME end - skip_tkspace - name_t2 = get_tk + dot = get_tk + if TkDOT === dot or TkCOLON2 === dot then + @scanner.instance_eval do @lex_state = EXPR_FNAME end + skip_tkspace + name_t2 = get_tk + + case name_t + when TkSELF, TkMOD then + name = name_t2.name + when TkCONSTANT then + name = name_t2.name + prev_container = container + container = container.find_module_named(name_t.name) + unless container then + added_container = true + obj = name_t.name.split("::").inject(Object) do |state, item| + state.const_get(item) + end rescue nil + + type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule + + unless [Class, Module].include?(obj.class) then + warn("Couldn't find #{name_t.name}. Assuming it's a module") + end - case name_t - when TkSELF then - name = name_t2.name - when TkCONSTANT then - name = name_t2.name - prev_container = container - container = container.find_module_named(name_t.name) - unless container then - added_container = true - obj = name_t.name.split("::").inject(Object) do |state, item| - state.const_get(item) - end rescue nil - - type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule - - unless [Class, Module].include?(obj.class) then - warn("Couldn't find #{name_t.name}. Assuming it's a module") - end + if type == RDoc::NormalClass then + sclass = obj.superclass ? obj.superclass.name : nil + container = prev_container.add_class type, name_t.name, sclass + else + container = prev_container.add_module type, name_t.name + end - if type == RDoc::NormalClass then - container = prev_container.add_class(type, name_t.name, obj.superclass.name) - else - container = prev_container.add_module(type, name_t.name) + container.record_location @top_level end - - container.record_location @top_level + when TkIDENTIFIER, TkIVAR then + dummy = RDoc::Context.new + dummy.parent = container + skip_method dummy + return + else + warn "unexpected method name token #{name_t.inspect}" + # break + skip_method container + return end + + meth = RDoc::AnyMethod.new(get_tkread, name) + meth.singleton = true else - # warn("Unexpected token '#{name_t2.inspect}'") - # break - skip_method(container) - return - end + unget_tk dot + back_tk.reverse_each do |token| + unget_tk token + end - meth = RDoc::AnyMethod.new(get_tkread, name) - meth.singleton = true - else - unget_tk dot - back_tk.reverse_each do |token| - unget_tk token - end - name = name_t.name + name = case name_t + when TkSTAR, TkAMPER then + name_t.text + else + unless name_t.respond_to? :name then + warn "expected method name token, . or ::, got #{name_t.inspect}" + skip_method container + return + end + name_t.name + end - meth = RDoc::AnyMethod.new get_tkread, name - meth.singleton = (single == SINGLE) + meth = RDoc::AnyMethod.new get_tkread, name + meth.singleton = (single == SINGLE) + end end - @stats.add_method meth - - remove_token_listener self - meth.start_collecting_tokens - indent = TkSPACE.new 1, 1 + indent = TkSPACE.new nil, 1, 1 indent.set_text " " * column - token = TkCOMMENT.new(line_no, 1, "# File #{@top_level.file_absolute_name}, line #{line_no}") + token = TkCOMMENT.new nil, line_no, 1 + token.set_text "# File #{@top_level.absolute_name}, line #{line_no}" meth.add_tokens [token, NEWLINE_TOKEN, indent] meth.add_tokens @token_stream - add_token_listener meth - - @scanner.instance_eval do @continue = false end - parse_method_parameters meth + token_listener meth do + @scanner.instance_eval do @continue = false end + parse_method_parameters meth - if meth.document_self then - container.add_method meth - elsif added_container then - container.document_self = false - end + if meth.document_self then + container.add_method meth + elsif added_container then + container.document_self = false + end - # Having now read the method parameters and documentation modifiers, we - # now know whether we have to rename #initialize to ::new + # Having now read the method parameters and documentation modifiers, we + # now know whether we have to rename #initialize to ::new - if name == "initialize" && !meth.singleton then - if meth.dont_rename_initialize then - meth.visibility = :protected - else - meth.singleton = true - meth.name = "new" - meth.visibility = :public + if name == "initialize" && !meth.singleton then + if meth.dont_rename_initialize then + meth.visibility = :protected + else + meth.singleton = true + meth.name = "new" + meth.visibility = :public + end end - end - - parse_statements(container, single, meth) - remove_token_listener(meth) + parse_statements container, single, meth + end extract_call_seq comment, meth meth.comment = comment + + @stats.add_method meth end def parse_method_or_yield_parameters(method = nil, modifiers = RDoc::METHOD_MODIFIERS) - skip_tkspace(false) + skip_tkspace false tk = get_tk # Little hack going on here. In the statement @@ -2232,33 +1024,39 @@ class RDoc::Parser::Ruby < RDoc::Parser nest = 0 loop do - case tk - when TkSEMICOLON - break - when TkLBRACE - nest += 1 - when TkRBRACE - # we might have a.each {|i| yield i } - unget_tk(tk) if nest.zero? + case tk + when TkSEMICOLON then + break + when TkLBRACE then + nest += 1 + when TkRBRACE then + # we might have a.each {|i| yield i } + unget_tk(tk) if nest.zero? + nest -= 1 + break if nest <= 0 + when TkLPAREN, TkfLPAREN then + nest += 1 + when end_token then + if end_token == TkRPAREN nest -= 1 - break if nest <= 0 - when TkLPAREN, TkfLPAREN - nest += 1 - when end_token - if end_token == TkRPAREN - nest -= 1 - break if @scanner.lex_state == EXPR_END and nest <= 0 - else - break unless @scanner.continue - end - when method && method.block_params.nil? && TkCOMMENT - unget_tk(tk) - read_documentation_modifiers(method, modifiers) + break if @scanner.lex_state == EXPR_END and nest <= 0 + else + break unless @scanner.continue end + when method && method.block_params.nil? && TkCOMMENT then + unget_tk tk + read_documentation_modifiers method, modifiers + @read.pop + when TkCOMMENT then + @read.pop + when nil then + break + end tk = get_tk end - res = get_tkread.tr("\n", " ").strip - res = "" if res == ";" + + res = get_tkread.gsub(/\s+/, ' ').strip + res = '' if res == ';' res end @@ -2271,56 +1069,53 @@ class RDoc::Parser::Ruby < RDoc::Parser # and add this as the block_params for the method def parse_method_parameters(method) - res = parse_method_or_yield_parameters(method) - res = "(" + res + ")" unless res[0] == ?( + res = parse_method_or_yield_parameters method + + res = "(#{res})" unless res =~ /\A\(/ method.params = res unless method.params - if method.block_params.nil? - skip_tkspace(false) + + if method.block_params.nil? then + skip_tkspace false read_documentation_modifiers method, RDoc::METHOD_MODIFIERS end end def parse_module(container, single, tk, comment) - container, name_t = get_class_or_module(container) + container, name_t = get_class_or_module container name = name_t.name mod = container.add_module RDoc::NormalModule, name mod.record_location @top_level - @stats.add_module mod - read_documentation_modifiers mod, RDoc::CLASS_MODIFIERS parse_statements(mod) mod.comment = comment + + @stats.add_module mod end def parse_require(context, comment) skip_tkspace_comment tk = get_tk + if TkLPAREN === tk then skip_tkspace_comment tk = get_tk end - name = nil - case tk - when TkSTRING - name = tk.text - # when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR - # name = tk.name - when TkDSTRING - warn "Skipping require of dynamic string: #{tk.text}" - # else - # warn "'require' used as variable" - end - if name + name = tk.text if TkSTRING === tk + + if name then context.add_require RDoc::Require.new(name, comment) else - unget_tk(tk) + unget_tk tk end end + ## + # The core of the ruby parser. + def parse_statements(container, single = NORMAL, current_method = nil, comment = '') nest = 1 @@ -2335,7 +1130,7 @@ class RDoc::Parser::Ruby < RDoc::Parser case tk when TkNL then - skip_tkspace true # Skip blanks and newlines + skip_tkspace tk = get_tk if TkCOMMENT === tk then @@ -2349,8 +1144,9 @@ class RDoc::Parser::Ruby < RDoc::Parser while TkCOMMENT === tk do comment << tk.text << "\n" - tk = get_tk # this is the newline - skip_tkspace(false) # leading spaces + + tk = get_tk # this is the newline + skip_tkspace false # leading spaces tk = get_tk end @@ -2393,11 +1189,11 @@ class RDoc::Parser::Ruby < RDoc::Parser when TkCONSTANT then if container.document_self then - parse_constant container, single, tk, comment + parse_constant container, tk, comment end when TkALIAS then - if container.document_self then + if container.document_self and not current_method then parse_alias container, single, tk, comment end @@ -2434,15 +1230,21 @@ class RDoc::Parser::Ruby < RDoc::Parser keep_comment = true when 'attr' then parse_attr container, single, tk, comment - when /^attr_(reader|writer|accessor)$/, @options.extra_accessors then + when /^attr_(reader|writer|accessor)$/ then parse_attr_accessor container, single, tk, comment when 'alias_method' then - if container.document_self then - parse_alias container, single, tk, comment - end + parse_alias container, single, tk, comment if + container.document_self + when 'require', 'include' then + # ignore else if container.document_self and comment =~ /\A#\#$/ then - parse_meta_method container, single, tk, comment + case comment + when /^# +:?attr(_reader|_writer|_accessor)?:/ then + parse_meta_attr container, single, tk, comment + else + parse_meta_method container, single, tk, comment + end end end end @@ -2459,16 +1261,18 @@ class RDoc::Parser::Ruby < RDoc::Parser if nest == 0 then read_documentation_modifiers container, RDoc::CLASS_MODIFIERS container.ongoing_visibility = save_visibility + + parse_comment container, tk, comment unless comment.empty? + return end - end comment = '' unless keep_comment begin get_tkread - skip_tkspace(false) + skip_tkspace false end while peek_tk == TkNL end end @@ -2503,7 +1307,7 @@ class RDoc::Parser::Ruby < RDoc::Parser end loop do - skip_tkspace(false) + skip_tkspace false tk1 = get_tk unless TkCOMMA === tk1 then @@ -2533,7 +1337,7 @@ class RDoc::Parser::Ruby < RDoc::Parser end end - def parse_toplevel_statements(container) + def parse_top_level_statements(container) comment = collect_first_comment look_for_directives_in(container, comment) container.comment = comment unless comment.empty? @@ -2559,7 +1363,7 @@ class RDoc::Parser::Ruby < RDoc::Parser singleton = true :public else - raise "Invalid visibility: #{tk.name}" + raise RDoc::Error, "Invalid visibility: #{tk.name}" end skip_tkspace_comment false @@ -2613,18 +1417,6 @@ class RDoc::Parser::Ruby < RDoc::Parser end end - def peek_read - @read.join('') - end - - ## - # Peek at the next token, but don't remove it from the stream - - def peek_tk - unget_tk(tk = get_tk) - tk - end - ## # Directives are modifier comments that can appear after class, module, or # method names. For example: @@ -2640,16 +1432,18 @@ class RDoc::Parser::Ruby < RDoc::Parser def read_directive(allowed) tk = get_tk result = nil - if TkCOMMENT === tk - if tk.text =~ /\s*:?(\w+):\s*(.*)/ + + if TkCOMMENT === tk then + if tk.text =~ /\s*:?(\w+):\s*(.*)/ then directive = $1.downcase - if allowed.include?(directive) + if allowed.include? directive then result = [directive, $2] end end else - unget_tk(tk) + unget_tk tk end + result end @@ -2687,40 +1481,40 @@ class RDoc::Parser::Ruby < RDoc::Parser comment.sub!(/^#--\n.*/m, '') end - def remove_token_listener(obj) - @token_listeners.delete(obj) - end - - def reset - @tokens = [] - @unget_read = [] - @read = [] - end - def scan reset - catch(:eof) do - catch(:enddoc) do + catch :eof do + catch :enddoc do begin - parse_toplevel_statements(@top_level) - rescue Exception => e - $stderr.puts <<-EOF - - -RDoc failure in #{@file_name} at or around line #{@scanner.line_no} column -#{@scanner.char_no} + parse_top_level_statements @top_level + rescue StandardError => e + bytes = '' + + 20.times do @scanner.ungetc end + count = 0 + 60.times do |i| + count = i + byte = @scanner.getc + break unless byte + bytes << byte + end + count -= 20 + count.times do @scanner.ungetc end -Before reporting this, could you check that the file you're documenting -compiles cleanly--RDoc is not a full Ruby parser, and gets confused easily if -fed invalid programs. + $stderr.puts <<-EOF -The internal error was: +#{self.class} failure around line #{@scanner.line_no} of +#{@file_name} EOF - e.set_backtrace(e.backtrace[0,4]) - raise + unless bytes.empty? then + $stderr.puts + $stderr.puts bytes.inspect + end + + raise e end end end @@ -2732,7 +1526,7 @@ The internal error was: # while, until, and for have an optional do def skip_optional_do_after_expression - skip_tkspace(false) + skip_tkspace false tk = get_tk case tk when TkLPAREN, TkfLPAREN @@ -2759,10 +1553,12 @@ The internal error was: else break unless @scanner.continue end + when nil then + break end tk = get_tk end - skip_tkspace(false) + skip_tkspace false get_tk if TkDO === peek_tk end @@ -2771,31 +1567,17 @@ The internal error was: # skip the var [in] part of a 'for' statement def skip_for_variable - skip_tkspace(false) + skip_tkspace false tk = get_tk - skip_tkspace(false) + skip_tkspace false tk = get_tk unget_tk(tk) unless TkIN === tk end - def skip_method(container) + def skip_method container meth = RDoc::AnyMethod.new "", "anon" - parse_method_parameters(meth) - parse_statements(container, false, meth) - end - - ## - # Skip spaces - - def skip_tkspace(skip_nl = true) - tokens = [] - - while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do - tokens.push tk - end - - unget_tk(tk) - tokens + parse_method_parameters meth + parse_statements container, false, meth end ## @@ -2803,22 +1585,12 @@ The internal error was: def skip_tkspace_comment(skip_nl = true) loop do - skip_tkspace(skip_nl) + skip_tkspace skip_nl return unless TkCOMMENT === peek_tk get_tk end end - def unget_tk(tk) - @tokens.unshift tk - @unget_read.unshift @read.pop - - # Remove this token from any listeners - @token_listeners.each do |obj| - obj.pop_token - end if @token_listeners - end - def warn(msg) return if @options.quiet msg = make_message msg diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb new file mode 100644 index 0000000000..90c03307b4 --- /dev/null +++ b/lib/rdoc/parser/ruby_tools.rb @@ -0,0 +1,157 @@ +## +# Collection of methods for writing parsers against RDoc::RubyLex and +# RDoc::RubyToken + +module RDoc::Parser::RubyTools + + include RDoc::RubyToken + + ## + # Adds a token listener +obj+, but you should probably use token_listener + + def add_token_listener(obj) + @token_listeners ||= [] + @token_listeners << obj + end + + ## + # Fetches the next token from the scanner + + def get_tk + tk = nil + + if @tokens.empty? then + tk = @scanner.token + @read.push @scanner.get_readed + puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG + else + @read.push @unget_read.shift + tk = @tokens.shift + puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG + end + + tk = nil if TkEND_OF_SCRIPT === tk + + if TkSYMBEG === tk then + set_token_position tk.line_no, tk.char_no + + case tk1 = get_tk + when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then + if tk1.respond_to?(:name) then + tk = Token(TkSYMBOL).set_text(":" + tk1.name) + else + tk = Token(TkSYMBOL).set_text(":" + tk1.text) + end + + # remove the identifier we just read (we're about to replace it with a + # symbol) + @token_listeners.each do |obj| + obj.pop_token + end if @token_listeners + else + warn("':' not followed by identifier or operator") + tk = tk1 + end + end + + # inform any listeners of our shiny new token + @token_listeners.each do |obj| + obj.add_token(tk) + end if @token_listeners + + tk + end + + def get_tk_until(*tokens) + read = [] + + loop do + tk = get_tk + case tk when *tokens then unget_tk tk; break end + read << tk + end + + read + end + + ## + # Retrieves a String representation of the read tokens + + def get_tkread + read = @read.join("") + @read = [] + read + end + + ## + # Peek equivalent for get_tkread + + def peek_read + @read.join('') + end + + ## + # Peek at the next token, but don't remove it from the stream + + def peek_tk + unget_tk(tk = get_tk) + tk + end + + ## + # Removes the token listener +obj+ + + def remove_token_listener(obj) + @token_listeners.delete(obj) + end + + ## + # Resets the tools + + def reset + @read = [] + @tokens = [] + @unget_read = [] + @nest = 0 + end + + ## + # Skips whitespace tokens including newlines if +skip_nl+ is true + + def skip_tkspace(skip_nl = true) # HACK dup + tokens = [] + + while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do + tokens.push tk + end + + unget_tk tk + tokens + end + + ## + # Has +obj+ listen to tokens + + def token_listener(obj) + add_token_listener obj + yield + ensure + remove_token_listener obj + end + + ## + # Returns +tk+ to the scanner + + def unget_tk(tk) + @tokens.unshift tk + @unget_read.unshift @read.pop + + # Remove this token from any listeners + @token_listeners.each do |obj| + obj.pop_token + end if @token_listeners + end + +end + + diff --git a/lib/rdoc/parser/simple.rb b/lib/rdoc/parser/simple.rb index cdfe686718..9072667f89 100644 --- a/lib/rdoc/parser/simple.rb +++ b/lib/rdoc/parser/simple.rb @@ -1,5 +1,3 @@ -require 'rdoc/parser' - ## # Parse a non-source file. We basically take the whole thing as one big # comment. If the first character in the file is '#', we strip leading pound @@ -23,10 +21,11 @@ class RDoc::Parser::Simple < RDoc::Parser end ## - # Extract the file contents and attach them to the toplevel as a comment + # Extract the file contents and attach them to the TopLevel as a comment def scan @top_level.comment = remove_private_comments(@content) + @top_level.parser = self.class @top_level end |