23 files changed, 1461 insertions, 3216 deletions

diff --git a/lib/rdoc.rb b/lib/rdoc.rb
index 8482e910f5..3856468421 100644
--- a/lib/rdoc.rb
+++ b/lib/rdoc.rb
@@ -65,7 +65,7 @@ module RDoc
   ##
   # RDoc version you are using
 
-  VERSION = '6.0.0.beta1'
+  VERSION = '6.0.0.beta2'
 
   ##
   # Method visibilities
@@ -148,7 +148,7 @@ module RDoc
 
   autoload :KNOWN_CLASSES,  'rdoc/known_classes'
 
-  autoload :RubyLex,        'rdoc/ruby_lex'
+  autoload :RipperStateLex, 'rdoc/parser/ripper_state_lex'
   autoload :RubyToken,      'rdoc/ruby_token'
   autoload :TokenStream,    'rdoc/token_stream'
 
diff --git a/lib/rdoc/constant.rb b/lib/rdoc/constant.rb
index 4fd5c5f10f..7e1e10da72 100644
--- a/lib/rdoc/constant.rb
+++ b/lib/rdoc/constant.rb
@@ -36,7 +36,7 @@ class RDoc::Constant < RDoc::CodeObject
     @value = value
 
     @is_alias_for = nil
-    @visibility   = nil
+    @visibility   = :public
 
     self.comment = comment
   end
@@ -136,7 +136,7 @@ class RDoc::Constant < RDoc::CodeObject
     initialize array[1], nil, array[5]
 
     @full_name     = array[2]
-    @visibility    = array[3]
+    @visibility    = array[3] || :public
     @is_alias_for  = array[4]
     #                      5 handled above
     #                      6 handled below
diff --git a/lib/rdoc/context.rb b/lib/rdoc/context.rb
index 7fd6bb0af9..3294b2561c 100644
--- a/lib/rdoc/context.rb
+++ b/lib/rdoc/context.rb
@@ -1079,6 +1079,7 @@ class RDoc::Context < RDoc::CodeObject
     return if [:private, :nodoc].include? min_visibility
     remove_invisible_in @method_list, min_visibility
     remove_invisible_in @attributes, min_visibility
+    remove_invisible_in @constants, min_visibility
   end
 
   ##
@@ -1166,6 +1167,17 @@ class RDoc::Context < RDoc::CodeObject
   end
 
   ##
+  # Given an array +names+ of constants, set the visibility of each constant to
+  # +visibility+
+
+  def set_constant_visibility_for(names, visibility)
+    names.each do |name|
+      constant = @constants_hash[name] or next
+      constant.visibility = visibility
+    end
+  end
+
+  ##
   # Sorts sections alphabetically (default) or in TomDoc fashion (none,
   # Public, Internal, Deprecated)
 
diff --git a/lib/rdoc/markdown.rb b/lib/rdoc/markdown.rb
index b10f5193da..9b433b3ee6 100644
--- a/lib/rdoc/markdown.rb
+++ b/lib/rdoc/markdown.rb
@@ -15280,7 +15280,7 @@ class RDoc::Markdown
         self.pos = _save
         break
       end
-      @result = begin;
+      @result = begin; 
                ref = [:inline, @note_order.length]
                @footnotes[ref] = paragraph a
 
diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb
index aa5de7bf66..98ed7926fd 100644
--- a/lib/rdoc/markup/to_html.rb
+++ b/lib/rdoc/markup/to_html.rb
@@ -200,10 +200,12 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
 
     content = if verbatim.ruby? or parseable? text then
                 begin
-                  tokens = RDoc::RubyLex.tokenize text, @options
+                  tokens = RDoc::RipperStateLex.parse text
                   klass  = ' class="ruby"'
 
-                  RDoc::TokenStream.to_html tokens
+                  result = RDoc::TokenStream.to_html tokens
+                  result = result + "\n" unless "\n" == result[-1]
+                  result
                 rescue RDoc::RubyLex::Error
                   CGI.escapeHTML text
                 end
@@ -212,7 +214,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter
               end
 
     if @options.pipe then
-      @res << "\n<pre><code>#{CGI.escapeHTML text}</code></pre>\n"
+      @res << "\n<pre><code>#{CGI.escapeHTML text}\n</code></pre>\n"
     else
       @res << "\n<pre#{klass}>#{content}</pre>\n"
     end
diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb
index 0e72df2ecd..5c940ab28e 100644
--- a/lib/rdoc/parser/c.rb
+++ b/lib/rdoc/parser/c.rb
@@ -666,8 +666,7 @@ class RDoc::Parser::C < RDoc::Parser
 
       #meth_obj.params = params
       meth_obj.start_collecting_tokens
-      tk = RDoc::RubyToken::Token.new nil, 1, 1
-      tk.set_text body
+      tk = { :line_no => 1, :char_no => 1, :text => body }
       meth_obj.add_token tk
       meth_obj.comment = comment
       meth_obj.line    = file_content[0, offset].count("\n") + 1
@@ -684,8 +683,7 @@ class RDoc::Parser::C < RDoc::Parser
       find_modifiers comment, meth_obj
 
       meth_obj.start_collecting_tokens
-      tk = RDoc::RubyToken::Token.new nil, 1, 1
-      tk.set_text body
+      tk = { :line_no => 1, :char_no => 1, :text => body }
       meth_obj.add_token tk
       meth_obj.comment = comment
       meth_obj.line    = file_content[0, offset].count("\n") + 1
diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb
new file mode 100644
index 0000000000..c9a0f5a21e
--- /dev/null
+++ b/lib/rdoc/parser/ripper_state_lex.rb
@@ -0,0 +1,587 @@
+require 'ripper'
+
+class RDoc::RipperStateLex
+  EXPR_NONE = 0
+  EXPR_BEG = 1
+  EXPR_END = 2
+  EXPR_ENDARG = 4
+  EXPR_ENDFN = 8
+  EXPR_ARG = 16
+  EXPR_CMDARG = 32
+  EXPR_MID = 64
+  EXPR_FNAME = 128
+  EXPR_DOT = 256
+  EXPR_CLASS = 512
+  EXPR_LABEL = 1024
+  EXPR_LABELED = 2048
+  EXPR_FITEM = 4096
+  EXPR_VALUE = EXPR_BEG
+  EXPR_BEG_ANY  =  (EXPR_BEG | EXPR_MID | EXPR_CLASS)
+  EXPR_ARG_ANY  =  (EXPR_ARG | EXPR_CMDARG)
+  EXPR_END_ANY  =  (EXPR_END | EXPR_ENDARG | EXPR_ENDFN)
+
+  class InnerStateLex < Ripper::Filter
+    attr_accessor :lex_state
+
+    def initialize(code)
+      @lex_state = EXPR_BEG
+      @in_fname = false
+      @continue = false
+      reset
+      super(code)
+    end
+
+    def reset
+      @command_start = false
+      @cmd_state = @command_start
+    end
+
+    def on_nl(tok, data)
+      case @lex_state
+      when EXPR_FNAME, EXPR_DOT
+        @continue = true
+      else
+        @continue = false
+        @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_ignored_nl(tok, data)
+      case @lex_state
+      when EXPR_FNAME, EXPR_DOT
+        @continue = true
+      else
+        @continue = false
+        @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_op(tok, data)
+      case tok
+      when '&', '|', '!', '!=', '!~'
+        case @lex_state
+        when EXPR_FNAME, EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_BEG
+        end
+      when '<<'
+        # TODO next token?
+        case @lex_state
+        when EXPR_FNAME, EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_BEG
+        end
+      when '?'
+        @lex_state = EXPR_BEG
+      when '&&', '||', '+=', '-=', '*=', '**=',
+           '&=', '|=', '^=', '<<=', '>>=', '||=', '&&='
+        @lex_state = EXPR_BEG
+      else
+        case @lex_state
+        when EXPR_FNAME, EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_BEG
+        end
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_kw(tok, data)
+      case tok
+      when 'class'
+        @lex_state = EXPR_CLASS
+        @in_fname = true
+      when 'def'
+        @lex_state = EXPR_FNAME
+        @continue = true
+        @in_fname = true
+      when 'if', 'unless', 'while', 'until'
+        if ((EXPR_END | EXPR_ENDARG | EXPR_ENDFN | EXPR_ARG | EXPR_CMDARG) & @lex_state) != 0 # postfix if
+          @lex_state = EXPR_BEG | EXPR_LABEL
+        else
+          @lex_state = EXPR_BEG
+        end
+      when 'begin'
+        @lex_state = EXPR_BEG
+      else
+        if @lex_state == EXPR_FNAME
+          @lex_state = EXPR_END
+        else
+          @lex_state = EXPR_END
+        end
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_tstring_beg(tok, data)
+      @lex_state = EXPR_BEG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_tstring_end(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_CHAR(tok, data)
+      @lex_state = EXPR_END
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_period(tok, data)
+      @lex_state = EXPR_DOT
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_int(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_float(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_rational(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_imaginary(tok, data)
+      @lex_state = EXPR_END | EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_symbeg(tok, data)
+      @lex_state = EXPR_FNAME
+      @continue = true
+      @in_fname = true
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    private def on_variables(event, tok, data)
+      if @in_fname
+        @lex_state = EXPR_ENDFN
+        @in_fname = false
+        @continue = false
+      elsif @continue
+        case @lex_state
+        when EXPR_DOT
+          @lex_state = EXPR_ARG
+        else
+          @lex_state = EXPR_ENDFN
+          @continue = false
+        end
+      else
+        @lex_state = EXPR_CMDARG
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+    end
+
+    def on_ident(tok, data)
+      on_variables(__method__, tok, data)
+    end
+
+    def on_ivar(tok, data)
+      @lex_state = EXPR_END
+      on_variables(__method__, tok, data)
+    end
+
+    def on_cvar(tok, data)
+      @lex_state = EXPR_END
+      on_variables(__method__, tok, data)
+    end
+
+    def on_gvar(tok, data)
+      @lex_state = EXPR_END
+      on_variables(__method__, tok, data)
+    end
+
+    def on_backref(tok, data)
+      @lex_state = EXPR_END
+      on_variables(__method__, tok, data)
+    end
+
+    def on_lparen(tok, data)
+      @lex_state = EXPR_LABEL | EXPR_BEG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_rparen(tok, data)
+      @lex_state = EXPR_ENDFN
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_lbrace(tok, data)
+      @lex_state = EXPR_LABEL | EXPR_BEG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_rbrace(tok, data)
+      @lex_state = EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_lbracket(tok, data)
+      @lex_state = EXPR_LABEL | EXPR_BEG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_rbracket(tok, data)
+      @lex_state = EXPR_ENDARG
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_const(tok, data)
+      case @lex_state
+      when EXPR_FNAME
+        @lex_state = EXPR_ENDFN
+      when EXPR_CLASS
+        @lex_state = EXPR_ARG
+      else
+        @lex_state = EXPR_CMDARG
+      end
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_sp(tok, data)
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_comma(tok, data)
+      @lex_state = EXPR_BEG | EXPR_LABEL if (EXPR_ARG_ANY & @lex_state) != 0
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_comment(tok, data)
+      @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_ignored_sp(tok, data)
+      @lex_state = EXPR_BEG unless (EXPR_LABEL & @lex_state) != 0
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+    end
+
+    def on_heredoc_end(tok, data)
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => __method__, :text => tok, :state => @lex_state})
+      @lex_state = EXPR_BEG
+    end
+
+    def on_default(event, tok, data)
+      reset
+      @callback.call({ :line_no => lineno, :char_no => column, :kind => event, :text => tok, :state => @lex_state})
+    end
+
+    def each(&block)
+      @callback = block
+      parse
+    end
+  end
+
+  def get_squashed_tk
+    if @buf.empty?
+      tk = @inner_lex_enumerator.next
+    else
+      tk = @buf.shift
+    end
+    case tk[:kind]
+    when :on_symbeg then
+      tk = get_symbol_tk(tk)
+    when :on_tstring_beg then
+      tk = get_string_tk(tk)
+    when :on_backtick then
+      if (EXPR_FNAME & tk[:state]) != 0
+        @inner_lex.lex_state = EXPR_ARG
+        tk[:kind] = :on_ident
+        tk[:state] = @inner_lex.lex_state
+      else
+        tk = get_string_tk(tk)
+      end
+    when :on_regexp_beg then
+      tk = get_regexp_tk(tk)
+    when :on_embdoc_beg then
+      tk = get_embdoc_tk(tk)
+    when :on_heredoc_beg then
+      @heredoc_queue << retrieve_heredoc_info(tk)
+      @inner_lex.lex_state = EXPR_END
+    when :on_nl, :on_ignored_nl, :on_comment, :on_heredoc_end then
+      unless @heredoc_queue.empty?
+        get_heredoc_tk(*@heredoc_queue.shift)
+      end
+    when :on_words_beg then
+      tk = get_words_tk(tk)
+    when :on_qwords_beg then
+      tk = get_words_tk(tk)
+    when :on_symbols_beg then
+      tk = get_words_tk(tk)
+    when :on_qsymbols_beg then
+      tk = get_words_tk(tk)
+    when :on_op then
+      if '&.' == tk[:text]
+        tk[:kind] = :on_period
+      else
+        tk = get_op_tk(tk)
+      end
+    end
+    tk
+  end
+
+  private def get_symbol_tk(tk)
+    is_symbol = true
+    symbol_tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => :on_symbol }
+    if ":'" == tk[:text] or ':"' == tk[:text]
+      tk1 = get_string_tk(tk)
+      symbol_tk[:text] = tk1[:text]
+      symbol_tk[:state] = tk1[:state]
+    else
+      case (tk1 = get_squashed_tk)[:kind]
+      when :on_ident
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_tstring_content
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = get_squashed_tk[:state] # skip :on_tstring_end
+      when :on_tstring_end
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_op
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_ivar
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_cvar
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_gvar
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_const
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      when :on_kw
+        symbol_tk[:text] = ":#{tk1[:text]}"
+        symbol_tk[:state] = tk1[:state]
+      else
+        is_symbol = false
+        tk = tk1
+      end
+    end
+    if is_symbol
+      tk = symbol_tk
+    end
+    tk
+  end
+
+  private def get_string_tk(tk)
+    string = tk[:text]
+    state = nil
+    kind = :on_tstring
+    loop do
+      inner_str_tk = get_squashed_tk
+      if inner_str_tk.nil?
+        break
+      elsif :on_tstring_end == inner_str_tk[:kind]
+        string = string + inner_str_tk[:text]
+        state = inner_str_tk[:state]
+        break
+      elsif :on_label_end == inner_str_tk[:kind]
+        string = string + inner_str_tk[:text]
+        state = inner_str_tk[:state]
+        kind = :on_symbol
+        break
+      else
+        string = string + inner_str_tk[:text]
+        if :on_embexpr_beg == inner_str_tk[:kind] then
+          kind = :on_dstring if :on_tstring == kind
+        end
+      end
+    end
+    {
+      :line_no => tk[:line_no],
+      :char_no => tk[:char_no],
+      :kind => kind,
+      :text => string,
+      :state => state
+    }
+  end
+
+  private def get_regexp_tk(tk)
+    string = tk[:text]
+    state = nil
+    loop do
+      inner_str_tk = get_squashed_tk
+      if inner_str_tk.nil?
+        break
+      elsif :on_regexp_end == inner_str_tk[:kind]
+        string = string + inner_str_tk[:text]
+        state = inner_str_tk[:state]
+        break
+      else
+        string = string + inner_str_tk[:text]
+      end
+    end
+    {
+      :line_no => tk[:line_no],
+      :char_no => tk[:char_no],
+      :kind => :on_regexp,
+      :text => string,
+      :state => state
+    }
+  end
+
+  private def get_embdoc_tk(tk)
+    string = tk[:text]
+    until :on_embdoc_end == (embdoc_tk = get_squashed_tk)[:kind] do
+      string = string + embdoc_tk[:text]
+    end
+    string = string + embdoc_tk[:text]
+    {
+      :line_no => tk[:line_no],
+      :char_no => tk[:char_no],
+      :kind => :on_embdoc,
+      :text => string,
+      :state => embdoc_tk[:state]
+    }
+  end
+
+  private def get_heredoc_tk(heredoc_name, indent)
+    string = ''
+    start_tk = nil
+    prev_tk = nil
+    until heredoc_end?(heredoc_name, indent, tk = @inner_lex_enumerator.next) do
+      start_tk = tk unless start_tk
+      if (prev_tk.nil? or "\n" == prev_tk[:text][-1]) and 0 != tk[:char_no]
+        string = string + (' ' * tk[:char_no])
+      end
+      string = string + tk[:text]
+      prev_tk = tk
+    end
+    start_tk = tk unless start_tk
+    prev_tk = tk unless prev_tk
+    @buf.unshift tk # closing heredoc
+    heredoc_tk = {
+      :line_no => start_tk[:line_no],
+      :char_no => start_tk[:char_no],
+      :kind => :on_heredoc,
+      :text => string,
+      :state => prev_tk[:state]
+    }
+    @buf.unshift heredoc_tk
+  end
+
+  private def retrieve_heredoc_info(tk)
+    name = tk[:text].gsub(/\A<<[-~]?(['"`]?)(.+)\1\z/, '\2')
+    indent = tk[:text] =~ /\A<<[-~]/
+    [name, indent]
+  end
+
+  private def heredoc_end?(name, indent, tk)
+    result = false
+    if :on_heredoc_end == tk[:kind] then
+      tk_name = (indent ? tk[:text].gsub(/^ *(.+)\n?$/, '\1') : tk[:text].gsub(/\n\z/, ''))
+      if name == tk_name
+        result = true
+      end
+    end
+    result
+  end
+
+  private def get_words_tk(tk)
+    string = ''
+    start_token = tk[:text]
+    start_quote = tk[:text].rstrip[-1]
+    line_no = tk[:line_no]
+    char_no = tk[:char_no]
+    state = tk[:state]
+    end_quote =
+      case start_quote
+      when ?( then ?)
+      when ?[ then ?]
+      when ?{ then ?}
+      when ?< then ?>
+      else start_quote
+      end
+    end_token = nil
+    loop do
+      tk = get_squashed_tk
+      if tk.nil?
+        end_token = end_quote
+        break
+      elsif :on_tstring_content == tk[:kind] then
+        string += tk[:text]
+      elsif :on_words_sep == tk[:kind] or :on_tstring_end == tk[:kind] then
+        if end_quote == tk[:text].strip then
+          end_token = tk[:text]
+          break
+        else
+          string += tk[:text]
+        end
+      else
+        string += tk[:text]
+      end
+    end
+    text = "#{start_token}#{string}#{end_token}"
+    {
+      :line_no => line_no,
+      :char_no => char_no,
+      :kind => :on_dstring,
+      :text => text,
+      :state => state
+    }
+  end
+
+  private def get_op_tk(tk)
+    redefinable_operators = %w[! != !~ % & * ** + +@ - -@ / < << <= <=> == === =~ > >= >> [] []= ^ ` | ~]
+    if redefinable_operators.include?(tk[:text]) and EXPR_ARG == tk[:state] then
+      @inner_lex.lex_state = EXPR_ARG
+      tk[:kind] = :on_ident
+      tk[:state] = @inner_lex.lex_state
+    elsif tk[:text] =~ /^[-+]$/ then
+      tk_ahead = get_squashed_tk
+      case tk_ahead[:kind]
+      when :on_int, :on_float, :on_rational, :on_imaginary then
+        tk[:text] += tk_ahead[:text]
+        tk[:kind] = tk_ahead[:kind]
+        tk[:state] = tk_ahead[:state]
+      else
+        @buf.unshift tk_ahead
+      end
+    end
+    tk
+  end
+
+  def initialize(code)
+    @buf = []
+    @heredoc_queue = []
+    @inner_lex = InnerStateLex.new(code)
+    @inner_lex_enumerator = Enumerator.new do |y|
+      @inner_lex.each do |tk|
+        y << tk
+      end
+    end
+  end
+
+  def self.parse(code)
+    lex = self.new(code)
+    tokens = []
+    begin
+      while tk = lex.get_squashed_tk
+        tokens.push tk
+      end
+    rescue StopIteration
+    end
+    tokens
+  end
+
+  def self.end?(token)
+    (token[:state] & EXPR_END)
+  end
+end
diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb
index e7900c0807..130eca89c7 100644
--- a/lib/rdoc/parser/ruby.rb
+++ b/lib/rdoc/parser/ruby.rb
@@ -24,6 +24,7 @@ $TOKEN_DEBUG ||= nil
 # * aliases
 # * private, public, protected
 # * private_class_function, public_class_function
+# * private_constant, public_constant
 # * module_function
 # * attr, attr_reader, attr_writer, attr_accessor
 # * extra accessors given on the command line
@@ -139,11 +140,12 @@ $TOKEN_DEBUG ||= nil
 # Note that by default, the :method: directive will be ignored if there is a
 # standard rdocable item following it.
 
+require 'ripper'
+
 class RDoc::Parser::Ruby < RDoc::Parser
 
   parse_files_matching(/\.rbw?$/)
 
-  include RDoc::RubyToken
   include RDoc::TokenStream
   include RDoc::Parser::RubyTools
 
@@ -163,10 +165,20 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def initialize(top_level, file_name, content, options, stats)
     super
 
+    if /\t/ =~ content then
+      tab_width = @options.tab_width
+      content = content.split(/\n/).map do |line|
+        1 while line.gsub!(/\t+/) {
+          ' ' * (tab_width*$&.length - $`.length % tab_width)
+        }  && $~
+        line
+      end.join("\n")
+    end
+
     @size = 0
     @token_listeners = nil
-    @scanner = RDoc::RubyLex.new content, @options
-    @scanner.exception_on_syntax_error = false
+    @scanner = RDoc::RipperStateLex.parse(content)
+    @scanner_point = 0
     @prev_seek = nil
     @markup = @options.markup
     @track_visibility = :nodoc != @options.visibility
@@ -175,6 +187,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     reset
   end
 
+  def tk_nl?(tk)
+    :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+  end
+
   ##
   # Retrieves the read token stream and replaces +pattern+ with +replacement+
   # using gsub.  If the result is only a ";" returns an empty string.
@@ -194,7 +210,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # methods.
 
   def get_visibility_information tk, single # :nodoc:
-    vis_type  = tk.name
+    vis_type  = tk[:text]
     singleton = single == SINGLE
 
     vis =
@@ -226,27 +242,27 @@ class RDoc::Parser::Ruby < RDoc::Parser
     comment = ''
     comment.force_encoding @encoding if @encoding
     first_line = true
-    first_comment_tk_class = nil
+    first_comment_tk_kind = nil
 
     tk = get_tk
 
-    while TkCOMMENT === tk
-      if first_line and tk.text =~ /\A#!/ then
+    while tk && (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
+      if first_line and tk[:text] =~ /\A#!/ then
         skip_tkspace
         tk = get_tk
-      elsif first_line and tk.text =~ /\A#\s*-\*-/ then
+      elsif first_line and tk[:text] =~ /\A#\s*-\*-/ then
         first_line = false
         skip_tkspace
         tk = get_tk
       else
-        break if first_comment_tk_class and not first_comment_tk_class === tk
-        first_comment_tk_class = tk.class
+        break if first_comment_tk_kind and not first_comment_tk_kind === tk[:kind]
+        first_comment_tk_kind = tk[:kind]
 
         first_line = false
-        comment << tk.text << "\n"
+        comment << tk[:text]
         tk = get_tk
 
-        if TkNL === tk then
+        if :on_nl === tk then
           skip_tkspace false
           tk = get_tk
         end
@@ -262,7 +278,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Consumes trailing whitespace from the token stream
 
   def consume_trailing_spaces # :nodoc:
-    get_tkread
     skip_tkspace false
   end
 
@@ -309,10 +324,9 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def get_bool
     skip_tkspace
     tk = get_tk
-    case tk
-    when TkTRUE
+    if :on_kw == tk[:kind] && 'true' == tk[:text]
       true
-    when TkFALSE, TkNIL
+    elsif :on_kw == tk[:kind] && ('false' == tk[:text] || 'nil' == tk[:text])
       false
     else
       unget_tk tk
@@ -331,24 +345,24 @@ class RDoc::Parser::Ruby < RDoc::Parser
     given_name = ''
 
     # class ::A -> A is in the top level
-    case name_t
-    when TkCOLON2, TkCOLON3 then # bug
+    if :on_op == name_t[:kind] and '::' == name_t[:text] then # bug
       name_t = get_tk
       container = @top_level
       given_name << '::'
     end
 
     skip_tkspace false
-    given_name << name_t.name
+    given_name << name_t[:text]
 
-    while TkCOLON2 === peek_tk do
+    is_self = name_t[:kind] == :on_op && name_t[:text] == '<<'
+    while !is_self && (tk = peek_tk) and :on_op == tk[:kind] and '::' == tk[:text] do
       prev_container = container
-      container = container.find_module_named name_t.name
+      container = container.find_module_named name_t[:text]
       container ||=
         if ignore_constants then
           RDoc::Context.new
         else
-          c = prev_container.add_module RDoc::NormalModule, name_t.name
+          c = prev_container.add_module RDoc::NormalModule, name_t[:text]
           c.ignore unless prev_container.document_children
           @top_level.add_to_classes_or_modules c
           c
@@ -359,7 +373,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
       get_tk
       skip_tkspace false
       name_t = get_tk
-      given_name << '::' << name_t.name
+      unless :on_const == name_t[:kind] || :on_ident == name_t[:kind]
+        raise RDoc::Error, "Invalid class or module definition: #{given_name}"
+      end
+      given_name << '::' << name_t[:text]
     end
 
     skip_tkspace false
@@ -371,9 +388,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Return a superclass, which can be either a constant of an expression
 
   def get_class_specification
-    case peek_tk
-    when TkSELF then return 'self'
-    when TkGVAR then return ''
+    tk = peek_tk
+    if tk.nil?
+      return ''
+    elsif :on_kw == tk[:kind] && 'self' == tk[:text]
+      return 'self'
+    elsif :on_gvar == tk[:kind]
+      return ''
     end
 
     res = get_constant
@@ -383,9 +404,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     get_tkread # empty out read buffer
 
     tk = get_tk
+    return res unless tk
 
-    case tk
-    when TkNL, TkCOMMENT, TkSEMICOLON then
+    case tk[:kind]
+    when :on_nl, :on_comment, :on_embdoc, :on_semicolon then
       unget_tk(tk)
       return res
     end
@@ -403,8 +425,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
     skip_tkspace false
     tk = get_tk
 
-    while TkCOLON2 === tk or TkCOLON3 === tk or TkCONSTANT === tk do
-      res += tk.name
+    while tk && ((:on_op == tk[:kind] && '::' == tk[:text]) || :on_const == tk[:kind]) do
+      res += tk[:text]
       tk = get_tk
     end
 
@@ -420,7 +442,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     nest = 0
 
-    while TkLPAREN === (tk = peek_tk) or TkfLPAREN === tk do
+    while :on_lparen == (tk = peek_tk)[:kind] do
       get_tk
       skip_tkspace
       nest += 1
@@ -431,7 +453,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     while nest > 0
       skip_tkspace
       tk = get_tk
-      nest -= 1 if TkRPAREN === tk
+      nest -= 1 if :on_rparen == tk[:kind]
     end
 
     name
@@ -446,13 +468,19 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # won't catch all cases (such as "a = yield + 1"
 
   def get_end_token tk # :nodoc:
-    case tk
-    when TkLPAREN, TkfLPAREN
-      TkRPAREN
-    when TkRPAREN
+    case tk[:kind]
+    when :on_lparen
+      {
+        :kind => :on_rparen,
+        :text => ')'
+      }
+    when :on_rparen
       nil
     else
-      TkNL
+      {
+        :kind => :on_nl,
+        :text => "\n"
+      }
     end
   end
 
@@ -461,11 +489,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def get_method_container container, name_t # :nodoc:
     prev_container = container
-    container = container.find_module_named(name_t.name)
+    container = container.find_module_named(name_t[:text])
 
     unless container then
       constant = prev_container.constants.find do |const|
-        const.name == name_t.name
+        const.name == name_t[:text]
       end
 
       if constant then
@@ -476,21 +504,21 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     unless container then
       # TODO seems broken, should starting at Object in @store
-      obj = name_t.name.split("::").inject(Object) do |state, item|
+      obj = name_t[:text].split("::").inject(Object) do |state, item|
         state.const_get(item)
       end rescue nil
 
       type = obj.class == Class ? RDoc::NormalClass : RDoc::NormalModule
 
       unless [Class, Module].include?(obj.class) then
-        warn("Couldn't find #{name_t.name}. Assuming it's a module")
+        warn("Couldn't find #{name_t[:text]}. Assuming it's a module")
       end
 
       if type == RDoc::NormalClass then
         sclass = obj.superclass ? obj.superclass.name : nil
-        container = prev_container.add_class type, name_t.name, sclass
+        container = prev_container.add_class type, name_t[:text], sclass
       else
-        container = prev_container.add_module type, name_t.name
+        container = prev_container.add_module type, name_t[:text]
       end
 
       record_location container
@@ -504,32 +532,26 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def get_symbol_or_name
     tk = get_tk
-    case tk
-    when TkSYMBOL then
-      text = tk.text.sub(/^:/, '')
+    case tk[:kind]
+    when :on_symbol then
+      text = tk[:text].sub(/^:/, '')
 
-      if TkASSIGN === peek_tk then
+      next_tk = peek_tk
+      if next_tk && :on_op == next_tk[:kind] && '=' == next_tk[:text] then
         get_tk
         text << '='
       end
 
       text
-    when TkId, TkOp then
-      tk.name
-    when TkAMPER,
-         TkDSTRING,
-         TkSTAR,
-         TkSTRING then
-      tk.text
+    when :on_ident, :on_const, :on_gvar, :on_cvar, :on_ivar, :on_op, :on_kw then
+      tk[:text]
+    when :on_tstring, :on_dstring then
+      tk[:text][1..-2]
     else
       raise RDoc::Error, "Name or symbol expected (got #{tk})"
     end
   end
 
-  def stop_at_EXPR_END # :nodoc:
-    @scanner.lex_state == :EXPR_END || !@scanner.continue
-  end
-
   ##
   # Marks containers between +container+ and +ancestor+ as ignored
 
@@ -570,7 +592,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def make_message message
     prefix = "#{@file_name}:"
 
-    prefix << "#{@scanner.line_no}:#{@scanner.char_no}:" if @scanner
+    tk = peek_tk
+    prefix << "#{tk[:line_no]}:#{tk[:char_no]}:" if tk
 
     "#{prefix} #{message}"
   end
@@ -589,7 +612,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # +comment+.
 
   def parse_attr(context, single, tk, comment)
-    line_no = tk.line_no
+    line_no = tk[:line_no]
 
     args = parse_symbol_arg 1
     if args.size > 0 then
@@ -598,7 +621,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       skip_tkspace false
       tk = get_tk
 
-      if TkCOMMA === tk then
+      if :on_comma == tk[:kind] then
         rw = "RW" if get_bool
       else
         unget_tk tk
@@ -618,7 +641,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # comment for each to +comment+.
 
   def parse_attr_accessor(context, single, tk, comment)
-    line_no = tk.line_no
+    line_no = tk[:line_no]
 
     args = parse_symbol_arg
     rw = "?"
@@ -629,7 +652,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     # and add found items appropriately but here we do not.  I'm not sure why.
     return if @track_visibility and not tmp.document_self
 
-    case tk.name
+    case tk[:text]
     when "attr_reader"   then rw = "R"
     when "attr_writer"   then rw = "W"
     when "attr_accessor" then rw = "RW"
@@ -647,21 +670,19 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses an +alias+ in +context+ with +comment+
 
   def parse_alias(context, single, tk, comment)
-    line_no = tk.line_no
+    line_no = tk[:line_no]
 
     skip_tkspace
 
-    if TkLPAREN === peek_tk then
+    if :on_lparen === peek_tk[:kind] then
       get_tk
       skip_tkspace
     end
 
     new_name = get_symbol_or_name
 
-    @scanner.lex_state = :EXPR_FNAME
-
     skip_tkspace
-    if TkCOMMA === peek_tk then
+    if :on_comma === peek_tk[:kind] then
       get_tk
       skip_tkspace
     end
@@ -688,34 +709,38 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Extracts call parameters from the token stream.
 
   def parse_call_parameters(tk)
-    end_token = case tk
-                when TkLPAREN, TkfLPAREN
-                  TkRPAREN
-                when TkRPAREN
+    end_token = case tk[:kind]
+                when :on_lparen
+                  :on_rparen
+                when :on_rparen
                   return ""
                 else
-                  TkNL
+                  :on_nl
                 end
     nest = 0
 
     loop do
-      case tk
-      when TkSEMICOLON
+      break if tk.nil?
+      case tk[:kind]
+      when :on_semicolon
         break
-      when TkLPAREN, TkfLPAREN
+      when :on_lparen
         nest += 1
       when end_token
-        if end_token == TkRPAREN
+        if end_token == :on_rparen
           nest -= 1
-          break if @scanner.lex_state == :EXPR_END and nest <= 0
+          break if RDoc::RipperStateLex.end?(tk) and nest <= 0
         else
-          break unless @scanner.continue
+          break if RDoc::RipperStateLex.end?(tk)
         end
-      when TkCOMMENT, TkASSIGN, TkOPASGN
+      when :on_comment, :on_embdoc
         unget_tk(tk)
         break
-      when nil then
-        break
+      when :on_op
+        if tk[:text] =~ /^(.{1,2})?=$/
+          unget_tk(tk)
+          break
+        end
       end
       tk = get_tk
     end
@@ -727,28 +752,27 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a class in +context+ with +comment+
 
   def parse_class container, single, tk, comment
-    line_no = tk.line_no
+    line_no = tk[:line_no]
 
     declaration_context = container
     container, name_t, given_name = get_class_or_module container
 
-    cls =
-      case name_t
-      when TkCONSTANT
-        parse_class_regular container, declaration_context, single,
-          name_t, given_name, comment
-      when TkLSHFT
-        case name = get_class_specification
-        when 'self', container.name
-          parse_statements container, SINGLE
-          return # don't update line
-        else
-          parse_class_singleton container, name, comment
-        end
+    if name_t[:kind] == :on_const
+      cls = parse_class_regular container, declaration_context, single,
+        name_t, given_name, comment
+    elsif name_t[:kind] == :on_op && name_t[:text] == '<<'
+      case name = get_class_specification
+      when 'self', container.name
+        read_documentation_modifiers cls, RDoc::CLASS_MODIFIERS
+        parse_statements container, SINGLE
+        return # don't update line
       else
-        warn "Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}"
-        return
+        cls = parse_class_singleton container, name, comment
       end
+    else
+      warn "Expected class name or '<<'. Got #{name_t[:kind]}: #{name_t[:text].inspect}"
+      return
+    end
 
     cls.line   = line_no
 
@@ -770,7 +794,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
       given_name = $'
     end
 
-    if TkLT === peek_tk then
+    tk = peek_tk
+    if tk[:kind] == :on_op && tk[:text] == '<' then
       get_tk
       skip_tkspace
       superclass = get_class_specification
@@ -840,40 +865,52 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # true, no found constants will be added to RDoc.
 
   def parse_constant container, tk, comment, ignore_constants = false
-    line_no = tk.line_no
+    line_no = tk[:line_no]
 
-    name = tk.name
+    name = tk[:text]
     skip_tkspace false
 
     return unless name =~ /^\w+$/
 
     eq_tk = get_tk
 
-    if TkCOLON2 === eq_tk then
+    if :on_op == eq_tk[:kind] && '::' == eq_tk[:text] then
       unget_tk eq_tk
       unget_tk tk
 
       container, name_t, = get_class_or_module container, ignore_constants
 
-      name = name_t.name
+      name = name_t[:text]
 
       eq_tk = get_tk
     end
 
-    unless TkASSIGN === eq_tk then
-      unget_tk eq_tk
-      return false
+    is_array_or_hash = false
+    if eq_tk && :on_lbracket == eq_tk[:kind]
+      nest = 1
+      while bracket_tk = get_tk
+        case bracket_tk[:kind]
+        when :on_lbracket
+          nest += 1
+        when :on_rbracket
+          nest -= 1
+          break if nest == 0
+        end
+      end
+      skip_tkspace false
+      eq_tk = get_tk
+      is_array_or_hash = true
     end
 
-    if TkGT === peek_tk then
+    unless eq_tk && :on_op == eq_tk[:kind] && '=' == eq_tk[:text] then
       unget_tk eq_tk
-      return
+      return false
     end
 
     value = ''
     con = RDoc::Constant.new name, value, comment
 
-    body = parse_constant_body container, con
+    body = parse_constant_body container, con, is_array_or_hash
 
     return unless body
 
@@ -882,13 +919,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
     con.line   = line_no
     read_documentation_modifiers con, RDoc::CONSTANT_MODIFIERS
 
+    return if is_array_or_hash
+
     @stats.add_constant con
     container.add_constant con
 
     true
   end
 
-  def parse_constant_body container, constant # :nodoc:
+  def parse_constant_body container, constant, is_array_or_hash # :nodoc:
     nest     = 0
     rhs_name = ''
 
@@ -896,44 +935,51 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     tk = get_tk
 
+    body = nil
     loop do
-      case tk
-      when TkSEMICOLON then
+      break if tk.nil?
+      if :on_semicolon == tk[:kind] then
         break if nest <= 0
-      when TkLPAREN, TkfLPAREN, TkLBRACE, TkfLBRACE, TkLBRACK, TkfLBRACK,
-           TkDO, TkIF, TkUNLESS, TkCASE, TkDEF, TkBEGIN then
+      elsif [:on_tlambeg, :on_lparen, :on_lbrace, :on_lbracket].include?(tk[:kind]) then
+        nest += 1
+      elsif (:on_kw == tk[:kind] && 'def' == tk[:text]) then
         nest += 1
-      when TkRPAREN, TkRBRACE, TkRBRACK, TkEND then
+      elsif (:on_kw == tk[:kind] && %w{do if unless case begin}.include?(tk[:text])) then
+        if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0
+          nest += 1
+        end
+      elsif [:on_rparen, :on_rbrace, :on_rbracket].include?(tk[:kind]) ||
+            (:on_kw == tk[:kind] && 'end' == tk[:text]) then
         nest -= 1
-      when TkCOMMENT then
-        if nest <= 0 and stop_at_EXPR_END then
-          unget_tk tk
+      elsif (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
+        unget_tk tk
+        if nest <= 0 and RDoc::RipperStateLex.end?(tk) then
+          body = get_tkread_clean(/^[ \t]+/, '')
+          read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
           break
         else
-          unget_tk tk
           read_documentation_modifiers constant, RDoc::CONSTANT_MODIFIERS
         end
-      when TkCONSTANT then
-        rhs_name << tk.name
+      elsif :on_const == tk[:kind] then
+        rhs_name << tk[:text]
 
-        if nest <= 0 and TkNL === peek_tk then
-          create_module_alias container, constant, rhs_name
+        next_tk = peek_tk
+        if nest <= 0 and (next_tk.nil? || :on_nl == next_tk[:kind]) then
+          create_module_alias container, constant, rhs_name unless is_array_or_hash
           break
         end
-      when TkNL then
-        if nest <= 0 and stop_at_EXPR_END then
+      elsif :on_nl == tk[:kind] then
+        if nest <= 0 and RDoc::RipperStateLex.end?(tk) then
           unget_tk tk
           break
         end
-      when TkCOLON2, TkCOLON3 then
+      elsif :on_op == tk[:kind] && '::' == tk[:text]
         rhs_name << '::'
-      when nil then
-        break
       end
       tk = get_tk
     end
 
-    get_tkread_clean(/^[ \t]+/, '')
+    body ? body : get_tkread_clean(/^[ \t]+/, '')
   end
 
   ##
@@ -942,8 +988,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_comment container, tk, comment
     return parse_comment_tomdoc container, tk, comment if @markup == 'tomdoc'
-    column  = tk.char_no
-    line_no = tk.line_no
+    column  = tk[:char_no]
+    line_no = tk[:line_no]
 
     text = comment.text
 
@@ -988,12 +1034,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
     record_location meth
 
     meth.start_collecting_tokens
-    indent = TkSPACE.new 0, 1, 1
-    indent.set_text " " * column
-
-    position_comment = TkCOMMENT.new 0, line_no, 1
-    position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
-    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+    indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+    position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+    position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+    newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+    meth.add_tokens [position_comment, newline, indent]
 
     meth.params =
       if text.sub!(/^#\s+:?args?:\s*(.*?)\s*$/i, '') then
@@ -1022,8 +1067,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_comment_tomdoc container, tk, comment
     return unless signature = RDoc::TomDoc.signature(comment)
-    column  = tk.char_no
-    line_no = tk.line_no
+    column  = tk[:char_no]
+    line_no = tk[:line_no]
 
     name, = signature.split %r%[ \(]%, 2
 
@@ -1032,12 +1077,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.line      = line_no
 
     meth.start_collecting_tokens
-    indent = TkSPACE.new 0, 1, 1
-    indent.set_text " " * column
-
-    position_comment = TkCOMMENT.new 0, line_no, 1
-    position_comment.set_text "# File #{@top_level.relative_name}, line #{line_no}"
-    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+    indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+    position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+    position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+    newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+    meth.add_tokens [position_comment, newline, indent]
 
     meth.call_seq = signature
 
@@ -1067,7 +1111,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
         record_location obj
       end
 
-      return unless TkCOMMA === peek_tk
+      return if peek_tk.nil? || :on_comma != peek_tk[:kind]
 
       get_tk
     end
@@ -1079,11 +1123,14 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Returns true if the comment was not consumed.
 
   def parse_identifier container, single, tk, comment # :nodoc:
-    case tk.name
+    case tk[:text]
     when 'private', 'protected', 'public', 'private_class_method',
          'public_class_method', 'module_function' then
       parse_visibility container, single, tk
       return true
+    when 'private_constant', 'public_constant'
+      parse_constant_visibility container, single, tk
+      return true
     when 'attr' then
       parse_attr container, single, tk, comment
     when /^attr_(reader|writer|accessor)$/ then
@@ -1172,8 +1219,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Parses a meta-programmed method
 
   def parse_meta_method(container, single, tk, comment)
-    column  = tk.char_no
-    line_no = tk.line_no
+    column  = tk[:char_no]
+    line_no = tk[:line_no]
 
     start_collecting_tokens
     add_token tk
@@ -1195,12 +1242,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
     remove_token_listener self
 
     meth.start_collecting_tokens
-    indent = TkSPACE.new 0, 1, 1
-    indent.set_text " " * column
-
-    position_comment = TkCOMMENT.new 0, line_no, 1
-    position_comment.value = "# File #{@top_level.relative_name}, line #{line_no}"
-    meth.add_tokens [position_comment, NEWLINE_TOKEN, indent]
+    indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+    position_comment = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+    position_comment[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+    newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+    meth.add_tokens [position_comment, newline, indent]
     meth.add_tokens @token_stream
 
     parse_meta_method_params container, single, meth, tk, comment
@@ -1224,17 +1270,16 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     name_t = get_tk
 
-    case name_t
-    when TkSYMBOL then
-      name_t.text[1..-1]
-    when TkSTRING then
-      name_t.value[1..-2]
-    when TkASSIGN then # ignore
+    if :on_symbol == name_t[:kind] then
+      name_t[:text][1..-1]
+    elsif :on_tstring == name_t[:kind] then
+      name_t[:text][1..-2]
+    elsif :on_op == name_t[:kind] && '=' == name_t[:text] then # ignore
       remove_token_listener self
 
       nil
     else
-      warn "unknown name token #{name_t.inspect} for meta-method '#{tk.name}'"
+      warn "unknown name token #{name_t.inspect} for meta-method '#{tk[:text]}'"
       'unknown'
     end
   end
@@ -1254,14 +1299,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
       last_tk = tk
 
       while tk = get_tk do
-        case tk
-        when TkSEMICOLON then
+        if :on_semicolon == tk[:kind] then
           break
-        when TkNL then
-          break unless last_tk and TkCOMMA === last_tk
-        when TkSPACE then
+        elsif :on_nl == tk[:kind] then
+          break unless last_tk and :on_comma == last_tk[:kind]
+        elsif :on_sp == tk[:kind] then
           # expression continues
-        when TkDO then
+        elsif :on_kw == tk[:kind] && 'do' == tk[:text] then
           parse_statements container, single, meth
           break
         else
@@ -1278,8 +1322,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
     singleton = nil
     added_container = false
     name = nil
-    column  = tk.char_no
-    line_no = tk.line_no
+    column  = tk[:char_no]
+    line_no = tk[:line_no]
 
     start_collecting_tokens
     add_token tk
@@ -1299,12 +1343,11 @@ class RDoc::Parser::Ruby < RDoc::Parser
     meth.line   = line_no
 
     meth.start_collecting_tokens
-    indent = TkSPACE.new 0, 1, 1
-    indent.set_text " " * column
-
-    token = TkCOMMENT.new 0, line_no, 1
-    token.set_text "# File #{@top_level.relative_name}, line #{line_no}"
-    meth.add_tokens [token, NEWLINE_TOKEN, indent]
+    indent = { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => ' ' * column }
+    token = { :line_no => line_no, :char_no => 1, :kind => :on_comment }
+    token[:text] = "# File #{@top_level.relative_name}, line #{line_no}"
+    newline = { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" }
+    meth.add_tokens [token, newline, indent]
     meth.add_tokens @token_stream
 
     parse_method_params_and_body container, single, meth, added_container
@@ -1325,7 +1368,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
   def parse_method_params_and_body container, single, meth, added_container
     token_listener meth do
-      @scanner.continue = false
       parse_method_parameters meth
 
       if meth.document_self or not @track_visibility then
@@ -1368,15 +1410,13 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # it is a singleton or regular method.
 
   def parse_method_name container # :nodoc:
-    @scanner.lex_state = :EXPR_FNAME
-
     skip_tkspace
     name_t = get_tk
-    back_tk = skip_tkspace
+    back_tk = skip_tkspace(false)
     singleton = false
 
-    case dot = get_tk
-    when TkDOT, TkCOLON2 then
+    dot = get_tk
+    if dot[:kind] == :on_period || (dot[:kind] == :on_op && dot[:text] == '::') then
       singleton = true
 
       name, container = parse_method_name_singleton container, name_t
@@ -1397,16 +1437,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # is parsed from the token stream for a regular method.
 
   def parse_method_name_regular container, name_t # :nodoc:
-    case name_t
-    when TkSTAR, TkAMPER then
-      name_t.text
+    if :on_op == name_t[:kind] && (%w{* & [] []= <<}.include?(name_t[:text])) then
+      name_t[:text]
     else
-      unless name_t.respond_to? :name then
+      unless [:on_kw, :on_const, :on_ident].include?(name_t[:kind]) then
         warn "expected method name token, . or ::, got #{name_t.inspect}"
         skip_method container
         return
       end
-      name_t.name
+      name_t[:text]
     end
   end
 
@@ -1416,47 +1455,43 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # for a singleton method.
 
   def parse_method_name_singleton container, name_t # :nodoc:
-    @scanner.lex_state = :EXPR_FNAME
     skip_tkspace
     name_t2 = get_tk
 
-    name =
-      case name_t
-      when TkSELF, TkMOD then
-        case name_t2
-          # NOTE: work around '[' being consumed early and not being re-tokenized
-          # as a TkAREF
-        when TkfLBRACK then
-          get_tk
-          '[]'
-        else
-          name_t2.name
-        end
-      when TkCONSTANT then
-        name = name_t2.name
+    if (:on_kw == name_t[:kind] && 'self' == name_t[:text]) || (:on_op == name_t[:kind] && '%' == name_t[:text]) then
+      # NOTE: work around '[' being consumed early and not being re-tokenized
+      # as a TkAREF
+      if :on_lbracket == name_t2[:kind]
+        get_tk
+        name = '[]'
+      else
+        name = name_t2[:text]
+      end
+    elsif :on_const == name_t[:kind] then
+      name = name_t2[:text]
 
-        container = get_method_container container, name_t
+      container = get_method_container container, name_t
 
-        return unless container
+      return unless container
 
-        name
-      when TkIDENTIFIER, TkIVAR, TkGVAR then
-        parse_method_dummy container
+      name
+    elsif :on_ident == name_t[:kind] || :on_ivar == name_t[:kind] || :on_gvar == name_t[:kind] then
+      parse_method_dummy container
 
-        nil
-      when TkTRUE, TkFALSE, TkNIL then
-        klass_name = "#{name_t.name.capitalize}Class"
-        container = @store.find_class_named klass_name
-        container ||= @top_level.add_class RDoc::NormalClass, klass_name
+      name = nil
+    elsif (:on_kw == name_t[:kind]) && ('true' == name_t[:text] || 'false' == name_t[:text] || 'nil' == name_t[:text]) then
+      klass_name = "#{name_t[:text].capitalize}Class"
+      container = @store.find_class_named klass_name
+      container ||= @top_level.add_class RDoc::NormalClass, klass_name
 
-        name_t2.name
-      else
-        warn "unexpected method name token #{name_t.inspect}"
-        # break
-        skip_method container
+      name = name_t2[:text]
+    else
+      warn "unexpected method name token #{name_t.inspect}"
+      # break
+      skip_method container
 
-        nil
-      end
+      name = nil
+    end
 
     return name, container
   end
@@ -1472,43 +1507,49 @@ class RDoc::Parser::Ruby < RDoc::Parser
     return '' unless end_token
 
     nest = 0
+    continue = false
 
-    loop do
-      case tk
-      when TkSEMICOLON then
+    while tk != nil do
+      case tk[:kind]
+      when :on_semicolon then
         break if nest == 0
-      when TkLBRACE, TkfLBRACE then
+      when :on_lbrace then
         nest += 1
-      when TkRBRACE then
+      when :on_rbrace then
         nest -= 1
         if nest <= 0
           # we might have a.each { |i| yield i }
           unget_tk(tk) if nest < 0
           break
         end
-      when TkLPAREN, TkfLPAREN then
+      when :on_lparen then
         nest += 1
-      when end_token then
-        if end_token == TkRPAREN
+      when end_token[:kind] then
+        if end_token[:kind] == :on_rparen
           nest -= 1
           break if nest <= 0
         else
-          break unless @scanner.continue
+          break
         end
-      when TkRPAREN then
+      when :on_rparen then
         nest -= 1
-      when method && method.block_params.nil? && TkCOMMENT then
-        unget_tk tk
-        read_documentation_modifiers method, modifiers
-        @read.pop
-      when TkCOMMENT then
+      when :on_comment, :on_embdoc then
         @read.pop
-      when nil then
-        break
+        if :on_nl == end_token[:kind] and "\n" == tk[:text][-1] and
+          (!continue or (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) != 0) then
+          if method && method.block_params.nil? then
+            unget_tk tk
+            read_documentation_modifiers method, modifiers
+          end
+          break if !continue and nest <= 0
+        end
+      when :on_comma then
+        continue = true
+      when :on_ident then
+        continue = false if continue
       end
       tk = get_tk
     end
-    @scanner.first_in_method_statement = true
 
     get_tkread_clean(/\s+/, ' ')
   end
@@ -1539,7 +1580,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def parse_module container, single, tk, comment
     container, name_t, = get_class_or_module container
 
-    name = name_t.name
+    name = name_t[:text]
 
     mod = container.add_module RDoc::NormalModule, name
     mod.ignore unless container.document_children
@@ -1562,12 +1603,12 @@ class RDoc::Parser::Ruby < RDoc::Parser
     skip_tkspace_comment
     tk = get_tk
 
-    if TkLPAREN === tk then
+    if :on_lparen == tk[:kind] then
       skip_tkspace_comment
       tk = get_tk
     end
 
-    name = tk.text if TkSTRING === tk
+    name = tk[:text][1..-2] if :on_tstring == tk[:kind]
 
     if name then
       @top_level.add_require RDoc::Require.new(name, comment)
@@ -1582,14 +1623,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def parse_rescue
     skip_tkspace false
 
+    nest = 0
     while tk = get_tk
-      case tk
-      when TkNL, TkSEMICOLON then
+      case tk[:kind]
+      when :on_nl, :on_semicolon, :on_comment then
         break
-      when TkCOMMA then
+      when :on_comma then
         skip_tkspace false
 
-        get_tk if TkNL === peek_tk
+        get_tk if :on_nl == peek_tk[:kind]
       end
 
       skip_tkspace false
@@ -1613,14 +1655,32 @@ class RDoc::Parser::Ruby < RDoc::Parser
       keep_comment = false
       try_parse_comment = false
 
-      non_comment_seen = true unless TkCOMMENT === tk
+      non_comment_seen = true unless (:on_comment == tk[:kind] or :on_embdoc == tk[:kind])
 
-      case tk
-      when TkNL then
-        skip_tkspace
-        tk = get_tk
+      case tk[:kind]
+      when :on_nl, :on_ignored_nl, :on_comment, :on_embdoc then
+        if :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
+          skip_tkspace
+          tk = get_tk
+        else
+          past_tokens = @read.size > 1 ? @read[0..-2] : []
+          nl_position = 0
+          past_tokens.reverse.each_with_index do |read_tk, i|
+            if read_tk =~ /^\n$/ then
+              nl_position = (past_tokens.size - 1) - i
+              break
+            elsif read_tk =~ /^#.*\n$/ then
+              nl_position = ((past_tokens.size - 1) - i) + 1
+              break
+            end
+          end
+          comment_only_line = past_tokens[nl_position..-1].all?{ |c| c =~ /^\s+$/ }
+          unless comment_only_line then
+            tk = get_tk
+          end
+        end
 
-        if TkCOMMENT === tk then
+        if tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) then
           if non_comment_seen then
             # Look for RDoc in a comment about to be thrown away
             non_comment_seen = parse_comment container, tk, comment unless
@@ -1630,15 +1690,14 @@ class RDoc::Parser::Ruby < RDoc::Parser
             comment.force_encoding @encoding if @encoding
           end
 
-          while TkCOMMENT === tk do
-            comment << tk.text << "\n"
-
-            tk = get_tk
+          while tk and (:on_comment == tk[:kind] or :on_embdoc == tk[:kind]) do
+            comment << tk[:text]
+            comment << "\n" unless "\n" == tk[:text].chars.to_a.last
 
-            if TkNL === tk then
+            if tk[:text].size > 1 && "\n" == tk[:text].chars.to_a.last then
               skip_tkspace false # leading spaces
-              tk = get_tk
             end
+            tk = get_tk
           end
 
           comment = new_comment comment
@@ -1661,59 +1720,76 @@ class RDoc::Parser::Ruby < RDoc::Parser
         keep_comment = true
         container.current_line_visibility = nil
 
-      when TkCLASS then
-        parse_class container, single, tk, comment
+      when :on_kw then
+        case tk[:text]
+        when 'class' then
+          parse_class container, single, tk, comment
 
-      when TkMODULE then
-        parse_module container, single, tk, comment
+        when 'module' then
+          parse_module container, single, tk, comment
 
-      when TkDEF then
-        parse_method container, single, tk, comment
+        when 'def' then
+          parse_method container, single, tk, comment
 
-      when TkCONSTANT then
-        unless parse_constant container, tk, comment, current_method then
-          try_parse_comment = true
-        end
+        when 'alias' then
+          parse_alias container, single, tk, comment unless current_method
 
-      when TkALIAS then
-        parse_alias container, single, tk, comment unless current_method
+        when 'yield' then
+          if current_method.nil? then
+            warn "Warning: yield outside of method" if container.document_self
+          else
+            parse_yield container, single, tk, current_method
+          end
 
-      when TkYIELD then
-        if current_method.nil? then
-          warn "Warning: yield outside of method" if container.document_self
-        else
-          parse_yield container, single, tk, current_method
-        end
+        when 'until', 'while' then
+          if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0
+            nest += 1
+            skip_optional_do_after_expression
+          end
 
-      # Until and While can have a 'do', which shouldn't increase the nesting.
-      # We can't solve the general case, but we can handle most occurrences by
-      # ignoring a do at the end of a line.
+        # Until and While can have a 'do', which shouldn't increase the nesting.
+        # We can't solve the general case, but we can handle most occurrences by
+        # ignoring a do at the end of a line.
 
-      when TkUNTIL, TkWHILE then
-        nest += 1
-        skip_optional_do_after_expression
+        # 'for' is trickier
+        when 'for' then
+          nest += 1
+          skip_for_variable
+          skip_optional_do_after_expression
 
-      # 'for' is trickier
-      when TkFOR then
-        nest += 1
-        skip_for_variable
-        skip_optional_do_after_expression
+        when 'case', 'do', 'if', 'unless', 'begin' then
+          if (RDoc::RipperStateLex::EXPR_LABEL & tk[:state]) == 0
+            nest += 1
+          end
 
-      when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN then
-        nest += 1
+        when 'super' then
+          current_method.calls_super = true if current_method
 
-      when TkSUPER then
-        current_method.calls_super = true if current_method
+        when 'rescue' then
+          parse_rescue
 
-      when TkRESCUE then
-        parse_rescue
+        when 'end' then
+          nest -= 1
+          if nest == 0 then
+            container.ongoing_visibility = save_visibility
+
+            parse_comment container, tk, comment unless comment.empty?
+
+            return
+          end
+        end
+
+      when :on_const then
+        unless parse_constant container, tk, comment, current_method then
+          try_parse_comment = true
+        end
 
-      when TkIDENTIFIER then
+      when :on_ident then
         if nest == 1 and current_method.nil? then
           keep_comment = parse_identifier container, single, tk, comment
         end
 
-        case tk.name
+        case tk[:text]
         when "require" then
           parse_require container, comment
         when "include" then
@@ -1722,15 +1798,6 @@ class RDoc::Parser::Ruby < RDoc::Parser
           parse_extend_or_include RDoc::Extend, container, comment
         end
 
-      when TkEND then
-        nest -= 1
-        if nest == 0 then
-          container.ongoing_visibility = save_visibility
-
-          parse_comment container, tk, comment unless comment.empty?
-
-          return
-        end
       else
         try_parse_comment = nest == 1
       end
@@ -1762,8 +1829,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def parse_symbol_arg(no = nil)
     skip_tkspace_comment
 
-    case tk = get_tk
-    when TkLPAREN
+    tk = get_tk
+    if tk[:kind] == :on_lparen
       parse_symbol_arg_paren no
     else
       parse_symbol_arg_space no, tk
@@ -1785,10 +1852,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
 
       skip_tkspace_comment
-      case tk2 = get_tk
-      when TkRPAREN
+      case (tk2 = get_tk)[:kind]
+      when :on_rparen
         break
-      when TkCOMMA
+      when :on_comma
       else
         warn("unexpected token: '#{tk2.inspect}'") if $DEBUG_RDOC
         break
@@ -1815,7 +1882,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       skip_tkspace false
 
       tk1 = get_tk
-      unless TkCOMMA === tk1 then
+      if tk1.nil? || :on_comma != tk1[:kind] then
         unget_tk tk1
         break
       end
@@ -1834,12 +1901,12 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # Returns symbol text from the next token
 
   def parse_symbol_in_arg
-    case tk = get_tk
-    when TkSYMBOL
-      tk.text.sub(/^:/, '')
-    when TkSTRING
-      eval @read[-1]
-    when TkDSTRING, TkIDENTIFIER then
+    tk = get_tk
+    if :on_symbol == tk[:kind] then
+      tk[:text].sub(/^:/, '')
+    elsif :on_tstring == tk[:kind] then
+      tk[:text][1..-2]
+    elsif :on_dstring == tk[:kind] or :on_ident == tk[:kind] then
       nil # ignore
     else
       warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG_RDOC
@@ -1873,15 +1940,15 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     skip_tkspace_comment false
 
-    case peek_tk
-      # Ryan Davis suggested the extension to ignore modifiers, because he
-      # often writes
-      #
-      #   protected unless $TESTING
-      #
-    when TkNL, TkUNLESS_MOD, TkIF_MOD, TkSEMICOLON then
+    ptk = peek_tk
+    # Ryan Davis suggested the extension to ignore modifiers, because he
+    # often writes
+    #
+    #   protected unless $TESTING
+    #
+    if [:on_nl, :on_semicolon].include?(ptk[:kind]) || (:on_kw == ptk[:kind] && (['if', 'unless'].include?(ptk[:text]))) then
       container.ongoing_visibility = vis
-    when TkDEF
+    elsif :on_kw == ptk[:kind] && 'def' == ptk[:text]
       container.current_line_visibility = vis
     else
       update_visibility container, vis_type, vis, singleton
@@ -1889,13 +1956,28 @@ class RDoc::Parser::Ruby < RDoc::Parser
   end
 
   ##
+  # Parses a Module#private_constant or Module#public_constant call from +tk+.
+
+  def parse_constant_visibility(container, single, tk)
+    args = parse_symbol_arg
+    case tk[:text]
+    when 'private_constant'
+      vis = :private
+    when 'public_constant'
+      vis = :public
+    else
+      raise RDoc::Error, 'Unreachable'
+    end
+    container.set_constant_visibility_for args, vis
+  end
+
+  ##
   # Determines the block parameter for +context+
 
   def parse_yield(context, single, tk, method)
     return if method.block_params
 
     get_tkread
-    @scanner.continue = false
     method.block_params = parse_method_or_yield_parameters
   end
 
@@ -1919,11 +2001,10 @@ class RDoc::Parser::Ruby < RDoc::Parser
     while tk = get_tk do
       tokens << tk
 
-      case tk
-      when TkNL, TkDEF then
+      if :on_nl == tk[:kind] or (:on_kw == tk[:kind] && 'def' == tk[:text]) then
         return
-      when TkCOMMENT then
-        return unless tk.text =~ /\s*:?([\w-]+):\s*(.*)/
+      elsif :on_comment == tk[:kind] or :on_embdoc == tk[:kind] then
+        return unless tk[:text] =~ /\s*:?([\w-]+):\s*(.*)/
 
         directive = $1.downcase
 
@@ -1933,7 +2014,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
       end
     end
   ensure
-    unless tokens.length == 1 and TkCOMMENT === tokens.first then
+    unless tokens.length == 1 and (:on_comment == tokens.first[:kind] or :on_embdoc == tokens.first[:kind]) then
       tokens.reverse_each do |token|
         unget_tk token
       end
@@ -1947,6 +2028,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
   # See also RDoc::Markup::PreProcess#handle_directive
 
   def read_documentation_modifiers context, allowed
+    skip_tkspace(false)
     directive, value = read_directive allowed
 
     return unless directive
@@ -1995,27 +2077,25 @@ class RDoc::Parser::Ruby < RDoc::Parser
       rescue StandardError => e
         bytes = ''
 
-        20.times do @scanner.ungetc end
-        count = 0
-        60.times do |i|
-          count = i
-          byte = @scanner.getc
-          break unless byte
-          bytes << byte
+        if @scanner_point >= @scanner.size
+          now_line_no = @scanner[@scanner.size - 1][:line_no]
+        else
+          now_line_no = peek_tk[:line_no]
         end
-        count -= 20
-        count.times do @scanner.ungetc end
 
         $stderr.puts <<-EOF
 
-#{self.class} failure around line #{@scanner.line_no} of
+#{self.class} failure around line #{now_line_no} of
 #{@file_name}
 
         EOF
 
         unless bytes.empty? then
           $stderr.puts
-          $stderr.puts bytes.inspect
+          now_line_no = peek_tk[:line_no]
+          start_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no }
+          end_index = @scanner.find_index { |tk| tk[:line_no] == now_line_no + 1 } - 1
+          $stderr.puts @scanner[start_index..end_index].join
         end
 
         raise e
@@ -2034,31 +2114,36 @@ class RDoc::Parser::Ruby < RDoc::Parser
 
     b_nest = 0
     nest = 0
-    @scanner.continue = false
 
     loop do
-      case tk
-      when TkSEMICOLON, TkNL then
+      break unless tk
+      case tk[:kind]
+      when :on_semicolon, :on_nl, :on_ignored_nl then
         break if b_nest.zero?
-      when TkLPAREN, TkfLPAREN then
+      when :on_lparen then
         nest += 1
-      when TkRPAREN then
+      when :on_rparen then
         nest -= 1
-      when TkBEGIN then
-        b_nest += 1
-      when TkEND then
-        b_nest -= 1
-      when TkDO
-        break if nest.zero?
-      when nil then
-        break
+      when :on_kw then
+        case tk[:text]
+        when 'begin'
+          b_nest += 1
+        when 'end'
+          b_nest -= 1
+        when 'do'
+          break if nest.zero?
+        end
+      when :on_comment, :on_embdoc then
+        if b_nest.zero? and "\n" == tk[:text][-1] then
+          break
+        end
       end
       tk = get_tk
     end
 
     skip_tkspace false
 
-    get_tk if TkDO === peek_tk
+    get_tk if peek_tk && :on_kw == peek_tk[:kind] && 'do' == peek_tk[:text]
   end
 
   ##
@@ -2069,7 +2154,7 @@ class RDoc::Parser::Ruby < RDoc::Parser
     get_tk
     skip_tkspace false
     tk = get_tk
-    unget_tk(tk) unless TkIN === tk
+    unget_tk(tk) unless :on_kw == tk[:kind] and 'in' == tk[:text]
   end
 
   ##
@@ -2087,7 +2172,8 @@ class RDoc::Parser::Ruby < RDoc::Parser
   def skip_tkspace_comment(skip_nl = true)
     loop do
       skip_tkspace skip_nl
-      return unless TkCOMMENT === peek_tk
+      next_tk = peek_tk
+      return if next_tk.nil? || (:on_comment != next_tk[:kind] and :on_embdoc != next_tk[:kind])
       get_tk
     end
   end
diff --git a/lib/rdoc/parser/ruby_tools.rb b/lib/rdoc/parser/ruby_tools.rb
index e35b420f30..0a566827a5 100644
--- a/lib/rdoc/parser/ruby_tools.rb
+++ b/lib/rdoc/parser/ruby_tools.rb
@@ -5,8 +5,6 @@
 
 module RDoc::Parser::RubyTools
 
-  include RDoc::RubyToken
-
   ##
   # Adds a token listener +obj+, but you should probably use token_listener
 
@@ -22,16 +20,25 @@ module RDoc::Parser::RubyTools
     tk = nil
 
     if @tokens.empty? then
-      tk = @scanner.token
-      @read.push @scanner.get_readed
-      puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+      if @scanner_point >= @scanner.size
+        return nil
+      else
+        tk = @scanner[@scanner_point]
+        @scanner_point += 1
+        @read.push tk[:text]
+        puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
+      end
     else
       @read.push @unget_read.shift
       tk = @tokens.shift
       puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
     end
 
-    tk = nil if TkEND_OF_SCRIPT === tk
+    if tk == nil || :on___end__ == tk[:kind]
+      tk = nil
+    end
+
+    return nil unless tk
 
     # inform any listeners of our shiny new token
     @token_listeners.each do |obj|
@@ -102,19 +109,24 @@ module RDoc::Parser::RubyTools
     @tokens     = []
     @unget_read = []
     @nest = 0
+    @scanner_point = 0
+  end
+
+  def tk_nl?(tk)
+    :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind]
   end
 
   ##
   # Skips whitespace tokens including newlines if +skip_nl+ is true
 
-  def skip_tkspace(skip_nl = true) # HACK dup
+  def skip_tkspace(skip_nl = true)
     tokens = []
 
-    while TkSPACE === (tk = get_tk) or (skip_nl and TkNL === tk) do
-      tokens.push tk
+    while (tk = get_tk) and (:on_sp == tk[:kind] or (skip_nl and tk_nl?(tk))) do
+      tokens.push(tk)
     end
 
-    unget_tk tk
+    unget_tk(tk)
     tokens
   end
 
diff --git a/lib/rdoc/rd/block_parser.rb b/lib/rdoc/rd/block_parser.rb
index 3f4941168f..8ac7cad996 100644
--- a/lib/rdoc/rd/block_parser.rb
+++ b/lib/rdoc/rd/block_parser.rb
@@ -677,54 +677,54 @@ Racc_debug_parser = false
 # reduce 0 omitted
 
 def _reduce_1(val, _values, result)
- result = RDoc::Markup::Document.new(*val[0])
+ result = RDoc::Markup::Document.new(*val[0]) 
     result
 end
 
 def _reduce_2(val, _values, result)
- raise ParseError, "file empty"
+ raise ParseError, "file empty" 
     result
 end
 
 def _reduce_3(val, _values, result)
- result = val[0].concat val[1]
+ result = val[0].concat val[1] 
     result
 end
 
 def _reduce_4(val, _values, result)
- result = val[0]
+ result = val[0] 
     result
 end
 
 def _reduce_5(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_6(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 7 omitted
 
 def _reduce_8(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_9(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_10(val, _values, result)
- result = [RDoc::Markup::BlankLine.new]
+ result = [RDoc::Markup::BlankLine.new] 
     result
 end
 
 def _reduce_11(val, _values, result)
- result = val[0].parts
+ result = val[0].parts 
     result
 end
 
@@ -732,30 +732,30 @@ def _reduce_12(val, _values, result)
       # val[0] is like [level, title]
       title = @inline_parser.parse(val[0][1])
       result = RDoc::Markup::Heading.new(val[0][0], title)
-
+    
     result
 end
 
 def _reduce_13(val, _values, result)
       result = RDoc::Markup::Include.new val[0], @include_path
-
+    
     result
 end
 
 def _reduce_14(val, _values, result)
       # val[0] is Array of String
       result = paragraph val[0]
-
+    
     result
 end
 
 def _reduce_15(val, _values, result)
- result << val[1].rstrip
+ result << val[1].rstrip 
     result
 end
 
 def _reduce_16(val, _values, result)
- result = [val[0].rstrip]
+ result = [val[0].rstrip] 
     result
 end
 
@@ -766,7 +766,7 @@ def _reduce_17(val, _values, result)
 
       # imform to lexer.
       @in_verbatim = false
-
+    
     result
 end
 
@@ -777,25 +777,25 @@ def _reduce_18(val, _values, result)
 
       # imform to lexer.
       @in_verbatim = false
-
+    
     result
 end
 
 def _reduce_19(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
 def _reduce_20(val, _values, result)
       result.concat val[2]
-
+    
     result
 end
 
 def _reduce_21(val, _values, result)
       result << "\n"
-
+    
     result
 end
 
@@ -803,7 +803,7 @@ def _reduce_22(val, _values, result)
       result = val
       # inform to lexer.
       @in_verbatim = true
-
+    
     result
 end
 
@@ -817,89 +817,89 @@ end
 
 def _reduce_27(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_28(val, _values, result)
       result = val[1]
-
+    
     result
 end
 
 def _reduce_29(val, _values, result)
       result = val[1].push(val[2])
-
+    
     result
 end
 
 def _reduce_30(val, _values, result)
- result = val[0] << val[1]
+ result = val[0] << val[1] 
     result
 end
 
 def _reduce_31(val, _values, result)
- result = [val[0]]
+ result = [val[0]] 
     result
 end
 
 def _reduce_32(val, _values, result)
       result = RDoc::Markup::List.new :BULLET, *val[0]
-
+    
     result
 end
 
 def _reduce_33(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_34(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_35(val, _values, result)
       result = RDoc::Markup::ListItem.new nil, val[0], *val[1]
-
+    
     result
 end
 
 def _reduce_36(val, _values, result)
       result = RDoc::Markup::List.new :NUMBER, *val[0]
-
+    
     result
 end
 
 def _reduce_37(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_38(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_39(val, _values, result)
       result = RDoc::Markup::ListItem.new nil, val[0], *val[1]
-
+    
     result
 end
 
 def _reduce_40(val, _values, result)
       result = RDoc::Markup::List.new :NOTE, *val[0]
-
+    
     result
 end
 
 def _reduce_41(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_42(val, _values, result)
- result = val
+ result = val 
     result
 end
 
@@ -907,77 +907,77 @@ def _reduce_43(val, _values, result)
       term = @inline_parser.parse val[0].strip
 
       result = RDoc::Markup::ListItem.new term, *val[1]
-
+    
     result
 end
 
 def _reduce_44(val, _values, result)
       result = RDoc::Markup::List.new :LABEL, *val[0]
-
+    
     result
 end
 
 def _reduce_45(val, _values, result)
- result.push(val[1])
+ result.push(val[1]) 
     result
 end
 
 def _reduce_46(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_47(val, _values, result)
       result = RDoc::Markup::ListItem.new "<tt>#{val[0].strip}</tt>", *val[1]
-
+    
     result
 end
 
 def _reduce_48(val, _values, result)
       result = [val[1]].concat(val[2])
-
+    
     result
 end
 
 def _reduce_49(val, _values, result)
       result = [val[1]]
-
+    
     result
 end
 
 def _reduce_50(val, _values, result)
       result = val[2]
-
+    
     result
 end
 
 def _reduce_51(val, _values, result)
       result = []
-
+    
     result
 end
 
 def _reduce_52(val, _values, result)
- result.concat val[1]
+ result.concat val[1] 
     result
 end
 
 # reduce 53 omitted
 
 def _reduce_54(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 def _reduce_55(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 56 omitted
 
 def _reduce_57(val, _values, result)
- result = []
+ result = [] 
     result
 end
 
@@ -991,58 +991,58 @@ end
 
 def _reduce_62(val, _values, result)
       result = paragraph [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_63(val, _values, result)
       result = paragraph [val[0]]
-
+    
     result
 end
 
 def _reduce_64(val, _values, result)
       result = paragraph [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_65(val, _values, result)
       result = paragraph [val[0]]
-
+    
     result
 end
 
 def _reduce_66(val, _values, result)
       result = [val[0]].concat(val[1])
-
+    
     result
 end
 
 def _reduce_67(val, _values, result)
- result.concat val[1]
+ result.concat val[1] 
     result
 end
 
 def _reduce_68(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_69(val, _values, result)
- result = val
+ result = val 
     result
 end
 
 # reduce 70 omitted
 
 def _reduce_71(val, _values, result)
- result = []
+ result = [] 
     result
 end
 
 def _reduce_72(val, _values, result)
- result = []
+ result = [] 
     result
 end
 
diff --git a/lib/rdoc/rd/inline_parser.rb b/lib/rdoc/rd/inline_parser.rb
index cc63ea6f70..4a3f3ed7b5 100644
--- a/lib/rdoc/rd/inline_parser.rb
+++ b/lib/rdoc/rd/inline_parser.rb
@@ -732,12 +732,12 @@ Racc_debug_parser = false
 # reduce 1 omitted
 
 def _reduce_2(val, _values, result)
- result.append val[1]
+ result.append val[1] 
     result
 end
 
 def _reduce_3(val, _values, result)
- result = val[0]
+ result = val[0] 
     result
 end
 
@@ -762,28 +762,28 @@ end
 def _reduce_13(val, _values, result)
       content = val[1]
       result = inline "<em>#{content}</em>", content
-
+    
     result
 end
 
 def _reduce_14(val, _values, result)
       content = val[1]
       result = inline "<code>#{content}</code>", content
-
+    
     result
 end
 
 def _reduce_15(val, _values, result)
       content = val[1]
       result = inline "+#{content}+", content
-
+    
     result
 end
 
 def _reduce_16(val, _values, result)
       content = val[1]
       result = inline "<tt>#{content}</tt>", content
-
+    
     result
 end
 
@@ -791,13 +791,13 @@ def _reduce_17(val, _values, result)
       label = val[1]
       @block_parser.add_label label.reference
       result = "<span id=\"label-#{label}\">#{label}</span>"
-
+    
     result
 end
 
 def _reduce_18(val, _values, result)
       result = "{#{val[1]}}[#{val[2].join}]"
-
+    
     result
 end
 
@@ -805,13 +805,13 @@ def _reduce_19(val, _values, result)
       scheme, inline = val[1]
 
       result = "{#{inline}}[#{scheme}#{inline.reference}]"
-
+    
     result
 end
 
 def _reduce_20(val, _values, result)
       result = [nil, inline(val[1])]
-
+    
     result
 end
 
@@ -820,25 +820,25 @@ def _reduce_21(val, _values, result)
         'rdoc-label:',
         inline("#{val[0].reference}/#{val[1].reference}")
       ]
-
+    
     result
 end
 
 def _reduce_22(val, _values, result)
       result = ['rdoc-label:', val[0].reference]
-
+    
     result
 end
 
 def _reduce_23(val, _values, result)
       result = ['rdoc-label:', "#{val[0].reference}/"]
-
+    
     result
 end
 
 def _reduce_24(val, _values, result)
       result = [nil, inline(val[1])]
-
+    
     result
 end
 
@@ -847,92 +847,92 @@ def _reduce_25(val, _values, result)
         'rdoc-label:',
         inline("#{val[0].reference}/#{val[1].reference}")
       ]
-
+    
     result
 end
 
 def _reduce_26(val, _values, result)
       result = ['rdoc-label:', val[0]]
-
+    
     result
 end
 
 def _reduce_27(val, _values, result)
       ref = val[0].reference
       result = ['rdoc-label:', inline(ref, "#{ref}/")]
-
+    
     result
 end
 
 # reduce 28 omitted
 
 def _reduce_29(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_30(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_31(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_32(val, _values, result)
       result = inline "\"#{val[1]}\""
-
+    
     result
 end
 
 def _reduce_33(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_34(val, _values, result)
       result = inline "\"#{val[1]}\""
-
+    
     result
 end
 
 # reduce 35 omitted
 
 def _reduce_36(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_37(val, _values, result)
- result = inline val[1]
+ result = inline val[1] 
     result
 end
 
 def _reduce_38(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_39(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_40(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_41(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
@@ -940,25 +940,25 @@ end
 
 def _reduce_43(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_44(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_45(val, _values, result)
       result = val[0].append val[1]
-
+    
     result
 end
 
 def _reduce_46(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
@@ -984,24 +984,24 @@ end
 
 def _reduce_57(val, _values, result)
       result = val[0]
-
+    
     result
 end
 
 def _reduce_58(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_59(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 def _reduce_60(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1009,7 +1009,7 @@ end
 
 def _reduce_62(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
@@ -1017,7 +1017,7 @@ end
 
 def _reduce_64(val, _values, result)
       result << val[1]
-
+    
     result
 end
 
@@ -1048,7 +1048,7 @@ end
 # reduce 77 omitted
 
 def _reduce_78(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1099,13 +1099,13 @@ end
 def _reduce_101(val, _values, result)
       index = @block_parser.add_footnote val[1].rdoc
       result = "{*#{index}}[rdoc-label:foottext-#{index}:footmark-#{index}]"
-
+    
     result
 end
 
 def _reduce_102(val, _values, result)
       result = inline "<tt>#{val[1]}</tt>", val[1]
-
+    
     result
 end
 
@@ -1122,7 +1122,7 @@ end
 # reduce 108 omitted
 
 def _reduce_109(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
@@ -1130,24 +1130,24 @@ end
 
 def _reduce_111(val, _values, result)
       result = inline val[0]
-
+    
     result
 end
 
 # reduce 112 omitted
 
 def _reduce_113(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_114(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
 def _reduce_115(val, _values, result)
- result = val[1]
+ result = val[1] 
     result
 end
 
@@ -1192,7 +1192,7 @@ end
 # reduce 135 omitted
 
 def _reduce_136(val, _values, result)
- result << val[1]
+ result << val[1] 
     result
 end
 
diff --git a/lib/rdoc/rdoc.gemspec b/lib/rdoc/rdoc.gemspec
index 1822c27366..4c7cd92549 100644
--- a/lib/rdoc/rdoc.gemspec
+++ b/lib/rdoc/rdoc.gemspec
@@ -1,11 +1,10 @@
-# -*- encoding: utf-8 -*-
-$:.unshift File.expand_path("../lib", __FILE__)
-require 'rdoc'
+require_relative "lib/rdoc"
 
 Gem::Specification.new do |s|
   s.name = "rdoc"
   s.version = RDoc::VERSION
-
+  s.date = "2017-09-12"
+  
   s.authors = [
     "Eric Hodel",
     "Dave Thomas",
@@ -28,7 +27,7 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
   s.executables = ["rdoc", "ri"]
   s.require_paths = ["lib"]
   # for ruby core repository. It was generated by `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
-  s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/gauntlet_rdoc.rb", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml","lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_lex.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"]
+  s.files = [".document", ".gitignore", ".travis.yml", "CONTRIBUTING.rdoc", "CVE-2013-0256.rdoc", "ExampleMarkdown.md", "ExampleRDoc.rdoc", "Gemfile", "History.rdoc", "LEGAL.rdoc", "LICENSE.rdoc", "README.rdoc", "RI.rdoc", "Rakefile", "TODO.rdoc", "appveyor.yml", "bin/console", "bin/setup", "exe/rdoc", "exe/ri", "lib/rdoc.rb", "lib/rdoc/alias.rb", "lib/rdoc/anon_class.rb", "lib/rdoc/any_method.rb", "lib/rdoc/attr.rb", "lib/rdoc/class_module.rb", "lib/rdoc/code_object.rb", "lib/rdoc/code_objects.rb", "lib/rdoc/comment.rb", "lib/rdoc/constant.rb", "lib/rdoc/context.rb", "lib/rdoc/context/section.rb", "lib/rdoc/cross_reference.rb", "lib/rdoc/encoding.rb", "lib/rdoc/erb_partial.rb", "lib/rdoc/erbio.rb", "lib/rdoc/extend.rb", "lib/rdoc/generator.rb", "lib/rdoc/generator/darkfish.rb", "lib/rdoc/generator/json_index.rb", "lib/rdoc/generator/markup.rb", "lib/rdoc/generator/pot.rb", "lib/rdoc/generator/pot/message_extractor.rb", "lib/rdoc/generator/pot/po.rb", "lib/rdoc/generator/pot/po_entry.rb", "lib/rdoc/generator/ri.rb", "lib/rdoc/generator/template/darkfish/.document", "lib/rdoc/generator/template/darkfish/_footer.rhtml", "lib/rdoc/generator/template/darkfish/_head.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_VCS_info.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_classes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_extends.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_in_files.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_includes.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_installed.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_methods.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_navigation.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_pages.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_parent.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_sections.rhtml", "lib/rdoc/generator/template/darkfish/_sidebar_table_of_contents.rhtml", "lib/rdoc/generator/template/darkfish/class.rhtml", "lib/rdoc/generator/template/darkfish/css/fonts.css", "lib/rdoc/generator/template/darkfish/css/rdoc.css", "lib/rdoc/generator/template/darkfish/fonts/Lato-Light.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-LightItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-Regular.ttf", "lib/rdoc/generator/template/darkfish/fonts/Lato-RegularItalic.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Bold.ttf", "lib/rdoc/generator/template/darkfish/fonts/SourceCodePro-Regular.ttf", "lib/rdoc/generator/template/darkfish/images/add.png", "lib/rdoc/generator/template/darkfish/images/arrow_up.png", "lib/rdoc/generator/template/darkfish/images/brick.png", "lib/rdoc/generator/template/darkfish/images/brick_link.png", "lib/rdoc/generator/template/darkfish/images/bug.png", "lib/rdoc/generator/template/darkfish/images/bullet_black.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_minus.png", "lib/rdoc/generator/template/darkfish/images/bullet_toggle_plus.png", "lib/rdoc/generator/template/darkfish/images/date.png", "lib/rdoc/generator/template/darkfish/images/delete.png", "lib/rdoc/generator/template/darkfish/images/find.png", "lib/rdoc/generator/template/darkfish/images/loadingAnimation.gif", "lib/rdoc/generator/template/darkfish/images/macFFBgHack.png", "lib/rdoc/generator/template/darkfish/images/package.png", "lib/rdoc/generator/template/darkfish/images/page_green.png", "lib/rdoc/generator/template/darkfish/images/page_white_text.png", "lib/rdoc/generator/template/darkfish/images/page_white_width.png", "lib/rdoc/generator/template/darkfish/images/plugin.png", "lib/rdoc/generator/template/darkfish/images/ruby.png", "lib/rdoc/generator/template/darkfish/images/tag_blue.png", "lib/rdoc/generator/template/darkfish/images/tag_green.png", "lib/rdoc/generator/template/darkfish/images/transparent.png", "lib/rdoc/generator/template/darkfish/images/wrench.png", "lib/rdoc/generator/template/darkfish/images/wrench_orange.png", "lib/rdoc/generator/template/darkfish/images/zoom.png", "lib/rdoc/generator/template/darkfish/index.rhtml", "lib/rdoc/generator/template/darkfish/js/darkfish.js", "lib/rdoc/generator/template/darkfish/js/jquery.js", "lib/rdoc/generator/template/darkfish/js/search.js", "lib/rdoc/generator/template/darkfish/page.rhtml", "lib/rdoc/generator/template/darkfish/servlet_not_found.rhtml", "lib/rdoc/generator/template/darkfish/servlet_root.rhtml", "lib/rdoc/generator/template/darkfish/table_of_contents.rhtml", "lib/rdoc/generator/template/json_index/.document", "lib/rdoc/generator/template/json_index/js/navigation.js", "lib/rdoc/generator/template/json_index/js/searcher.js", "lib/rdoc/ghost_method.rb", "lib/rdoc/i18n.rb", "lib/rdoc/i18n/locale.rb", "lib/rdoc/i18n/text.rb", "lib/rdoc/include.rb", "lib/rdoc/known_classes.rb", "lib/rdoc/markdown.kpeg", "lib/rdoc/markdown/entities.rb", "lib/rdoc/markdown/literals.kpeg", "lib/rdoc/markdown/literals.rb", "lib/rdoc/markup.rb", "lib/rdoc/markup/attr_changer.rb", "lib/rdoc/markup/attr_span.rb", "lib/rdoc/markup/attribute_manager.rb", "lib/rdoc/markup/attributes.rb", "lib/rdoc/markup/blank_line.rb", "lib/rdoc/markup/block_quote.rb", "lib/rdoc/markup/document.rb", "lib/rdoc/markup/formatter.rb", "lib/rdoc/markup/formatter_test_case.rb", "lib/rdoc/markup/hard_break.rb", "lib/rdoc/markup/heading.rb", "lib/rdoc/markup/include.rb", "lib/rdoc/markup/indented_paragraph.rb", "lib/rdoc/markup/inline.rb", "lib/rdoc/markup/list.rb", "lib/rdoc/markup/list_item.rb", "lib/rdoc/markup/paragraph.rb", "lib/rdoc/markup/parser.rb", "lib/rdoc/markup/pre_process.rb", "lib/rdoc/markup/raw.rb", "lib/rdoc/markup/rule.rb", "lib/rdoc/markup/special.rb", "lib/rdoc/markup/text_formatter_test_case.rb", "lib/rdoc/markup/to_ansi.rb", "lib/rdoc/markup/to_bs.rb", "lib/rdoc/markup/to_html.rb", "lib/rdoc/markup/to_html_crossref.rb", "lib/rdoc/markup/to_html_snippet.rb", "lib/rdoc/markup/to_joined_paragraph.rb", "lib/rdoc/markup/to_label.rb", "lib/rdoc/markup/to_markdown.rb", "lib/rdoc/markup/to_rdoc.rb", "lib/rdoc/markup/to_table_of_contents.rb", "lib/rdoc/markup/to_test.rb", "lib/rdoc/markup/to_tt_only.rb", "lib/rdoc/markup/verbatim.rb", "lib/rdoc/meta_method.rb", "lib/rdoc/method_attr.rb", "lib/rdoc/mixin.rb", "lib/rdoc/normal_class.rb", "lib/rdoc/normal_module.rb", "lib/rdoc/options.rb", "lib/rdoc/parser.rb", "lib/rdoc/parser/c.rb", "lib/rdoc/parser/changelog.rb", "lib/rdoc/parser/markdown.rb", "lib/rdoc/parser/rd.rb", "lib/rdoc/parser/ripper_state_lex.rb", "lib/rdoc/parser/ruby.rb", "lib/rdoc/parser/ruby_tools.rb", "lib/rdoc/parser/simple.rb", "lib/rdoc/parser/text.rb", "lib/rdoc/rd.rb", "lib/rdoc/rd/block_parser.ry", "lib/rdoc/rd/inline.rb", "lib/rdoc/rd/inline_parser.ry", "lib/rdoc/rdoc.rb", "lib/rdoc/require.rb", "lib/rdoc/ri.rb", "lib/rdoc/ri/driver.rb", "lib/rdoc/ri/formatter.rb", "lib/rdoc/ri/paths.rb", "lib/rdoc/ri/store.rb", "lib/rdoc/ri/task.rb", "lib/rdoc/ruby_token.rb", "lib/rdoc/rubygems_hook.rb", "lib/rdoc/servlet.rb", "lib/rdoc/single_class.rb", "lib/rdoc/stats.rb", "lib/rdoc/stats/normal.rb", "lib/rdoc/stats/quiet.rb", "lib/rdoc/stats/verbose.rb", "lib/rdoc/store.rb", "lib/rdoc/task.rb", "lib/rdoc/test_case.rb", "lib/rdoc/text.rb", "lib/rdoc/token_stream.rb", "lib/rdoc/tom_doc.rb", "lib/rdoc/top_level.rb", "rdoc.gemspec"]
   # files from .gitignore
   s.files << "lib/rdoc/rd/block_parser.rb" << "lib/rdoc/rd/inline_parser.rb" << "lib/rdoc/markdown.rb"
 
@@ -54,4 +53,5 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat
   s.add_development_dependency("racc", "> 1.4.10")
   s.add_development_dependency("kpeg")
   s.add_development_dependency("minitest", "~> 4")
+  s.add_development_dependency("json")
 end
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
deleted file mode 100644
index e76fdf0414..0000000000
--- a/lib/rdoc/ruby_lex.rb
+++ /dev/null
@@ -1,1521 +0,0 @@
-# coding: US-ASCII
-# frozen_string_literal: false
-
-#--
-#   irb/ruby-lex.rb - ruby lexcal analyzer
-#   	$Release Version: 0.9.5$
-#   	$Revision: 17979 $
-#   	$Date: 2008-07-09 10:17:05 -0700 (Wed, 09 Jul 2008) $
-#   	by Keiju ISHITSUKA(keiju@ruby-lang.org)
-#
-#++
-
-require "e2mmap"
-require "irb/slex"
-require "stringio"
-
-##
-# Ruby lexer adapted from irb.
-#
-# The internals are not documented because they are scary.
-
-class RDoc::RubyLex
-
-  ##
-  # Raised upon invalid input
-
-  class Error < RDoc::Error
-  end
-
-  # :stopdoc:
-
-  extend Exception2MessageMapper
-
-  def_exception(:AlreadyDefinedToken, "Already defined token(%s)")
-  def_exception(:TkReading2TokenNoKey, "key nothing(key='%s')")
-  def_exception(:TkSymbol2TokenNoKey, "key nothing(key='%s')")
-  def_exception(:TkReading2TokenDuplicateError,
-                "key duplicate(token_n='%s', key='%s')")
-  def_exception(:SyntaxError, "%s")
-
-  def_exception(:TerminateLineInput, "Terminate Line Input")
-
-  include RDoc::RubyToken
-  include IRB
-
-  attr_accessor :continue
-  attr_accessor :lex_state
-  attr_accessor :first_in_method_statement
-  attr_reader :reader
-
-  class << self
-    attr_accessor :debug_level
-  end
-
-  def self.debug?
-    @debug_level > 0
-  end
-
-  self.debug_level = 0
-
-  # :startdoc:
-
-  ##
-  # Returns an Array of +ruby+ tokens.  See ::new for a description of
-  # +options+.
-
-  def self.tokenize ruby, options
-    tokens = []
-
-    scanner = RDoc::RubyLex.new ruby, options
-    scanner.exception_on_syntax_error = true
-
-    while token = scanner.token do
-      tokens << token
-    end
-
-    tokens
-  end
-
-  ##
-  # Creates a new lexer for +content+.  +options+ is an RDoc::Options, only
-  # +tab_width is used.
-
-  def initialize(content, options)
-    lex_init
-
-    if /\t/ =~ content then
-      tab_width = options.tab_width
-      content = content.split(/\n/).map do |line|
-        1 while line.gsub!(/\t+/) {
-          ' ' * (tab_width*$&.length - $`.length % tab_width)
-        }  && $~
-        line
-      end.join("\n")
-    end
-
-    content << "\n" unless content[-1, 1] == "\n"
-
-    set_input StringIO.new content
-
-    @base_char_no = 0
-    @char_no = 0
-    @exp_line_no = @line_no = 1
-    @here_readed = []
-    @readed = []
-    @current_readed = @readed
-    @rests = []
-    @seek = 0
-
-    @heredoc_queue = []
-
-    @indent = 0
-    @indent_stack = []
-    @lex_state = :EXPR_BEG
-    @space_seen = false
-    @escaped_nl = false
-    @first_in_method_statement = false
-    @after_question = false
-
-    @continue = false
-    @line = ""
-
-    @skip_space = false
-    @readed_auto_clean_up = false
-    @exception_on_syntax_error = true
-
-    @prompt = nil
-    @prev_seek = nil
-    @ltype = nil
-  end
-
-  # :stopdoc:
-
-  def inspect # :nodoc:
-    "#<%s:0x%x pos %d lex_state %p space_seen %p>" % [
-      self.class, object_id,
-      @io.pos, @lex_state, @space_seen,
-    ]
-  end
-
-  attr_accessor :skip_space
-  attr_accessor :readed_auto_clean_up
-  attr_accessor :exception_on_syntax_error
-
-  attr_reader :seek
-  attr_reader :char_no
-  attr_reader :line_no
-  attr_reader :indent
-
-  # io functions
-  def set_input(io, p = nil, &block)
-    @io = io
-    if p.respond_to?(:call)
-      @input = p
-    elsif block_given?
-      @input = block
-    else
-      @input = Proc.new{@io.gets}
-    end
-  end
-
-  def get_readed
-    if idx = @readed.rindex("\n")
-      @base_char_no = @readed.size - (idx + 1)
-    else
-      @base_char_no += @readed.size
-    end
-
-    readed = @readed.join("")
-    @readed.clear
-    readed
-  end
-
-  def getc
-    while @rests.empty?
-      #      return nil unless buf_input
-      @rests.push nil unless buf_input
-    end
-    c = @rests.shift
-    @current_readed.push c
-    @seek += 1
-    if c == "\n".freeze
-      @line_no += 1
-      @char_no = 0
-    else
-      @char_no += 1
-    end
-
-    c
-  end
-
-  def gets
-    l = ""
-    while c = getc
-      l.concat(c)
-      break if c == "\n"
-    end
-    return nil if l == "" and c.nil?
-    l
-  end
-
-  def eof?
-    @io.eof?
-  end
-
-  def getc_of_rests
-    if @rests.empty?
-      nil
-    else
-      getc
-    end
-  end
-
-  def ungetc(c = nil)
-    if @here_readed.empty?
-      c2 = @readed.pop
-    else
-      c2 = @here_readed.pop
-    end
-    c = c2 unless c
-    @rests.unshift c #c =
-    @seek -= 1
-    if c == "\n"
-      @line_no -= 1
-      if idx = @readed.rindex("\n")
-        @char_no = idx + 1
-      else
-        @char_no = @base_char_no + @readed.size
-      end
-    else
-      @char_no -= 1
-    end
-  end
-
-  def peek_equal?(str)
-    chrs = str.split(//)
-    until @rests.size >= chrs.size
-      return false unless buf_input
-    end
-    @rests[0, chrs.size] == chrs
-  end
-
-  def peek_match?(regexp)
-    while @rests.empty?
-      return false unless buf_input
-    end
-    regexp =~ @rests.join("")
-  end
-
-  def peek(i = 0)
-    while @rests.size <= i
-      return nil unless buf_input
-    end
-    @rests[i]
-  end
-
-  def buf_input
-    prompt
-    line = @input.call
-    return nil unless line
-    @rests.concat line.split(//)
-    true
-  end
-  private :buf_input
-
-  def set_prompt(p = nil, &block)
-    p = block if block_given?
-    if p.respond_to?(:call)
-      @prompt = p
-    else
-      @prompt = Proc.new{print p}
-    end
-  end
-
-  def prompt
-    if @prompt
-      @prompt.call(@ltype, @indent, @continue, @line_no)
-    end
-  end
-
-  def initialize_input
-    @ltype = nil
-    @quoted = nil
-    @indent = 0
-    @indent_stack = []
-    @lex_state = :EXPR_BEG
-    @space_seen = false
-    @current_readed = @readed
-
-    @continue = false
-    prompt
-
-    @line = ""
-    @exp_line_no = @line_no
-  end
-
-  def each_top_level_statement
-    initialize_input
-    catch(:TERM_INPUT) do
-      loop do
-        begin
-          @continue = false
-          prompt
-          unless l = lex
-            throw :TERM_INPUT if @line == ''
-          else
-            #p l
-            @line.concat l
-            if @ltype or @continue or @indent > 0
-              next
-            end
-          end
-          if @line != "\n"
-            yield @line, @exp_line_no
-          end
-          break unless l
-          @line = ''
-          @exp_line_no = @line_no
-
-          @indent = 0
-          @indent_stack = []
-          prompt
-        rescue TerminateLineInput
-          initialize_input
-          prompt
-          get_readed
-        end
-      end
-    end
-  end
-
-  def lex
-    until (((tk = token).kind_of?(TkNL) || tk.kind_of?(TkEND_OF_SCRIPT)) &&
-           !@continue or
-      tk.nil?)
-      #p tk
-      #p @lex_state
-      #p self
-    end
-    line = get_readed
-    #      print self.inspect
-    if line == "" and tk.kind_of?(TkEND_OF_SCRIPT) || tk.nil?
-      nil
-    else
-      line
-    end
-  end
-
-  def token
-    #      require "tracer"
-    #      Tracer.on
-    @prev_seek = @seek
-    @prev_line_no = @line_no
-    @prev_char_no = @char_no
-    begin
-      begin
-        tk = @OP.match(self)
-        @space_seen = tk.kind_of?(TkSPACE)
-        @first_in_method_statement = false if !@space_seen && @first_in_method_statement
-      rescue SyntaxError => e
-        raise Error, "syntax error: #{e.message}" if
-          @exception_on_syntax_error
-
-        tk = TkError.new(@seek, @line_no, @char_no)
-      end
-    end while @skip_space and tk.kind_of?(TkSPACE)
-
-    if @readed_auto_clean_up
-      get_readed
-    end
-
-    if TkSYMBEG === tk then
-      tk1 = token
-      set_token_position tk.seek, tk.line_no, tk.char_no
-
-      case tk1
-      when TkId, TkOp, TkSTRING, TkDSTRING, TkSTAR, TkAMPER then
-        if tk1.respond_to?(:name) then
-          tk = Token(TkSYMBOL, ":" + tk1.name)
-        else
-          tk = Token(TkSYMBOL, ":" + tk1.text)
-        end
-      else
-        tk = tk1
-      end
-    elsif (TkPLUS === tk or TkMINUS === tk) and peek(0) =~ /\d/ then
-      tk1 = token
-      set_token_position tk.seek, tk.line_no, tk.char_no
-      tk = Token(tk1.class, tk.text + tk1.text)
-    end
-    @after_question = false if @after_question and !(TkQUESTION === tk)
-
-    #      Tracer.off
-    tk
-  end
-
-  ENINDENT_CLAUSE = [
-    "case", "class", "def", "do", "for", "if",
-    "module", "unless", "until", "while", "begin" #, "when"
-  ]
-
-  DEINDENT_CLAUSE = ["end" #, "when"
-  ]
-
-  PERCENT_LTYPE = {
-    "q" => "\'",
-    "Q" => "\"",
-    "x" => "\`",
-    "r" => "/",
-    "w" => "]",
-    "W" => "]",
-    "s" => ":",
-    "i" => "]",
-    "I" => "]"
-  }
-
-  PERCENT_PAREN = {
-    "{" => "}",
-    "[" => "]",
-    "<" => ">",
-    "(" => ")"
-  }
-
-  PERCENT_PAREN_REV = PERCENT_PAREN.invert
-
-  Ltype2Token = {
-    "\'" => TkSTRING,
-    "\"" => TkSTRING,
-    "\`" => TkXSTRING,
-    "/" => TkREGEXP,
-    "]" => TkDSTRING,
-    ":" => TkSYMBOL
-  }
-  DLtype2Token = {
-    "\"" => TkDSTRING,
-    "\`" => TkDXSTRING,
-    "/" => TkDREGEXP,
-  }
-
-  def lex_init()
-    @OP = IRB::SLex.new
-    @OP.def_rules("\0", "\004", "\032") do |op, io|
-      Token(TkEND_OF_SCRIPT, '')
-    end
-
-    @OP.def_rules(" ", "\t", "\f", "\r", "\13") do |op, io|
-      @space_seen = true
-      str = op
-      while (ch = getc) =~ /[ \t\f\r\13]/ do
-        str << ch
-      end
-      ungetc
-      Token TkSPACE, str
-    end
-
-    @OP.def_rule("#") do |op, io|
-      identify_comment
-    end
-
-    @OP.def_rule("=begin",
-                 proc{|op, io| @prev_char_no == 0 && peek(0) =~ /\s/}) do
-      |op, io|
-      @ltype = "="
-      res = op
-      until (ch = getc) == "\n" do
-        res << ch
-      end
-      res << ch
-
-      until ( peek_equal?("=end") && peek(4) =~ /\s/ ) do
-        (ch = getc)
-        res << ch
-      end
-
-      res << gets # consume =end
-
-      @ltype = nil
-      Token(TkRD_COMMENT, res)
-    end
-
-    @OP.def_rule("\n") do |op, io|
-      print "\\n\n" if RDoc::RubyLex.debug?
-      unless @heredoc_queue.empty?
-        info = @heredoc_queue[0]
-        if !info[:started] # "\n"
-          info[:started] = true
-          ungetc "\n"
-        elsif info[:heredoc_end].nil? # heredoc body
-          tk, heredoc_end = identify_here_document_body(info[:quoted], info[:lt], info[:indent])
-          info[:heredoc_end] = heredoc_end
-          ungetc "\n"
-        else # heredoc end
-          @heredoc_queue.shift
-          @lex_state = :EXPR_BEG
-          tk = Token(TkHEREDOCEND, info[:heredoc_end])
-          if !@heredoc_queue.empty?
-            @heredoc_queue[0][:started] = true
-            ungetc "\n"
-          end
-        end
-      end
-      unless tk
-        case @lex_state
-        when :EXPR_BEG, :EXPR_FNAME, :EXPR_DOT
-          @continue = true
-        else
-          @continue = false
-          @lex_state = :EXPR_BEG unless @escaped_nl
-          until (@indent_stack.empty? ||
-                 [TkLPAREN, TkLBRACK, TkLBRACE,
-                   TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
-            @indent_stack.pop
-          end
-        end
-        @current_readed = @readed
-        @here_readed.clear
-        tk = Token(TkNL)
-      end
-      @escaped_nl = false
-      tk
-    end
-
-    @OP.def_rules("=") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        @lex_state = :EXPR_ARG
-      else
-        @lex_state = :EXPR_BEG
-      end
-      Token(op)
-    end
-
-    @OP.def_rules("*", "**",
-                  "==", "===",
-                  "=~", "<=>",
-                  "<", "<=",
-                  ">", ">=", ">>", "=>") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        tk = Token(TkId, op)
-        @lex_state = :EXPR_ARG
-      else
-        tk = Token(op)
-        @lex_state = :EXPR_BEG
-      end
-      tk
-    end
-
-    @OP.def_rules("->") do
-      |op, io|
-      @lex_state = :EXPR_ENDFN
-      Token(op)
-    end
-
-    @OP.def_rules("!", "!=", "!~") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      else
-        @lex_state = :EXPR_BEG
-        Token(op)
-      end
-    end
-
-    @OP.def_rules("<<") do
-      |op, io|
-      tk = nil
-      if @lex_state != :EXPR_END && @lex_state != :EXPR_CLASS &&
-         (@lex_state != :EXPR_ARG || @space_seen)
-        c = peek(0)
-        if /\S/ =~ c && (/["'`]/ =~ c || /\w/ =~ c || c == "-" || c == "~")
-          tk = identify_here_document(op)
-        end
-      end
-      unless tk
-        case @lex_state
-        when :EXPR_FNAME, :EXPR_DOT
-          tk = Token(TkId, op)
-          @lex_state = :EXPR_ARG
-        else
-          tk = Token(op)
-          @lex_state = :EXPR_BEG
-        end
-      end
-      tk
-    end
-
-    @OP.def_rules("'", '"') do
-      |op, io|
-      identify_string(op)
-    end
-
-    @OP.def_rules("`") do
-      |op, io|
-      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      else
-        identify_string(op)
-      end
-    end
-
-    @OP.def_rules('?') do
-      |op, io|
-      if @lex_state == :EXPR_END
-        @lex_state = :EXPR_BEG
-        @after_question = true
-        Token(TkQUESTION)
-      else
-        ch = getc
-        if @lex_state == :EXPR_ARG && ch =~ /\s/
-          ungetc
-          @lex_state = :EXPR_BEG;
-          Token(TkQUESTION)
-        else
-          @lex_state = :EXPR_END
-          ch << getc if "\\" == ch
-          Token(TkCHAR, "?#{ch}")
-        end
-      end
-    end
-
-    @OP.def_rules("&&", "||") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      Token(op)
-    end
-
-    @OP.def_rules("&", "|") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        tk = Token(TkId, op)
-        @lex_state = :EXPR_ARG
-      else
-        tk = Token(op)
-        @lex_state = :EXPR_BEG
-      end
-      tk
-    end
-
-    @OP.def_rules("+=", "-=", "*=", "**=",
-                  "&=", "|=", "^=", "<<=", ">>=", "||=", "&&=") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      op =~ /^(.*)=$/
-      Token(TkOPASGN, $1)
-    end
-
-    @OP.def_rule("+@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_ARG
-      Token(TkId, op)
-    end
-
-    @OP.def_rule("-@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_ARG
-      Token(TkId, op)
-    end
-
-    @OP.def_rules("+", "-") do
-      |op, io|
-      catch(:RET) do
-        if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-          tk = Token(TkId, op)
-          @lex_state = :EXPR_ARG
-        elsif @lex_state == :EXPR_ARG
-          if @space_seen and peek(0) =~ /[0-9]/
-            throw :RET, identify_number(op)
-          else
-            @lex_state = :EXPR_BEG
-          end
-        elsif @lex_state != :EXPR_END and peek(0) =~ /[0-9]/
-          throw :RET, identify_number(op)
-        else
-          @lex_state = :EXPR_BEG
-        end
-        tk = Token(op) unless tk
-        tk
-      end
-    end
-
-    @OP.def_rules(".", "&.") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      if peek(0) =~ /[0-9]/
-        ungetc
-        identify_number
-      else
-        # for "obj.if" or "obj&.if" etc.
-        @lex_state = :EXPR_DOT
-        Token(op)
-      end
-    end
-
-    @OP.def_rules("..", "...") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      Token(op)
-    end
-
-    lex_int2
-  end
-
-  def lex_int2
-    @OP.def_rules("]", "}", ")") do
-      |op, io|
-      @lex_state = :EXPR_END
-      @indent -= 1
-      @indent_stack.pop
-      Token(op)
-    end
-
-    @OP.def_rule(":") do
-      |op, io|
-      if @lex_state == :EXPR_END || peek(0) =~ /\s/
-        @lex_state = :EXPR_BEG
-        Token(TkCOLON)
-      else
-        @lex_state = :EXPR_FNAME;
-        Token(TkSYMBEG)
-      end
-    end
-
-    @OP.def_rule("::") do
-      |op, io|
-      #      p @lex_state.id2name, @space_seen
-      if @lex_state == :EXPR_BEG or @lex_state == :EXPR_ARG && @space_seen
-        @lex_state = :EXPR_BEG
-        Token(TkCOLON3)
-      else
-        @lex_state = :EXPR_DOT
-        Token(TkCOLON2)
-      end
-    end
-
-    @OP.def_rule("/") do
-      |op, io|
-      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID || @first_in_method_statement
-        identify_string(op)
-      elsif peek(0) == '='
-        getc
-        @lex_state = :EXPR_BEG
-        Token(TkOPASGN, "/") #/)
-      elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
-        identify_string(op)
-      else
-        @lex_state = :EXPR_BEG
-        Token("/") #/)
-      end
-    end
-
-    @OP.def_rules("^") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        tk = Token(TkId, op)
-        @lex_state = :EXPR_ARG
-      else
-        tk = Token(op)
-        @lex_state = :EXPR_BEG
-      end
-      tk
-    end
-
-    #       @OP.def_rules("^=") do
-    # 	@lex_state = :EXPR_BEG
-    # 	Token(OP_ASGN, :^)
-    #       end
-
-    @OP.def_rules(",") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      Token(op)
-    end
-
-    @OP.def_rules(";") do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      until (@indent_stack.empty? ||
-             [TkLPAREN, TkLBRACK, TkLBRACE,
-               TkfLPAREN, TkfLBRACK, TkfLBRACE].include?(@indent_stack.last))
-        @indent_stack.pop
-      end
-      Token(op)
-    end
-
-    @OP.def_rule("~") do
-      |op, io|
-      case @lex_state
-      when :EXPR_FNAME, :EXPR_DOT
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      else
-        @lex_state = :EXPR_BEG
-        Token(op)
-      end
-    end
-
-    @OP.def_rule("~@", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_BEG
-      Token("~")
-    end
-
-    @OP.def_rule("(") do
-      |op, io|
-      @indent += 1
-      if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
-        @lex_state = :EXPR_BEG
-        tk_c = TkfLPAREN
-      else
-        @lex_state = :EXPR_BEG
-        tk_c = TkLPAREN
-      end
-      @indent_stack.push tk_c
-      Token tk_c
-    end
-
-    @OP.def_rule("[]", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_ARG
-      Token(TkId, op)
-    end
-
-    @OP.def_rule("[]=", proc{|op, io| @lex_state == :EXPR_FNAME}) do
-      |op, io|
-      @lex_state = :EXPR_ARG
-      Token(TkId, op)
-    end
-
-    @OP.def_rule("[") do
-      |op, io|
-      text = nil
-      @indent += 1
-      if @lex_state == :EXPR_FNAME
-        tk_c = TkfLBRACK
-      else
-        if @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
-          tk_c = TkLBRACK
-        elsif @lex_state == :EXPR_ARG && @space_seen
-          tk_c = TkLBRACK
-        elsif @lex_state == :EXPR_DOT
-          if peek(0) == "]"
-            tk_c = TkIDENTIFIER
-            getc
-            if peek(0) == "="
-              text = "[]="
-            else
-              text = "[]"
-            end
-          else
-            tk_c = TkOp
-          end
-        else
-          tk_c = TkfLBRACK
-        end
-        @lex_state = :EXPR_BEG
-      end
-      @indent_stack.push tk_c
-      Token(tk_c, text)
-    end
-
-    @OP.def_rule("{") do
-      |op, io|
-      @indent += 1
-      if @lex_state != :EXPR_END && @lex_state != :EXPR_ARG
-        tk_c = TkLBRACE
-      else
-        tk_c = TkfLBRACE
-      end
-      @lex_state = :EXPR_BEG
-      @indent_stack.push tk_c
-      Token(tk_c)
-    end
-
-    @OP.def_rule('\\') do
-      |op, io|
-      if peek(0) == "\n"
-        @space_seen = true
-        @continue = true
-        @escaped_nl = true
-      end
-      Token("\\")
-    end
-
-    @OP.def_rule('%') do
-      |op, io|
-      if :EXPR_FNAME == @lex_state or :EXPR_DOT == @lex_state
-        @lex_state = :EXPR_ARG
-        Token(TkId, op)
-      elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_MID
-        identify_quotation
-      elsif peek(0) == '='
-        getc
-        @lex_state = :EXPR_BEG
-        Token(TkOPASGN, '%')
-      elsif @lex_state == :EXPR_ARG and @space_seen and peek(0) !~ /\s/
-        identify_quotation
-      else
-        @lex_state = :EXPR_BEG
-        Token("%") #))
-      end
-    end
-
-    @OP.def_rule('$') do
-      |op, io|
-      identify_gvar
-    end
-
-    @OP.def_rule('@') do
-      |op, io|
-      if peek(0) =~ /[\w@]/
-        ungetc
-        identify_identifier
-      else
-        Token("@")
-      end
-    end
-
-    #       @OP.def_rule("def", proc{|op, io| /\s/ =~ io.peek(0)}) do
-    # 	|op, io|
-    # 	@indent += 1
-    # 	@lex_state = :EXPR_FNAME
-    # #	@lex_state = :EXPR_END
-    # #	until @rests[0] == "\n" or @rests[0] == ";"
-    # #	  rests.shift
-    # #	end
-    #       end
-
-    @OP.def_rule("_") do
-      if peek_match?(/_END__/) and @lex_state == :EXPR_BEG then
-        6.times { getc }
-        Token(TkEND_OF_SCRIPT, '__END__')
-      else
-        ungetc
-        identify_identifier
-      end
-    end
-
-    @OP.def_rule("") do
-      |op, io|
-      printf "MATCH: start %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
-      if peek(0) =~ /[0-9]/
-        t = identify_number
-      else
-        t = identify_identifier
-      end
-      printf "MATCH: end %s: %s\n", op, io.inspect if RDoc::RubyLex.debug?
-      t
-    end
-
-    p @OP if RDoc::RubyLex.debug?
-  end
-
-  def identify_gvar
-    @lex_state = :EXPR_END
-
-    case ch = getc
-    when /[~_*$?!@\/\\;,=:<>".]/   #"
-      Token(TkGVAR, "$" + ch)
-    when "-"
-      Token(TkGVAR, "$-" + getc)
-    when "&", "`", "'", "+"
-      Token(TkBACK_REF, "$"+ch)
-    when /[1-9]/
-      ref = ch
-      while (ch = getc) =~ /[0-9]/ do ref << ch end
-      ungetc
-      Token(TkNTH_REF, "$#{ref}")
-    when /\w/
-      ungetc
-      ungetc
-      identify_identifier
-    else
-      ungetc
-      Token("$")
-    end
-  end
-
-  IDENT_RE = eval '/[\w\u{0080}-\u{FFFFF}]/u'
-
-  def identify_identifier
-    token = ""
-    if peek(0) =~ /[$@]/
-      token.concat(c = getc)
-      if c == "@" and peek(0) == "@"
-        token.concat getc
-      end
-    end
-
-    while (ch = getc) =~ IDENT_RE do
-      print " :#{ch}: " if RDoc::RubyLex.debug?
-      token.concat ch
-    end
-
-    ungetc
-
-    if ((ch == "!" && peek(1) != "=") || ch == "?") && token[0,1] =~ /\w/
-      token.concat getc
-    end
-
-    # almost fix token
-
-    case token
-    when /^\$/
-      return Token(TkGVAR, token)
-    when /^\@\@/
-      @lex_state = :EXPR_END
-      # p Token(TkCVAR, token)
-      return Token(TkCVAR, token)
-    when /^\@/
-      @lex_state = :EXPR_END
-      return Token(TkIVAR, token)
-    end
-
-    if @lex_state != :EXPR_DOT
-      print token, "\n" if RDoc::RubyLex.debug?
-
-      token_c, *trans = TkReading2Token[token]
-      if token_c
-        # reserved word?
-
-        if (@lex_state != :EXPR_BEG &&
-            @lex_state != :EXPR_FNAME &&
-            trans[1])
-          # modifiers
-          token_c = TkSymbol2Token[trans[1]]
-          @lex_state = trans[0]
-        else
-          if @lex_state != :EXPR_FNAME
-            if ENINDENT_CLAUSE.include?(token)
-              valid = peek(0) != ':'
-
-              # check for ``class = val'' etc.
-              case token
-              when "class"
-                valid = false unless peek_match?(/^\s*(<<|\w|::)/)
-              when "def"
-                valid = false if peek_match?(/^\s*(([+-\/*&\|^]|<<|>>|\|\||\&\&)=|\&\&|\|\|)/)
-              when "do"
-                valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&)/)
-              when *ENINDENT_CLAUSE
-                valid = false if peek_match?(/^\s*([+-\/*]?=|\*|<|>|\&|\|)/)
-              else
-                # no nothing
-              end if valid
-
-              if valid
-                if token == "do"
-                  if ![TkFOR, TkWHILE, TkUNTIL].include?(@indent_stack.last)
-                    @indent += 1
-                    @indent_stack.push token_c
-                  end
-                else
-                  @indent += 1
-                  @indent_stack.push token_c
-                end
-              else
-                token_c = TkIDENTIFIER
-              end
-
-            elsif DEINDENT_CLAUSE.include?(token)
-              @indent -= 1
-              @indent_stack.pop
-            end
-            @lex_state = trans[0]
-          else
-            @lex_state = :EXPR_END
-          end
-        end
-        if token_c.ancestors.include?(TkId) and peek(0) == ':' and !peek_match?(/^::/)
-          token.concat getc
-          token_c = TkSYMBOL
-        end
-        return Token(token_c, token)
-      end
-    end
-
-    if @lex_state == :EXPR_FNAME
-      @lex_state = :EXPR_END
-      if peek(0) == '=' and peek(1) != '>'
-        token.concat getc
-      end
-    elsif @lex_state == :EXPR_BEG || @lex_state == :EXPR_DOT ||
-          @lex_state == :EXPR_ARG || @lex_state == :EXPR_MID
-      @lex_state = :EXPR_ARG
-    else
-      @lex_state = :EXPR_END
-    end
-
-    if token[0, 1] =~ /[A-Z]/
-      if token[-1] =~ /[!?]/
-        token_c = TkIDENTIFIER
-      else
-        token_c = TkCONSTANT
-      end
-    elsif token[token.size - 1, 1] =~ /[!?]/
-      token_c = TkFID
-    else
-      token_c = TkIDENTIFIER
-    end
-    if peek(0) == ':' and !peek_match?(/^::/)
-      token.concat getc
-      return Token(TkSYMBOL, token)
-    else
-      return Token(token_c, token)
-    end
-  end
-
-  def identify_here_document(op)
-    ch = getc
-    start_token = op
-    #    if lt = PERCENT_LTYPE[ch]
-    if ch == "-" or ch == "~"
-      start_token.concat ch
-      ch = getc
-      indent = true
-    end
-    if /['"`]/ =~ ch
-      start_token.concat ch
-      user_quote = lt = ch
-      quoted = ""
-      while (c = getc) && c != lt
-        quoted.concat c
-      end
-      start_token.concat quoted
-      start_token.concat lt
-    else
-      user_quote = nil
-      lt = '"'
-      quoted = ch.dup
-      while (c = getc) && c =~ /\w/
-        quoted.concat c
-      end
-      start_token.concat quoted
-      ungetc
-    end
-
-    @heredoc_queue << {
-      quoted: quoted,
-      lt: lt,
-      indent: indent,
-      started: false
-    }
-    @lex_state = :EXPR_END
-    Token(RDoc::RubyLex::TkHEREDOCBEG, start_token)
-  end
-
-  def identify_here_document_body(quoted, lt, indent)
-    ltback, @ltype = @ltype, lt
-
-    doc = ""
-    heredoc_end = nil
-    while l = gets
-      l = l.sub(/(:?\r)?\n\z/, "\n")
-      if (indent ? l.strip : l.chomp) == quoted
-        heredoc_end = l
-        break
-      end
-      doc << l
-    end
-    raise Error, "Missing terminating #{quoted} for string" unless heredoc_end
-
-    @ltype = ltback
-    @lex_state = :EXPR_BEG
-    [Token(RDoc::RubyLex::TkHEREDOC, doc), heredoc_end]
-  end
-
-  def identify_quotation
-    type = ch = getc
-    if lt = PERCENT_LTYPE[type]
-      ch = getc
-    elsif type =~ /\W/
-      type = nil
-      lt = "\""
-    else
-      return Token(TkMOD, '%')
-    end
-    #     if ch !~ /\W/
-    #       ungetc
-    #       next
-    #     end
-    #@ltype = lt
-    @quoted = ch unless @quoted = PERCENT_PAREN[ch]
-    identify_string(lt, @quoted, type)
-  end
-
-  def identify_number(op = "")
-    @lex_state = :EXPR_END
-
-    num = op
-
-    if peek(0) == "0" && peek(1) !~ /[.eEri]/
-      num << getc
-
-      case peek(0)
-      when /[xX]/
-        ch = getc
-        match = /[0-9a-fA-F_]/
-      when /[bB]/
-        ch = getc
-        match = /[01_]/
-      when /[oO]/
-        ch = getc
-        match = /[0-7_]/
-      when /[dD]/
-        ch = getc
-        match = /[0-9_]/
-      when /[0-7]/
-        match = /[0-7_]/
-      when /[89]/
-        raise Error, "Illegal octal digit"
-      else
-        return Token(TkINTEGER, num)
-      end
-
-      num << ch if ch
-
-      len0 = true
-      non_digit = false
-      while ch = getc
-        num << ch
-        if match =~ ch
-          if ch == "_"
-            if non_digit
-              raise Error, "trailing `#{ch}' in number"
-            else
-              non_digit = ch
-            end
-          else
-            non_digit = false
-            len0 = false
-          end
-        else
-          ungetc
-          num[-1, 1] = ''
-          if len0
-            raise Error, "numeric literal without digits"
-          end
-          if non_digit
-            raise Error, "trailing `#{non_digit}' in number"
-          end
-          break
-        end
-      end
-      return Token(TkINTEGER, num)
-    end
-
-    type = TkINTEGER
-    allow_point = true
-    allow_e = true
-    allow_ri = true
-    non_digit = false
-    while ch = getc
-      num << ch
-      case ch
-      when /[0-9]/
-        non_digit = false
-      when "_"
-        non_digit = ch
-      when allow_point && "."
-        if non_digit
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        type = TkFLOAT
-        if peek(0) !~ /[0-9]/
-          type = TkINTEGER
-          ungetc
-          num[-1, 1] = ''
-          break
-        end
-        allow_point = false
-      when allow_e && "e", allow_e && "E"
-        if non_digit
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        type = TkFLOAT
-        if peek(0) =~ /[+-]/
-          num << getc
-        end
-        allow_e = false
-        allow_ri = false
-        allow_point = false
-        non_digit = ch
-      when allow_ri && "r"
-        if non_digit
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        type = TkRATIONAL
-        if peek(0) == 'i'
-          type = TkIMAGINARY
-          num << getc
-        end
-        break
-      when allow_ri && "i"
-        if non_digit && non_digit != "r"
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        type = TkIMAGINARY
-        break
-      else
-        if non_digit
-          raise Error, "trailing `#{non_digit}' in number"
-        end
-        ungetc
-        num[-1, 1] = ''
-        break
-      end
-    end
-
-    Token(type, num)
-  end
-
-  def identify_string(ltype, quoted = ltype, type = nil)
-    close = PERCENT_PAREN.values.include?(quoted)
-    @ltype = ltype
-    @quoted = quoted
-
-    str = if ltype == quoted and %w[" ' / `].include? ltype and type.nil? then
-            ltype.dup
-          else
-            "%#{type}#{PERCENT_PAREN_REV[quoted]||quoted}"
-          end
-
-    subtype = nil
-    begin
-      nest = 0
-
-      while ch = getc
-        str << ch
-
-        if @quoted == ch and nest <= 0
-          break
-        elsif @ltype != "'" && @ltype != "]" && @ltype != ":" and ch == "#"
-          ch = getc
-          if ch == "{" then
-            subtype = true
-            str << ch << skip_inner_expression
-            next
-          else
-            ungetc
-          end
-        elsif ch == '\\'
-          case @ltype
-          when "'" then
-            case ch = getc
-            when "'", '\\' then
-              str << ch
-            else
-              str << ch
-            end
-          else
-            str << read_escape
-          end
-        end
-
-        if close then
-          if PERCENT_PAREN[ch] == @quoted
-            nest += 1
-          elsif ch == @quoted
-            nest -= 1
-          end
-        end
-      end
-
-      if @ltype == "/"
-        while peek(0) =~ /i|m|x|o|e|s|u|n/
-          str << getc
-        end
-      end
-
-      if peek(0) == ':' and !peek_match?(/^::/) and :EXPR_BEG == @lex_state and !@after_question
-        str.concat getc
-        return Token(TkSYMBOL, str)
-      elsif subtype
-        Token(DLtype2Token[ltype], str)
-      else
-        Token(Ltype2Token[ltype], str)
-      end
-    ensure
-      @ltype = nil
-      @quoted = nil
-      @lex_state = :EXPR_END
-    end
-  end
-
-  def skip_inner_expression
-    res = ""
-    nest = 0
-    while ch = getc
-      res << ch
-      if ch == '}'
-        break if nest.zero?
-        nest -= 1
-      elsif ch == '{'
-        nest += 1
-      end
-    end
-    res
-  end
-
-  def identify_comment
-    @ltype = "#"
-
-    comment = '#'
-
-    while ch = getc
-      # if ch == "\\" #"
-      #   read_escape
-      # end
-      if ch == "\n"
-        @ltype = nil
-        ungetc
-        break
-      end
-
-      comment << ch
-    end
-
-    return Token(TkCOMMENT, comment)
-  end
-
-  def read_escape
-    escape = ''
-    ch = getc
-
-    case ch
-    when "\n", "\r", "\f"
-      escape << ch
-    when "\\", "n", "t", "r", "f", "v", "a", "e", "b", "s" #"
-      escape << ch
-    when /[0-7]/
-      ungetc ch
-      3.times do
-        ch = getc
-        case ch
-        when /[0-7]/
-          escape << ch
-        when nil
-          break
-        else
-          ungetc
-          break
-        end
-      end
-
-    when "x"
-      escape << ch
-
-      2.times do
-        ch = getc
-        case ch
-        when /[0-9a-fA-F]/
-          escape << ch
-        when nil
-          break
-        else
-          ungetc
-          break
-        end
-      end
-
-    when "M"
-      escape << ch
-
-      ch = getc
-      if ch != '-'
-        ungetc
-      else
-        escape << ch
-
-        ch = getc
-        if ch == "\\" #"
-          ungetc
-          escape << read_escape
-        else
-          escape << ch
-        end
-      end
-
-    when "C", "c" #, "^"
-      escape << ch
-
-      if ch == "C"
-        ch = getc
-
-        if ch == "-"
-          escape << ch
-          ch = getc
-          escape << ch
-
-          escape << read_escape if ch == "\\"
-        else
-          ungetc
-        end
-      elsif (ch = getc) == "\\" #"
-        escape << ch << read_escape
-      end
-    else
-      escape << ch
-
-      # other characters
-    end
-
-    escape
-  end
-
-  # :startdoc:
-
-end
-
-#RDoc::RubyLex.debug_level = 1
diff --git a/lib/rdoc/stats/normal.rb b/lib/rdoc/stats/normal.rb
index ba00b6cbdf..8ad0e453c6 100644
--- a/lib/rdoc/stats/normal.rb
+++ b/lib/rdoc/stats/normal.rb
@@ -1,5 +1,5 @@
 # frozen_string_literal: false
-require 'io/console/size'
+require 'io/console'
 
 ##
 # Stats printer that prints just the files being documented with a progress
@@ -23,7 +23,7 @@ class RDoc::Stats::Normal < RDoc::Stats::Quiet
 
     # Print a progress bar, but make sure it fits on a single line. Filename
     # will be truncated if necessary.
-    terminal_width = IO.console_size[1].to_i.nonzero? || 80
+    terminal_width = IO.console.winsize[1].to_i.nonzero? || 80
     max_filename_size = terminal_width - progress_bar.size
 
     if filename.size > max_filename_size then
diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb
index 0cfa2f1384..df5232f7ea 100644
--- a/lib/rdoc/token_stream.rb
+++ b/lib/rdoc/token_stream.rb
@@ -30,28 +30,41 @@ module RDoc::TokenStream
     token_stream.map do |t|
       next unless t
 
-      style = case t
-              when RDoc::RubyToken::TkCONSTANT then 'ruby-constant'
-              when RDoc::RubyToken::TkKW       then 'ruby-keyword'
-              when RDoc::RubyToken::TkIVAR     then 'ruby-ivar'
-              when RDoc::RubyToken::TkOp       then 'ruby-operator'
-              when RDoc::RubyToken::TkId       then 'ruby-identifier'
-              when RDoc::RubyToken::TkREGEXP   then 'ruby-regexp'
-              when RDoc::RubyToken::TkDREGEXP  then 'ruby-regexp'
-              when RDoc::RubyToken::TkNode     then 'ruby-node'
-              when RDoc::RubyToken::TkCOMMENT  then 'ruby-comment'
-              when RDoc::RubyToken::TkXSTRING  then 'ruby-string'
-              when RDoc::RubyToken::TkSTRING   then 'ruby-string'
-              when RDoc::RubyToken::TkVal      then 'ruby-value'
+      style = case t[:kind]
+              when :on_const   then 'ruby-constant'
+              when :on_kw      then 'ruby-keyword'
+              when :on_ivar    then 'ruby-ivar'
+              when :on_cvar    then 'ruby-identifier'
+              when :on_gvar    then 'ruby-identifier'
+              when '=' != t[:text] && :on_op then
+                if RDoc::RipperStateLex::EXPR_ARG == t[:state] then
+                                    'ruby-identifier'
+                else
+                                    'ruby-operator'
+                end
+              when :on_tlambda then 'ruby-operator'
+              when :on_ident   then 'ruby-identifier'
+              when :on_label   then 'ruby-value'
+              when :on_backref, :on_dstring
+                               then 'ruby-node'
+              when :on_comment then 'ruby-comment'
+              when :on_embdoc  then 'ruby-comment'
+              when :on_regexp  then 'ruby-regexp'
+              when :on_tstring then 'ruby-string'
+              when :on_int, :on_float,
+                   :on_rational, :on_imaginary,
+                   :on_heredoc,
+                   :on_symbol, :on_CHAR then 'ruby-value'
+              when :on_heredoc_beg, :on_heredoc_end
+                               then 'ruby-identifier'
               end
 
       comment_with_nl = false
-      case t
-      when RDoc::RubyToken::TkRD_COMMENT, RDoc::RubyToken::TkHEREDOCEND
-        comment_with_nl = true if t.text =~ /\n$/
-        text = t.text.rstrip
+      if :on_comment == t[:kind] or :on_embdoc == t[:kind] or :on_heredoc_end == t[:kind]
+        comment_with_nl = true if "\n" == t[:text][-1]
+        text = t[:text].rstrip
       else
-        text = t.text
+        text = t[:text]
       end
       text = CGI.escapeHTML text
 
diff --git a/test/rdoc/test_rdoc_any_method.rb b/test/rdoc/test_rdoc_any_method.rb
index fdbb62efa6..fdfb0f3995 100644
--- a/test/rdoc/test_rdoc_any_method.rb
+++ b/test/rdoc/test_rdoc_any_method.rb
@@ -74,7 +74,7 @@ method(a, b) { |c, d| ... }
 
   def test_markup_code
     tokens = [
-      RDoc::RubyToken::TkCONSTANT. new(0, 0, 0, 'CONSTANT'),
+      { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' },
     ]
 
     @c2_a.collect_tokens
diff --git a/test/rdoc/test_rdoc_constant.rb b/test/rdoc/test_rdoc_constant.rb
index c43aa7dc2b..a03ab97a7d 100644
--- a/test/rdoc/test_rdoc_constant.rb
+++ b/test/rdoc/test_rdoc_constant.rb
@@ -86,7 +86,7 @@ class TestRDocConstant < XrefTestCase
     assert_equal top_level,      loaded.file
     assert_equal 'Klass::CONST', loaded.full_name
     assert_equal 'CONST',        loaded.name
-    assert_nil                   loaded.visibility
+    assert_equal :public,        loaded.visibility
     assert_equal cm,             loaded.parent
     assert_equal section,        loaded.section
   end
@@ -114,7 +114,7 @@ class TestRDocConstant < XrefTestCase
     assert_equal top_level,      loaded.file
     assert_equal 'Klass::CONST', loaded.full_name
     assert_equal 'CONST',        loaded.name
-    assert_nil                   loaded.visibility
+    assert_equal :public,        loaded.visibility
     assert_equal cm,             loaded.parent
     assert_equal section,        loaded.section
 
@@ -146,7 +146,7 @@ class TestRDocConstant < XrefTestCase
     assert_equal top_level,      loaded.file
     assert_equal 'Klass::CONST', loaded.full_name
     assert_equal 'CONST',        loaded.name
-    assert_nil                   loaded.visibility
+    assert_equal :public,        loaded.visibility
     assert_equal cm,             loaded.parent
     assert_equal section,        loaded.section
 
diff --git a/test/rdoc/test_rdoc_context.rb b/test/rdoc/test_rdoc_context.rb
index 54e365f7f4..ceac14ac90 100644
--- a/test/rdoc/test_rdoc_context.rb
+++ b/test/rdoc/test_rdoc_context.rb
@@ -719,6 +719,7 @@ class TestRDocContext < XrefTestCase
 
     assert_equal [@pub, @prot, @priv], @vis.method_list
     assert_equal [@apub, @aprot, @apriv], @vis.attributes
+    assert_equal [@cpub, @cpriv], @vis.constants
   end
 
   def test_remove_invisible_nodoc
@@ -728,6 +729,7 @@ class TestRDocContext < XrefTestCase
 
     assert_equal [@pub, @prot, @priv], @vis.method_list
     assert_equal [@apub, @aprot, @apriv], @vis.attributes
+    assert_equal [@cpub, @cpriv], @vis.constants
   end
 
   def test_remove_invisible_protected
@@ -737,6 +739,7 @@ class TestRDocContext < XrefTestCase
 
     assert_equal [@pub, @prot], @vis.method_list
     assert_equal [@apub, @aprot], @vis.attributes
+    assert_equal [@cpub], @vis.constants
   end
 
   def test_remove_invisible_public
@@ -746,6 +749,7 @@ class TestRDocContext < XrefTestCase
 
     assert_equal [@pub], @vis.method_list
     assert_equal [@apub], @vis.attributes
+    assert_equal [@cpub], @vis.constants
   end
 
   def test_remove_invisible_public_force
@@ -755,11 +759,13 @@ class TestRDocContext < XrefTestCase
     @prot.force_documentation = true
     @apriv.force_documentation = true
     @aprot.force_documentation = true
+    @cpriv.force_documentation = true
 
     @vis.remove_invisible :public
 
     assert_equal [@pub, @prot, @priv], @vis.method_list
     assert_equal [@apub, @aprot, @apriv], @vis.attributes
+    assert_equal [@cpub, @cpriv], @vis.constants
   end
 
   def test_remove_invisible_in_protected
@@ -922,6 +928,9 @@ class TestRDocContext < XrefTestCase
     @aprot = RDoc::Attr.new nil, 'prot', 'RW', nil
     @apriv = RDoc::Attr.new nil, 'priv', 'RW', nil
 
+    @cpub  = RDoc::Constant.new 'CONST_PUBLIC', nil, nil
+    @cpriv = RDoc::Constant.new 'CONST_PRIVATE', nil, nil
+
     @vis = RDoc::NormalClass.new 'Vis'
     @vis.add_method @pub
     @vis.add_method @prot
@@ -931,11 +940,16 @@ class TestRDocContext < XrefTestCase
     @vis.add_attribute @aprot
     @vis.add_attribute @apriv
 
+    @vis.add_constant @cpub
+    @vis.add_constant @cpriv
+
     @prot.visibility = :protected
     @priv.visibility = :private
 
     @aprot.visibility = :protected
     @apriv.visibility = :private
+
+    @cpriv.visibility = :private
   end
 
 end
diff --git a/test/rdoc/test_rdoc_markup_to_html.rb b/test/rdoc/test_rdoc_markup_to_html.rb
index fd8a7bb0ca..b5edeb04d2 100644
--- a/test/rdoc/test_rdoc_markup_to_html.rb
+++ b/test/rdoc/test_rdoc_markup_to_html.rb
@@ -452,6 +452,9 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
   end
 
   def test_accept_verbatim_nl_after_backslash
+    # TODO: Remove "skip" after the issue is resolved: https://github.com/jruby/jruby/issues/4787
+    # This "skip" is for strange behavior around escaped newline on JRuby
+    skip if defined? JRUBY_VERSION
     verb = @RM::Verbatim.new("a = 1 if first_flag_var and \\\n", "  this_is_flag_var\n")
 
     @to.start_accepting
diff --git a/test/rdoc/test_rdoc_parser_c.rb b/test/rdoc/test_rdoc_parser_c.rb
index d0732597dc..5341da16e2 100644
--- a/test/rdoc/test_rdoc_parser_c.rb
+++ b/test/rdoc/test_rdoc_parser_c.rb
@@ -1037,7 +1037,7 @@ Init_Foo(void) {
                  other_function.comment.text
     assert_equal '()', other_function.params
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "VALUE\nother_function() {\n}", code
   end
@@ -1107,7 +1107,7 @@ Init_Foo(void) {
                  other_function.comment.text
     assert_equal '()', other_function.params
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "VALUE\nother_function() {\n}", code
   end
@@ -1141,7 +1141,7 @@ Init_Foo(void) {
     assert_equal '()', other_function.params
     assert_equal 8, other_function.line
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "VALUE\nrb_other_function() {\n}", code
   end
@@ -1174,7 +1174,7 @@ Init_Foo(void) {
     assert_equal '()', other_function.params
     assert_equal 4, other_function.line
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "#define other_function rb_other_function", code
   end
@@ -1314,7 +1314,7 @@ Init_Foo(void) {
                  other_function.comment.text
     assert_equal '()', other_function.params
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "DLL_LOCAL VALUE\nother_function() {\n}", code
   end
diff --git a/test/rdoc/test_rdoc_parser_ruby.rb b/test/rdoc/test_rdoc_parser_ruby.rb
index 55c03d4628..cda407ed51 100644
--- a/test/rdoc/test_rdoc_parser_ruby.rb
+++ b/test/rdoc/test_rdoc_parser_ruby.rb
@@ -74,7 +74,7 @@ class C; end
 
     comment = parser.collect_first_comment
 
-    assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n\n", @top_level), comment
+    assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n", @top_level), comment
   end
 
   def test_get_class_or_module
@@ -84,7 +84,7 @@ class C; end
     cont, name_t, given_name = util_parser('A')    .get_class_or_module ctxt
 
     assert_equal ctxt, cont
-    assert_equal 'A', name_t.text
+    assert_equal 'A', name_t[:text]
     assert_equal 'A', given_name
 
     cont, name_t, given_name = util_parser('B::C') .get_class_or_module ctxt
@@ -92,16 +92,16 @@ class C; end
     b = @store.find_module_named('B')
     assert_equal b, cont
     assert_equal [@top_level], b.in_files
-    assert_equal 'C', name_t.text
+    assert_equal 'C', name_t[:text]
     assert_equal 'B::C', given_name
 
     cont, name_t, given_name = util_parser('D:: E').get_class_or_module ctxt
 
     assert_equal @store.find_module_named('D'), cont
-    assert_equal 'E', name_t.text
+    assert_equal 'E', name_t[:text]
     assert_equal 'D::E', given_name
 
-    assert_raises NoMethodError do
+    assert_raises RDoc::Error do
       util_parser("A::\nB").get_class_or_module ctxt
     end
   end
@@ -1194,10 +1194,12 @@ EOF
     assert_equal klass.current_section, foo.section
 
     stream = [
-      tk(:COMMENT, 0, 1, 1, nil,
-         "# File #{@top_level.relative_name}, line 1"),
-      RDoc::Parser::Ruby::NEWLINE_TOKEN,
-      tk(:SPACE,   0, 1, 1, nil, ''),
+      {
+        :line_no => 1, :char_no => 1, :kind => :on_comment,
+        :text => "# File #{@top_level.relative_name}, line 1"
+      },
+      { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+      { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }
     ]
 
     assert_equal stream, foo.token_stream
@@ -1358,6 +1360,33 @@ A::B::C = 1
     assert_equal 'comment', c.comment
   end
 
+  def test_parse_constant_with_bracket
+    util_parser <<-RUBY
+class Klass
+end
+
+class Klass2
+  CONSTANT = Klass
+end
+
+class Klass3
+  CONSTANT_2 = {}
+  CONSTANT_2[1] = Klass
+end
+    RUBY
+
+    @parser.scan
+
+    klass = @store.find_class_named 'Klass'
+    klass2 = @store.find_class_named 'Klass2'
+    klass3 = @store.find_class_named 'Klass3'
+    constant = klass2.find_module_named 'CONSTANT'
+    constant2 = klass3.find_module_named 'CONSTANT_2'
+    assert_equal klass, klass2.constants.first.is_alias_for
+    refute_equal klass, klass3.constants.first.is_alias_for
+    assert_nil klass3.find_module_named 'CONSTANT_2'
+  end
+
   def test_parse_extend_or_include_extend
     klass = RDoc::NormalClass.new 'C'
     klass.parent = @top_level
@@ -1434,20 +1463,30 @@ A::B::C = 1
     assert_equal klass.current_section, foo.section
 
     stream = [
-      tk(:COMMENT,     0, 1, 1,  nil,
-         "# File #{@top_level.relative_name}, line 1"),
-      RDoc::Parser::Ruby::NEWLINE_TOKEN,
-      tk(:SPACE,       0, 1, 1,  nil, ''),
-      tk(:IDENTIFIER,  0, 1, 0,  'add_my_method', 'add_my_method'),
-      tk(:SPACE,       0, 1, 13, nil, ' '),
-      tk(:SYMBOL,      0, 1, 14, nil, ':foo'),
-      tk(:COMMA,       0, 1, 18, nil, ','),
-      tk(:SPACE,       0, 1, 19, nil, ' '),
-      tk(:SYMBOL,      0, 1, 20, nil, ':bar'),
-      tk(:NL,          0, 1, 24, nil, "\n"),
+      {
+        :line_no => 1, :char_no => 1, :kind => :on_comment,
+        :text => "# File #{@top_level.relative_name}, line 1"
+      },
+      { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+      { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+      { :line_no => 1, :char_no => 0, :kind => :on_ident, :text => 'add_my_method' },
+      { :line_no => 1, :char_no => 13, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 14, :kind => :on_symbol, :text => ':foo' },
+      { :line_no => 1, :char_no => 18, :kind => :on_comma, :text => ',' },
+      { :line_no => 1, :char_no => 19, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 20, :kind => :on_symbol, :text => ':bar' },
+      { :line_no => 1, :char_no => 24, :kind => :on_nl, :text => "\n" }
     ]
+    parsed_stream = foo.token_stream.map { |t|
+      {
+        :line_no => t[:line_no],
+        :char_no => t[:char_no],
+        :kind => t[:kind],
+        :text => t[:text]
+      }
+    }
 
-    assert_equal stream, foo.token_stream
+    assert_equal stream, parsed_stream
   end
 
   def test_parse_meta_method_block
@@ -1468,7 +1507,10 @@ end
 
     @parser.parse_meta_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
 
-    assert_equal tk(:NL, 0, 3, 3, 3, "\n"), @parser.get_tk
+    rest = { :line_no => 3, :char_no => 3, :kind => :on_nl, :text => "\n" }
+    tk = @parser.get_tk
+    tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => tk[:kind], :text => tk[:text] }
+    assert_equal rest, tk
   end
 
   def test_parse_meta_method_define_method
@@ -1631,23 +1673,30 @@ end
     assert_equal klass.current_section, foo.section
 
     stream = [
-      tk(:COMMENT,     0, 1, 1,  nil,
-         "# File #{@top_level.relative_name}, line 1"),
-      RDoc::Parser::Ruby::NEWLINE_TOKEN,
-      tk(:SPACE,       0, 1, 1,  nil,   ''),
-      tk(:DEF,         0, 1, 0,  'def', 'def'),
-      tk(:SPACE,       3, 1, 3,  nil,   ' '),
-      tk(:IDENTIFIER,  4, 1, 4,  'foo', 'foo'),
-      tk(:LPAREN,      7, 1, 7,  nil,   '('),
-      tk(:RPAREN,      8, 1, 8,  nil,   ')'),
-      tk(:SPACE,       9, 1, 9,  nil,   ' '),
-      tk(:COLON,      10, 1, 10, nil,   ':'),
-      tk(:IDENTIFIER, 11, 1, 11, 'bar', 'bar'),
-      tk(:SPACE,      14, 1, 14, nil,   ' '),
-      tk(:END,        15, 1, 15, 'end', 'end'),
+      {
+        :line_no => 1, :char_no => 1, :kind => :on_comment,
+        :text => "# File #{@top_level.relative_name}, line 1" },
+      { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+      { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+      { :line_no => 1, :char_no => 0, :kind => :on_kw, :text => 'def' },
+      { :line_no => 1, :char_no => 3, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 4, :kind => :on_ident, :text => 'foo' },
+      { :line_no => 1, :char_no => 7, :kind => :on_lparen, :text => '(' },
+      { :line_no => 1, :char_no => 8, :kind => :on_rparen, :text => ')' },
+      { :line_no => 1, :char_no => 9, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 10, :kind => :on_symbol, :text => ':bar' },
+      { :line_no => 1, :char_no => 14, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 15, :kind => :on_kw, :text => 'end' }
     ]
-
-    assert_equal stream, foo.token_stream
+    parsed_stream = foo.token_stream.map { |t|
+      {
+        :line_no => t[:line_no],
+        :char_no => t[:char_no],
+        :kind => t[:kind],
+        :text => t[:text]
+      }
+    }
+    assert_equal stream, parsed_stream
   end
 
   def test_parse_redefinable_methods
@@ -1664,8 +1713,8 @@ end
     end
 
     klass.method_list.each do |method|
-      assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5]
-      assert_includes redefinable_ops, method.token_stream[5].text
+      assert_equal :on_ident, method.token_stream[5][:kind]
+      assert_includes redefinable_ops, method.token_stream[5][:text]
     end
   end
 
@@ -1909,6 +1958,20 @@ end
     assert_equal '(arg1, arg2, arg3)', foo.params
   end
 
+  def test_parse_method_parameters_with_paren_comment_continue
+    klass = RDoc::NormalClass.new 'Foo'
+    klass.parent = @top_level
+
+    util_parser "def foo(arg1, arg2, # some useful comment\narg3)\nend"
+
+    tk = @parser.get_tk
+
+    @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, @comment
+
+    foo = klass.method_list.first
+    assert_equal '(arg1, arg2, arg3)', foo.params
+  end
+
   def test_parse_method_star
     klass = RDoc::NormalClass.new 'Foo'
     klass.parent = @top_level
@@ -2091,22 +2154,34 @@ end
     assert_equal 2, x.method_list.length
     a = x.method_list.first
 
+
     expected = [
-      tk(:COMMENT,     0, 2, 1, nil,   "# File #{@filename}, line 2"),
-      tk(:NL,          0, 0, 0, nil,   "\n"),
-      tk(:SPACE,       0, 1, 1, nil,   ''),
-      tk(:DEF,         8, 2, 0, 'def', 'def'),
-      tk(:SPACE,      11, 2, 3, nil,   ' '),
-      tk(:IDENTIFIER, 12, 2, 4, 'a',   'a'),
-      tk(:NL,         13, 2, 5, nil,   "\n"),
-      tk(:REGEXP,     14, 3, 0, nil,   '%r{#}'),
-      tk(:NL,         19, 3, 5, nil,   "\n"),
-      tk(:DREGEXP,    20, 4, 0, nil,   '%r{#{}}'),
-      tk(:NL,         27, 4, 7, nil,   "\n"),
-      tk(:END,        28, 5, 0, 'end', 'end'),
+      {
+        :line_no => 2, :char_no => 1, :kind => :on_comment,
+        :text => "# File #{@filename}, line 2"
+      },
+      { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+      { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+      { :line_no => 2, :char_no => 0, :kind => :on_kw, :text => 'def' },
+      { :line_no => 2, :char_no => 3, :kind => :on_sp, :text => ' ' },
+      { :line_no => 2, :char_no => 4, :kind => :on_ident, :text => 'a' },
+      { :line_no => 2, :char_no => 5, :kind => :on_nl, :text => "\n" },
+      { :line_no => 3, :char_no => 0, :kind => :on_regexp, :text => '%r{#}' },
+      { :line_no => 3, :char_no => 5, :kind => :on_nl, :text => "\n" },
+      { :line_no => 4, :char_no => 0, :kind => :on_regexp, :text => '%r{#{}}' },
+      { :line_no => 4, :char_no => 7, :kind => :on_nl, :text => "\n" },
+      { :line_no => 5, :char_no => 0, :kind => :on_kw, :text => 'end' }
     ]
+    parsed_stream = a.token_stream.map { |tk|
+      {
+        :line_no => tk[:line_no],
+        :char_no => tk[:char_no],
+        :kind => tk[:kind],
+        :text => tk[:text]
+      }
+    }
 
-    assert_equal expected, a.token_stream
+    assert_equal expected, parsed_stream
   end
 
   def test_parse_statements_encoding
@@ -2286,6 +2361,9 @@ class Foo
   SIXTH_CONSTANT = #{sixth_constant}
 
   SEVENTH_CONSTANT = proc { |i| begin i end }
+
+  EIGHTH_CONSTANT = "a" \\
+                    "b"
 end
 EOF
 
@@ -2331,6 +2409,11 @@ EOF
     assert_equal 'SEVENTH_CONSTANT', constant.name
     assert_equal "proc { |i| begin i end }", constant.value
     assert_equal @top_level, constant.file
+
+    constant = constants[7]
+    assert_equal 'EIGHTH_CONSTANT', constant.name
+    assert_equal "\"a\" \\\n\"b\"", constant.value
+    assert_equal @top_level, constant.file
   end
 
   def test_parse_statements_identifier_attr
@@ -2504,7 +2587,7 @@ EXPTECTED
     util_parser <<RUBY
 class Foo
   def blah()
-    <<~EOM if true
+    <<-EOM if true
     EOM
   end
 end
@@ -2512,7 +2595,7 @@ RUBY
 
     expected = <<EXPTECTED
   <span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>()
-    <span class="ruby-identifier">&lt;&lt;~EOM</span> <span class="ruby-keyword">if</span> <span class="ruby-keyword">true</span>
+    <span class="ruby-identifier">&lt;&lt;-EOM</span> <span class="ruby-keyword">if</span> <span class="ruby-keyword">true</span>
 <span class="ruby-value"></span><span class="ruby-identifier">    EOM</span>
   <span class="ruby-keyword">end</span>
 EXPTECTED
@@ -2525,7 +2608,7 @@ EXPTECTED
 
     blah = foo.method_list.first
     markup_code = blah.markup_code.sub(/^.*\n/, '')
-    assert_equal markup_code, expected
+    assert_equal expected, markup_code
   end
 
   def test_parse_statements_method_oneliner_with_regexp
@@ -2770,17 +2853,21 @@ RUBY
   end
 
   def test_parse_symbol_in_arg
-    util_parser ':blah "blah" "#{blah}" blah'
+    util_parser '[:blah, "blah", "#{blah}", blah]'
+    @parser.get_tk # skip '['
 
     assert_equal 'blah', @parser.parse_symbol_in_arg
+    @parser.get_tk # skip ','
 
     @parser.skip_tkspace
 
     assert_equal 'blah', @parser.parse_symbol_in_arg
+    @parser.get_tk # skip ','
 
     @parser.skip_tkspace
 
     assert_equal nil, @parser.parse_symbol_in_arg
+    @parser.get_tk # skip ','
 
     @parser.skip_tkspace
 
@@ -2888,7 +2975,7 @@ end
     assert_equal 'category', directive
     assert_equal 'test', value
 
-    assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+    assert_equal nil, parser.get_tk
   end
 
   def test_read_directive_allow
@@ -2898,7 +2985,7 @@ end
 
     assert_nil directive
 
-    assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+    assert_equal nil, parser.get_tk
   end
 
   def test_read_directive_empty
@@ -2908,7 +2995,7 @@ end
 
     assert_nil directive
 
-    assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+    assert_equal nil, parser.get_tk
   end
 
   def test_read_directive_no_comment
@@ -2918,18 +3005,18 @@ end
 
     assert_nil directive
 
-    assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+    assert_equal nil, parser.get_tk
   end
 
   def test_read_directive_one_liner
-    parser = util_parser '; end # :category: test'
+    parser = util_parser 'AAA = 1 # :category: test'
 
     directive, value = parser.read_directive %w[category]
 
     assert_equal 'category', directive
     assert_equal 'test', value
 
-    assert_kind_of RDoc::RubyToken::TkSEMICOLON, parser.get_tk
+    assert_equal :on_const, parser.get_tk[:kind]
   end
 
   def test_read_documentation_modifiers
@@ -2974,10 +3061,10 @@ end
 
   def test_sanity_integer
     util_parser '1'
-    assert_equal '1', @parser.get_tk.text
+    assert_equal '1', @parser.get_tk[:text]
 
     util_parser '1.0'
-    assert_equal '1.0', @parser.get_tk.text
+    assert_equal '1.0', @parser.get_tk[:text]
   end
 
   def test_sanity_interpolation
@@ -2986,7 +3073,7 @@ end
 
     while tk = @parser.get_tk do last_tk = tk end
 
-    assert_equal "\n", last_tk.text
+    assert_equal 'end', last_tk[:text]
   end
 
   # If you're writing code like this you're doing it wrong
@@ -2994,15 +3081,15 @@ end
   def test_sanity_interpolation_crazy
     util_parser '"#{"#{"a")}" if b}"'
 
-    assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk.text
-    assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
+    assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk[:text]
+    assert_equal nil, @parser.get_tk
   end
 
   def test_sanity_interpolation_curly
     util_parser '%{ #{} }'
 
-    assert_equal '%{ #{} }', @parser.get_tk.text
-    assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
+    assert_equal '%{ #{} }', @parser.get_tk[:text]
+    assert_equal nil, @parser.get_tk
   end
 
   def test_sanity_interpolation_format
@@ -3537,6 +3624,53 @@ end
     assert_equal 2, public_method_count
   end
 
+  def test_scan_constant_visibility
+    util_parser <<-RUBY
+class C
+   CONST_A = 123
+
+   CONST_B = 234
+   private_constant :CONST_B
+
+   CONST_C = 345
+   public_constant :CONST_C
+end
+    RUBY
+
+    @parser.scan
+
+    c = @store.find_class_named 'C'
+    const_a, const_b, const_c, const_d = c.constants.sort_by(&:name)
+
+    assert_equal 'CONST_A', const_a.name
+    assert_equal :public, const_a.visibility
+
+    assert_equal 'CONST_B', const_b.name
+    assert_equal :private, const_b.visibility
+
+    assert_equal 'CONST_C', const_c.name
+    assert_equal :public, const_c.visibility
+  end
+
+  def test_document_after_rescue_inside_paren
+    util_parser <<-RUBY
+class C
+  attr_accessor :sample if (1.inexistent_method rescue false)
+  # first
+  # second
+  def a
+  end
+end
+    RUBY
+
+    @parser.scan
+
+    c = @store.find_class_named 'C'
+
+    c_a = c.find_method_named 'a'
+    assert_equal "first\nsecond", c_a.comment.text
+  end
+
   def test_singleton_method_via_eigenclass
     util_parser <<-RUBY
 class C
diff --git a/test/rdoc/test_rdoc_ruby_lex.rb b/test/rdoc/test_rdoc_ruby_lex.rb
deleted file mode 100644
index 9ab5844b54..0000000000
--- a/test/rdoc/test_rdoc_ruby_lex.rb
+++ /dev/null
@@ -1,1095 +0,0 @@
-# coding: UTF-8
-# frozen_string_literal: false
-
-require 'rdoc/test_case'
-
-class TestRDocRubyLex < RDoc::TestCase
-
-  def setup
-    @TK = RDoc::RubyToken
-  end
-
-  def test_token_position
-    tokens = RDoc::RubyLex.tokenize '[ 1, :a, nil ]', nil
-
-    assert_equal '[', tokens[0].text
-    assert_equal 0, tokens[0].seek
-    assert_equal 1, tokens[0].line_no
-    assert_equal 0, tokens[0].char_no
-    assert_equal '1', tokens[2].text
-    assert_equal 2, tokens[2].seek
-    assert_equal 1, tokens[2].line_no
-    assert_equal 2, tokens[2].char_no
-    assert_equal ':a', tokens[5].text
-    assert_equal 5, tokens[5].seek
-    assert_equal 1, tokens[5].line_no
-    assert_equal 5, tokens[5].char_no
-    assert_equal 'nil', tokens[8].text
-    assert_equal 9, tokens[8].seek
-    assert_equal 1, tokens[8].line_no
-    assert_equal 9, tokens[8].char_no
-    assert_equal ']', tokens[10].text
-    assert_equal 13, tokens[10].seek
-    assert_equal 1, tokens[10].line_no
-    assert_equal 13, tokens[10].char_no
-  end
-
-  def test_class_tokenize
-    tokens = RDoc::RubyLex.tokenize "def x() end", nil
-
-    expected = [
-      @TK::TkDEF       .new( 0, 1,  0, "def"),
-      @TK::TkSPACE     .new( 3, 1,  3, " "),
-      @TK::TkIDENTIFIER.new( 4, 1,  4, "x"),
-      @TK::TkLPAREN    .new( 5, 1,  5, "("),
-      @TK::TkRPAREN    .new( 6, 1,  6, ")"),
-      @TK::TkSPACE     .new( 7, 1,  7, " "),
-      @TK::TkEND       .new( 8, 1,  8, "end"),
-      @TK::TkNL        .new(11, 1, 11, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize___END__
-    tokens = RDoc::RubyLex.tokenize '__END__', nil
-
-    expected = [
-      @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'),
-      @TK::TkNL           .new(7, 1, 7, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize___ENCODING__
-    tokens = RDoc::RubyLex.tokenize '__ENCODING__', nil
-
-    expected = [
-      @TK::Tk__ENCODING__.new( 0, 1,  0, '__ENCODING__'),
-      @TK::TkNL          .new(12, 1, 12, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_character_literal
-    tokens = RDoc::RubyLex.tokenize "?c", nil
-
-    expected = [
-      @TK::TkCHAR.new( 0, 1,  0, "?c"),
-      @TK::TkNL  .new( 2, 1,  2, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_character_literal_with_escape
-    tokens = RDoc::RubyLex.tokenize "?\\s", nil
-
-    expected = [
-      @TK::TkCHAR.new( 0, 1,  0, "?\\s"),
-      @TK::TkNL  .new( 3, 1,  3, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_def_heredoc
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-def x
-  <<E
-Line 1
-Line 2
-E
-end
-    RUBY
-
-    expected = [
-      @TK::TkDEF       .new( 0, 1,  0, 'def'),
-      @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
-      @TK::TkNL        .new( 5, 1,  5, "\n"),
-      @TK::TkSPACE     .new( 6, 2,  0, '  '),
-
-      @TK::TkHEREDOCBEG.new( 8, 2,  2, '<<E'),
-      @TK::TkNL        .new(11, 2,  6, "\n"),
-      @TK::TkHEREDOC   .new(11, 2,  6, "Line 1\nLine 2\n"),
-      @TK::TkHEREDOCEND.new(27, 5, 26, "E\n"),
-      @TK::TkEND       .new(28, 6,  0, 'end'),
-      @TK::TkNL        .new(31, 6, 28, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_reserved_keyword_with_args
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-yield :foo
-super :bar
-defined? :baz
-    RUBY
-
-    expected = [
-      @TK::TkYIELD  .new( 0, 1,  0, "yield"),
-      @TK::TkSPACE  .new( 5, 1,  5, " "),
-      @TK::TkSYMBOL .new( 6, 1,  6, ":foo"),
-      @TK::TkNL     .new(10, 1, 10,  "\n"),
-      @TK::TkSUPER  .new(11, 2,  0, "super"),
-      @TK::TkSPACE  .new(16, 2,  5, " "),
-      @TK::TkSYMBOL .new(17, 2,  6, ":bar"),
-      @TK::TkNL     .new(21, 2, 11,  "\n"),
-      @TK::TkDEFINED.new(22, 3,  0, "defined?"),
-      @TK::TkSPACE  .new(30, 3,  8, " "),
-      @TK::TkSYMBOL .new(31, 3,  9, ":baz"),
-      @TK::TkNL     .new(35, 3, 22,  "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_hash_symbol
-    tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
-
-    expected = [
-      @TK::TkLBRACE.new( 0, 1,  0, '{'),
-      @TK::TkSPACE .new( 1, 1,  1, ' '),
-      @TK::TkSYMBOL.new( 2, 1,  2, 'class:'),
-      @TK::TkSTRING.new( 8, 1,  8, '"foo"'),
-      @TK::TkSPACE .new(13, 1, 13, ' '),
-      @TK::TkRBRACE.new(14, 1, 14, '}'),
-      @TK::TkNL    .new(15, 1, 15, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_double_colon_is_not_hash_symbol
-    tokens = RDoc::RubyLex.tokenize 'self.class::Row', nil
-
-    expected = [
-      @TK::TkSELF      .new( 0, 1,  0, "self"),
-      @TK::TkDOT       .new( 4, 1,  4, "."),
-      @TK::TkIDENTIFIER.new( 5, 1,  5, "class"),
-      @TK::TkCOLON2    .new(10, 1, 10, "::"),
-      @TK::TkCONSTANT  .new(12, 1, 12, "Row"),
-      @TK::TkNL        .new(15, 1, 15, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_safe_nav_operator
-    tokens = RDoc::RubyLex.tokenize 'receiver&.meth', nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, "receiver"),
-      @TK::TkSAFENAV   .new( 8, 1,  8, "&."),
-      @TK::TkIDENTIFIER.new(10, 1, 10, "meth"),
-      @TK::TkNL        .new(14, 1, 14, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_hash_rocket
-    tokens = RDoc::RubyLex.tokenize '{ :class => "foo" }', nil
-
-    expected = [
-      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkSYMBOL    .new( 2, 1,  2, ':class'),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHASHROCKET.new( 9, 1,  9, '=>'),
-      @TK::TkSPACE     .new(11, 1, 11, ' '),
-      @TK::TkSTRING    .new(12, 1, 12, '"foo"'),
-      @TK::TkSPACE     .new(17, 1, 17, ' '),
-      @TK::TkRBRACE    .new(18, 1, 18, '}'),
-      @TK::TkNL        .new(19, 1, 19, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_heredoc_CR_NL
-    tokens = RDoc::RubyLex.tokenize <<-RUBY, nil
-string = <<-STRING\r
-Line 1\r
-Line 2\r
-  STRING\r
-    RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
-      @TK::TkSPACE     .new( 6, 1,  6, ' '),
-      @TK::TkASSIGN    .new( 7, 1,  7, '='),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
-      @TK::TkSPACE     .new(18, 1, 18, "\r"),
-      @TK::TkNL        .new(19, 1, 19, "\n"),
-      @TK::TkHEREDOC   .new(19, 1, 19,
-                            %Q{Line 1\nLine 2\n}),
-      @TK::TkHEREDOCEND.new(45, 4, 36, "  STRING\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_opassign
-    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-a %= b
-a /= b
-a -= b
-a += b
-a |= b
-a &= b
-a >>= b
-a <<= b
-a *= b
-a &&= b
-a ||= b
-a **= b
-RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1, 0, "a"),
-      @TK::TkSPACE     .new( 1, 1, 1, " "),
-      @TK::TkOPASGN    .new( 2, 1, 2, "%"),
-      @TK::TkSPACE     .new( 4, 1, 4, " "),
-      @TK::TkIDENTIFIER.new( 5, 1, 5, "b"),
-      @TK::TkNL        .new( 6, 1, 6, "\n"),
-      @TK::TkIDENTIFIER.new( 7, 2, 0, "a"),
-      @TK::TkSPACE     .new( 8, 2, 1, " "),
-      @TK::TkOPASGN    .new( 9, 2, 2, "/"),
-      @TK::TkSPACE     .new( 11, 2, 4, " "),
-      @TK::TkIDENTIFIER.new( 12, 2, 5, "b"),
-      @TK::TkNL        .new( 13, 2, 7, "\n"),
-      @TK::TkIDENTIFIER.new( 14, 3, 0, "a"),
-      @TK::TkSPACE     .new( 15, 3, 1, " "),
-      @TK::TkOPASGN    .new( 16, 3, 2, "-"),
-      @TK::TkSPACE     .new( 18, 3, 4, " "),
-      @TK::TkIDENTIFIER.new( 19, 3, 5, "b"),
-      @TK::TkNL        .new( 20, 3, 14, "\n"),
-      @TK::TkIDENTIFIER.new( 21, 4, 0, "a"),
-      @TK::TkSPACE     .new( 22, 4, 1, " "),
-      @TK::TkOPASGN    .new( 23, 4, 2, "+"),
-      @TK::TkSPACE     .new( 25, 4, 4, " "),
-      @TK::TkIDENTIFIER.new( 26, 4, 5, "b"),
-      @TK::TkNL        .new( 27, 4, 21, "\n"),
-      @TK::TkIDENTIFIER.new( 28, 5, 0, "a"),
-      @TK::TkSPACE     .new( 29, 5, 1, " "),
-      @TK::TkOPASGN    .new( 30, 5, 2, "|"),
-      @TK::TkSPACE     .new( 32, 5, 4, " "),
-      @TK::TkIDENTIFIER.new( 33, 5, 5, "b"),
-      @TK::TkNL        .new( 34, 5, 28, "\n"),
-      @TK::TkIDENTIFIER.new( 35, 6, 0, "a"),
-      @TK::TkSPACE     .new( 36, 6, 1, " "),
-      @TK::TkOPASGN    .new( 37, 6, 2, "&"),
-      @TK::TkSPACE     .new( 39, 6, 4, " "),
-      @TK::TkIDENTIFIER.new( 40, 6, 5, "b"),
-      @TK::TkNL        .new( 41, 6, 35, "\n"),
-      @TK::TkIDENTIFIER.new( 42, 7, 0, "a"),
-      @TK::TkSPACE     .new( 43, 7, 1, " "),
-      @TK::TkOPASGN    .new( 44, 7, 2, ">>"),
-      @TK::TkSPACE     .new( 47, 7, 5, " "),
-      @TK::TkIDENTIFIER.new( 48, 7, 6, "b"),
-      @TK::TkNL        .new( 49, 7, 42, "\n"),
-      @TK::TkIDENTIFIER.new( 50, 8, 0, "a"),
-      @TK::TkSPACE     .new( 51, 8, 1, " "),
-      @TK::TkOPASGN    .new( 52, 8, 2, "<<"),
-      @TK::TkSPACE     .new( 55, 8, 5, " "),
-      @TK::TkIDENTIFIER.new( 56, 8, 6, "b"),
-      @TK::TkNL        .new( 57, 8, 50, "\n"),
-      @TK::TkIDENTIFIER.new( 58, 9, 0, "a"),
-      @TK::TkSPACE     .new( 59, 9, 1, " "),
-      @TK::TkOPASGN    .new( 60, 9, 2, "*"),
-      @TK::TkSPACE     .new( 62, 9, 4, " "),
-      @TK::TkIDENTIFIER.new( 63, 9, 5, "b"),
-      @TK::TkNL        .new( 64, 9, 58, "\n"),
-      @TK::TkIDENTIFIER.new( 65, 10, 0, "a"),
-      @TK::TkSPACE     .new( 66, 10, 1, " "),
-      @TK::TkOPASGN    .new( 67, 10, 2, "&&"),
-      @TK::TkSPACE     .new( 70, 10, 5, " "),
-      @TK::TkIDENTIFIER.new( 71, 10, 6, "b"),
-      @TK::TkNL        .new( 72, 10, 65, "\n"),
-      @TK::TkIDENTIFIER.new( 73, 11, 0, "a"),
-      @TK::TkSPACE     .new( 74, 11, 1, " "),
-      @TK::TkOPASGN    .new( 75, 11, 2, "||"),
-      @TK::TkSPACE     .new( 78, 11, 5, " "),
-      @TK::TkIDENTIFIER.new( 79, 11, 6, "b"),
-      @TK::TkNL        .new( 80, 11, 73, "\n"),
-      @TK::TkIDENTIFIER.new( 81, 12, 0, "a"),
-      @TK::TkSPACE     .new( 82, 12, 1, " "),
-      @TK::TkOPASGN    .new( 83, 12, 2, "**"),
-      @TK::TkSPACE     .new( 86, 12, 5, " "),
-      @TK::TkIDENTIFIER.new( 87, 12, 6, "b"),
-      @TK::TkNL        .new( 88, 12, 81, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_heredoc_call
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-string = <<-STRING.chomp
-Line 1
-Line 2
-  STRING
-    RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
-      @TK::TkSPACE     .new( 6, 1,  6, ' '),
-      @TK::TkASSIGN    .new( 7, 1,  7, '='),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
-      @TK::TkDOT       .new(18, 1, 18, '.'),
-      @TK::TkIDENTIFIER.new(19, 1, 19, 'chomp'),
-      @TK::TkNL        .new(24, 1, 24, "\n"),
-      @TK::TkHEREDOC   .new(24, 1, 24, "Line 1\nLine 2\n"),
-      @TK::TkHEREDOCEND.new(47, 4, 39, "  STRING\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_heredoc_indent
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-string = <<-STRING
-Line 1
-Line 2
-  STRING
-    RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
-      @TK::TkSPACE     .new( 6, 1,  6, ' '),
-      @TK::TkASSIGN    .new( 7, 1,  7, '='),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-
-
-      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
-      @TK::TkNL        .new(18, 1, 18, "\n"),
-      @TK::TkHEREDOC   .new(18, 1, 18, "Line 1\nLine 2\n"),
-      @TK::TkHEREDOCEND.new(41, 4, 33, "  STRING\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_heredoc_missing_end
-    e = assert_raises RDoc::RubyLex::Error do
-      RDoc::RubyLex.tokenize <<-'RUBY', nil
->> string1 = <<-TXT
->" That's swell
->" TXT
-      RUBY
-    end
-
-    assert_equal 'Missing terminating TXT for string', e.message
-  end
-
-  def test_class_tokenize_heredoc_percent_N
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-a b <<-U
-%N
-U
-    RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
-      @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkHEREDOCBEG.new( 4, 1,  4, '<<-U'),
-      @TK::TkNL        .new( 8, 1,  8, "\n"),
-      @TK::TkHEREDOC   .new( 8, 1,  8, "%N\n"),
-      @TK::TkHEREDOCEND.new(13, 3, 12, "U\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_identifier_high_unicode
-    tokens = RDoc::RubyLex.tokenize '𝖒', nil
-
-    expected = @TK::TkIDENTIFIER.new(0, 1, 0, '𝖒')
-
-    assert_equal expected, tokens.first
-  end
-
-  def test_class_tokenize_lambda
-    tokens = RDoc::RubyLex.tokenize 'a = -> x, y { x + y }', nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkASSIGN    .new( 2, 1,  2, '='),
-      @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkLAMBDA    .new( 4, 1,  4, '->'),
-      @TK::TkSPACE     .new( 6, 1,  6, ' '),
-      @TK::TkIDENTIFIER.new( 7, 1,  7, 'x'),
-      @TK::TkCOMMA     .new( 8, 1,  8, ','),
-      @TK::TkSPACE     .new( 9, 1,  9, ' '),
-      @TK::TkIDENTIFIER.new(10, 1, 10, 'y'),
-      @TK::TkSPACE     .new(11, 1, 11, ' '),
-      @TK::TkfLBRACE   .new(12, 1, 12, '{'),
-      @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkIDENTIFIER.new(14, 1, 14, 'x'),
-      @TK::TkSPACE     .new(15, 1, 15, ' '),
-      @TK::TkPLUS      .new(16, 1, 16, '+'),
-      @TK::TkSPACE     .new(17, 1, 17, ' '),
-      @TK::TkIDENTIFIER.new(18, 1, 18, 'y'),
-      @TK::TkSPACE     .new(19, 1, 19, ' '),
-      @TK::TkRBRACE    .new(20, 1, 20, '}'),
-      @TK::TkNL        .new(21, 1, 21, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_1
-    tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new(0, 1, 0, 'v'),
-      @TK::TkMOD.new(       1, 1, 1, '%'),
-      @TK::TkINTEGER.new(   2, 1, 2, '10'),
-      @TK::TkEQ.new(        4, 1, 4, '=='),
-      @TK::TkINTEGER.new(   6, 1, 6, '10'),
-      @TK::TkNL.new(        8, 1, 8, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_r
-    tokens = RDoc::RubyLex.tokenize '%r[hi]', nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, '%r[hi]'),
-      @TK::TkNL    .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_r_with_slash
-    tokens = RDoc::RubyLex.tokenize '%r/hi/', nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, '%r/hi/'),
-      @TK::TkNL    .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_large_q
-    tokens = RDoc::RubyLex.tokenize '%Q/hi/', nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1,  0, '%Q/hi/'),
-      @TK::TkNL    .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_large_q_with_double_quote
-    tokens = RDoc::RubyLex.tokenize '%Q"hi"', nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1,  0, '%Q"hi"'),
-      @TK::TkNL    .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_w
-    tokens = RDoc::RubyLex.tokenize '%w[hi]', nil
-
-    expected = [
-      @TK::TkDSTRING.new( 0, 1,  0, '%w[hi]'),
-      @TK::TkNL     .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_w_quote
-    tokens = RDoc::RubyLex.tokenize '%w"hi"', nil
-
-    expected = [
-      @TK::TkDSTRING.new( 0, 1,  0, '%w"hi"'),
-      @TK::TkNL     .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_hash_rocket
-    tokens = RDoc::RubyLex.tokenize "{ :foo=> 1 }", nil
-
-    expected = [
-      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkSYMBOL    .new( 2, 1,  2, ':foo'),
-      @TK::TkHASHROCKET.new( 6, 1,  6, '=>'),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkINTEGER   .new( 9, 1,  9, '1'),
-      @TK::TkSPACE     .new(10, 1, 10, ' '),
-      @TK::TkRBRACE    .new(11, 1, 11, '}'),
-      @TK::TkNL        .new(12, 1, 12, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_sign_quote
-    tokens = RDoc::RubyLex.tokenize '%%hi%', nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1, 0, '%%hi%'),
-      @TK::TkNL    .new( 5, 1, 5, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp
-    tokens = RDoc::RubyLex.tokenize "/hay/", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/hay/"),
-      @TK::TkNL    .new( 5, 1,  5, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp_options
-    tokens = RDoc::RubyLex.tokenize "/hAY/i", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/hAY/i"),
-      @TK::TkNL    .new( 6, 1,  6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-
-    tokens = RDoc::RubyLex.tokenize "/hAY/ix", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/hAY/ix"),
-      @TK::TkNL    .new( 7, 1,  7, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp_backref
-    tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/[csh](..) [csh]\\1 in/"),
-      @TK::TkNL    .new(22, 1, 22, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp_escape
-    tokens = RDoc::RubyLex.tokenize "/\\//", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/\\//"),
-      @TK::TkNL    .new( 4, 1,  4, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_number_with_sign_character
-    tokens = RDoc::RubyLex.tokenize "+3--3r", nil
-
-    expected = [
-      @TK::TkINTEGER .new(0, 1, 0, "+3"),
-      @TK::TkMINUS   .new(2, 1, 2, "-"),
-      @TK::TkRATIONAL.new(3, 1, 3, "-3r"),
-      @TK::TkNL      .new(6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp_continuing_backslash
-    tokens = RDoc::RubyLex.tokenize "/(?<!\\\\)\\n\z/", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/(?<!\\\\)\\n\z/"),
-      @TK::TkNL    .new(12, 1, 12, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_single_quote_escape
-    tokens = RDoc::RubyLex.tokenize %q{'\\\\ \\' \\&'}, nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1,  0, %q{'\\\\ \\' \\&'}),
-      @TK::TkNL    .new(10, 1, 10, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_string
-    tokens = RDoc::RubyLex.tokenize "'hi'", nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1,  0, "'hi'"),
-      @TK::TkNL    .new( 4, 1,  4, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_string_with_escape
-    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-[
-  '\\',
-  '\'',
-  "'",
-  "\'\"\`",
-  "\#",
-  "\#{}",
-  "#",
-  "#{}",
-  /'"/,
-  /\'\"/,
-  /\//,
-  /\\/,
-  /\#/,
-  /\#{}/,
-  /#/,
-  /#{}/
-]
-RUBY
-
-    expected = [
-      @TK::TkLBRACK .new(  0,  1,   0, "["),
-      @TK::TkNL     .new(  1,  1,   1, "\n"),
-      @TK::TkSPACE  .new(  2,  2,   0, "  "),
-      @TK::TkSTRING .new(  4,  2,   2, "'\\\\'"),
-      @TK::TkCOMMA  .new(  8,  2,   6, ","),
-      @TK::TkNL     .new(  9,  2,   2, "\n"),
-      @TK::TkSPACE  .new( 10,  3,   0, "  "),
-      @TK::TkSTRING .new( 12,  3,   2, "'\\''"),
-      @TK::TkCOMMA  .new( 16,  3,   6, ","),
-      @TK::TkNL     .new( 17,  3,  10, "\n"),
-      @TK::TkSPACE  .new( 18,  4,   0, "  "),
-      @TK::TkSTRING .new( 20,  4,   2, "\"'\""),
-      @TK::TkCOMMA  .new( 23,  4,   5, ","),
-      @TK::TkNL     .new( 24,  4,  18, "\n"),
-      @TK::TkSPACE  .new( 25,  5,   0, "  "),
-      @TK::TkSTRING .new( 27,  5,   2, "\"\\'\\\"\\`\""),
-      @TK::TkCOMMA  .new( 35,  5,  10, ","),
-      @TK::TkNL     .new( 36,  5,  25, "\n"),
-      @TK::TkSPACE  .new( 37,  6,   0, "  "),
-      @TK::TkSTRING .new( 39,  6,   2, "\"\\#\""),
-      @TK::TkCOMMA  .new( 43,  6,   6, ","),
-      @TK::TkNL     .new( 44,  6,  37, "\n"),
-      @TK::TkSPACE  .new( 45,  7,   0, "  "),
-      @TK::TkSTRING .new( 47,  7,   2, "\"\\\#{}\""),
-      @TK::TkCOMMA  .new( 53,  7,   8, ","),
-      @TK::TkNL     .new( 54,  7,  45, "\n"),
-      @TK::TkSPACE  .new( 55,  8,   0, "  "),
-      @TK::TkSTRING .new( 57,  8,   2, "\"#\""),
-      @TK::TkCOMMA  .new( 60,  8,   5, ","),
-      @TK::TkNL     .new( 61,  8,  55, "\n"),
-      @TK::TkSPACE  .new( 62,  9,   0, "  "),
-      @TK::TkDSTRING.new( 64,  9,   2, "\"\#{}\""),
-      @TK::TkCOMMA  .new( 69,  9,   7, ","),
-      @TK::TkNL     .new( 70,  9,  62, "\n"),
-      @TK::TkSPACE  .new( 71, 10,   0, "  "),
-      @TK::TkREGEXP .new( 73, 10,   2, "/'\"/"),
-      @TK::TkCOMMA  .new( 77, 10,   6, ","),
-      @TK::TkNL     .new( 78, 10,  71, "\n"),
-      @TK::TkSPACE  .new( 79, 11,   0, "  "),
-      @TK::TkREGEXP .new( 81, 11,   2, "/\\'\\\"/"),
-      @TK::TkCOMMA  .new( 87, 11,   8, ","),
-      @TK::TkNL     .new( 88, 11,  79, "\n"),
-      @TK::TkSPACE  .new( 89, 12,   0, "  "),
-      @TK::TkREGEXP .new( 91, 12,   2, "/\\//"),
-      @TK::TkCOMMA  .new( 95, 12,   6, ","),
-      @TK::TkNL     .new( 96, 12,  89, "\n"),
-      @TK::TkSPACE  .new( 97, 13,   0, "  "),
-      @TK::TkREGEXP .new( 99, 13,   2, "/\\\\/"),
-      @TK::TkCOMMA  .new(103, 13,   6, ","),
-      @TK::TkNL     .new(104, 13,  97, "\n"),
-      @TK::TkSPACE  .new(105, 14,   0, "  "),
-      @TK::TkREGEXP .new(107, 14,   2, "/\\#/"),
-      @TK::TkCOMMA  .new(111, 14,   6, ","),
-      @TK::TkNL     .new(112, 14, 105, "\n"),
-      @TK::TkSPACE  .new(113, 15,   0, "  "),
-      @TK::TkREGEXP .new(115, 15,   2, "/\\\#{}/"),
-      @TK::TkCOMMA  .new(121, 15,   8, ","),
-      @TK::TkNL     .new(122, 15, 113, "\n"),
-      @TK::TkSPACE  .new(123, 16,   0, "  "),
-      @TK::TkREGEXP .new(125, 16,   2, "/#/"),
-      @TK::TkCOMMA  .new(128, 16,   5, ","),
-      @TK::TkNL     .new(129, 16, 123, "\n"),
-      @TK::TkSPACE  .new(130, 17,   0, "  "),
-      @TK::TkDREGEXP.new(132, 17,   2, "/\#{}/"),
-      @TK::TkNL     .new(137, 17,   7, "\n"),
-      @TK::TkRBRACK .new(138, 18,   0, "]"),
-      @TK::TkNL     .new(139, 18, 138, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_postfix_if_after_escaped_newline
-    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-def a
-  1 if true
-  1 \
-    if true
-end
-RUBY
-
-    expected = [
-      @TK::TkDEF       .new( 0, 1, 0,  "def"),
-      @TK::TkSPACE     .new( 3, 1, 3,  " "),
-      @TK::TkIDENTIFIER.new( 4, 1, 4,  "a"),
-      @TK::TkNL        .new( 5, 1, 5,  "\n"),
-      @TK::TkSPACE     .new( 6, 2, 0,  "  "),
-      @TK::TkINTEGER   .new( 8, 2, 2,  "1"),
-      @TK::TkSPACE     .new( 9, 2, 3,  " "),
-      @TK::TkIF_MOD    .new(10, 2, 4,  "if"),
-      @TK::TkSPACE     .new(12, 2, 6,  " "),
-      @TK::TkTRUE      .new(13, 2, 7,  "true"),
-      @TK::TkNL        .new(17, 2, 6,  "\n"),
-      @TK::TkSPACE     .new(18, 3, 0,  "  "),
-      @TK::TkINTEGER   .new(20, 3, 2,  "1"),
-      @TK::TkSPACE     .new(21, 3, 3,  " "),
-      @TK::TkBACKSLASH .new(22, 3, 4,  "\\"),
-      @TK::TkNL        .new(23, 3, 18, "\n"),
-      @TK::TkSPACE     .new(24, 4, 0,  "    "),
-      @TK::TkIF_MOD    .new(28, 4, 4,  "if"),
-      @TK::TkSPACE     .new(30, 4, 6,  " "),
-      @TK::TkTRUE      .new(31, 4, 7,  "true"),
-      @TK::TkNL        .new(35, 4, 24, "\n"),
-      @TK::TkEND       .new(36, 5, 0,  "end"),
-      @TK::TkNL        .new(39, 5, 36, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_backtick_with_escape
-    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-[
-  `\\`,
-  `\'\"\``,
-  `\#`,
-  `\#{}`,
-  `#`,
-  `#{}`
-]
-RUBY
-
-    expected = [
-      @TK::TkLBRACK  .new( 0, 1,  0, "["),
-      @TK::TkNL      .new( 1, 1,  1, "\n"),
-      @TK::TkSPACE   .new( 2, 2,  0, "  "),
-      @TK::TkXSTRING .new( 4, 2,  2, "`\\\\`"),
-      @TK::TkCOMMA   .new( 8, 2,  6, ","),
-      @TK::TkNL      .new( 9, 2,  2, "\n"),
-      @TK::TkSPACE   .new(10, 3,  0, "  "),
-      @TK::TkXSTRING .new(12, 3,  2, "`\\'\\\"\\``"),
-      @TK::TkCOMMA   .new(20, 3, 10, ","),
-      @TK::TkNL      .new(21, 3, 10, "\n"),
-      @TK::TkSPACE   .new(22, 4,  0, "  "),
-      @TK::TkXSTRING .new(24, 4,  2, "`\\#`"),
-      @TK::TkCOMMA   .new(28, 4,  6, ","),
-      @TK::TkNL      .new(29, 4, 22, "\n"),
-      @TK::TkSPACE   .new(30, 5,  0, "  "),
-      @TK::TkXSTRING .new(32, 5,  2, "`\\\#{}`"),
-      @TK::TkCOMMA   .new(38, 5,  8, ","),
-      @TK::TkNL      .new(39, 5, 30, "\n"),
-      @TK::TkSPACE   .new(40, 6,  0, "  "),
-      @TK::TkXSTRING .new(42, 6,  2, "`#`"),
-      @TK::TkCOMMA   .new(45, 6,  5, ","),
-      @TK::TkNL      .new(46, 6, 40, "\n"),
-      @TK::TkSPACE   .new(47, 7,  0, "  "),
-      @TK::TkDXSTRING.new(49, 7,  2, "`\#{}`"),
-      @TK::TkNL      .new(54, 7,  7, "\n"),
-      @TK::TkRBRACK  .new(55, 8,  0, "]"),
-      @TK::TkNL      .new(56, 8, 55, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_string_escape
-    tokens = RDoc::RubyLex.tokenize '"\\n"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\n\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\r"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\r\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\f"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\f\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\\\"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\\\\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\t"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\t\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\v"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\v\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\a\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\e"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\e\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\b"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\b\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\s"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\s\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\d"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\d\""), tokens.first
-
-  end
-
-  def test_class_tokenize_string_escape_control
-    tokens = RDoc::RubyLex.tokenize '"\\C-a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\C-a\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\c\\a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\c\\a\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\C-\\M-a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\C-\\M-a\""), tokens.first
-  end
-
-  def test_class_tokenize_string_escape_meta
-    tokens = RDoc::RubyLex.tokenize '"\\M-a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\M-a\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\M-\\C-a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\M-\\C-a\""), tokens.first
-  end
-
-  def test_class_tokenize_string_escape_hexadecimal
-    tokens = RDoc::RubyLex.tokenize '"\\x0"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x0\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\x00"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x00\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\x000"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x000\""), tokens.first
-  end
-
-  def test_class_tokenize_string_escape_octal
-    tokens = RDoc::RubyLex.tokenize '"\\0"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\0\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\00"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\00\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\000"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\000\""), tokens.first
-  end
-
-  def test_class_tokenize_symbol
-    tokens = RDoc::RubyLex.tokenize 'scope module: :v1', nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'scope'),
-      @TK::TkSPACE     .new( 5, 1,  5, ' '),
-      @TK::TkSYMBOL    .new( 6, 1,  6, 'module:'),
-      @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkSYMBOL    .new(14, 1, 14, ':v1'),
-      @TK::TkNL        .new(17, 1, 17, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_particular_kind_of_symbols
-    tokens = RDoc::RubyLex.tokenize '{ Thomas: :Thomas, Dave!: :Dave!, undef: :undef }', nil
-
-    expected = [
-      @TK::TkLBRACE.new( 0, 1,  0, "{"),
-      @TK::TkSPACE .new( 1, 1,  1, " "),
-      @TK::TkSYMBOL.new( 2, 1,  2, "Thomas:"),
-      @TK::TkSPACE .new( 9, 1,  9, " "),
-      @TK::TkSYMBOL.new(10, 1, 10, ":Thomas"),
-      @TK::TkCOMMA .new(17, 1, 17, ","),
-      @TK::TkSPACE .new(18, 1, 18, " "),
-      @TK::TkSYMBOL.new(19, 1, 19, "Dave!:"),
-      @TK::TkSPACE .new(25, 1, 25, " "),
-      @TK::TkSYMBOL.new(26, 1, 26, ":Dave!"),
-      @TK::TkCOMMA .new(32, 1, 32, ","),
-      @TK::TkSPACE .new(33, 1, 33, " "),
-      @TK::TkSYMBOL.new(34, 1, 34, "undef:"),
-      @TK::TkSPACE .new(40, 1, 40, " "),
-      @TK::TkSYMBOL.new(41, 1, 41, ":undef"),
-      @TK::TkSPACE .new(47, 1, 47, " "),
-      @TK::TkRBRACE.new(48, 1, 48, "}"),
-      @TK::TkNL    .new(49, 1, 49, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_symbol_for_nested_method
-    tokens = RDoc::RubyLex.tokenize 'return untrace_var :name', nil
-
-    expected = [
-      @TK::TkRETURN    .new( 0, 1,  0, "return"),
-      @TK::TkSPACE     .new( 6, 1,  6, " "),
-      @TK::TkIDENTIFIER.new( 7, 1,  7, "untrace_var"),
-      @TK::TkSPACE     .new(18, 1, 18, " "),
-      @TK::TkSYMBOL    .new(19, 1, 19, ":name"),
-      @TK::TkNL        .new(24, 1, 24, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_symbol_with_quote
-    tokens = RDoc::RubyLex.tokenize <<RUBY, nil
-a.include?()?"a":"b"
-{"t":1,'t2':2}
-RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, "a"),
-      @TK::TkDOT       .new( 1, 1,  1, "."),
-      @TK::TkFID       .new( 2, 1,  2, "include?"),
-      @TK::TkLPAREN    .new(10, 1, 10, "("),
-      @TK::TkRPAREN    .new(11, 1, 11, ")"),
-      @TK::TkQUESTION  .new(12, 1, 12, "?"),
-      @TK::TkSTRING    .new(13, 1, 13, "\"a\""),
-      @TK::TkCOLON     .new(16, 1, 16, ":"),
-      @TK::TkSTRING    .new(17, 1, 17, "\"b\""),
-      @TK::TkNL        .new(20, 1, 20, "\n"),
-      @TK::TkLBRACE    .new(21, 2,  0, "{"),
-      @TK::TkSYMBOL    .new(22, 2,  1, "\"t\":"),
-      @TK::TkINTEGER   .new(26, 2,  5, "1"),
-      @TK::TkCOMMA     .new(27, 2,  6, ","),
-      @TK::TkSYMBOL    .new(28, 2,  7, "'t2':"),
-      @TK::TkINTEGER   .new(33, 2, 12, "2"),
-      @TK::TkRBRACE    .new(34, 2, 13, "}"),
-      @TK::TkNL        .new(35, 2, 21, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_unary_minus
-    ruby_lex = RDoc::RubyLex.new("-1", nil)
-    assert_equal("-1", ruby_lex.token.value)
-
-    ruby_lex = RDoc::RubyLex.new("a[-2]", nil)
-    2.times { ruby_lex.token } # skip "a" and "["
-    assert_equal("-2", ruby_lex.token.value)
-
-    ruby_lex = RDoc::RubyLex.new("a[0..-12]", nil)
-    4.times { ruby_lex.token } # skip "a", "[", "0", and ".."
-    assert_equal("-12", ruby_lex.token.value)
-
-    ruby_lex = RDoc::RubyLex.new("0+-0.1", nil)
-    2.times { ruby_lex.token } # skip "0" and "+"
-    assert_equal("-0.1", ruby_lex.token.value)
-  end
-
-  def test_rational_imaginary_tokenize
-    tokens = RDoc::RubyLex.tokenize '1.11r + 2.34i + 5.55ri + 0i', nil
-
-    expected = [
-      @TK::TkRATIONAL .new( 0, 1,  0, '1.11r'),
-      @TK::TkSPACE    .new( 5, 1,  5, ' '),
-      @TK::TkPLUS     .new( 6, 1,  6, '+'),
-      @TK::TkSPACE    .new( 7, 1,  7, ' '),
-      @TK::TkIMAGINARY.new( 8, 1,  8, '2.34i'),
-      @TK::TkSPACE    .new(13, 1, 13, ' '),
-      @TK::TkPLUS     .new(14, 1, 14, '+'),
-      @TK::TkSPACE    .new(15, 1, 15, ' '),
-      @TK::TkIMAGINARY.new(16, 1, 16, '5.55ri'),
-      @TK::TkSPACE    .new(22, 1, 22, ' '),
-      @TK::TkPLUS     .new(23, 1, 23, '+'),
-      @TK::TkSPACE    .new(24, 1, 24, ' '),
-      @TK::TkIMAGINARY.new(25, 1, 25, '0i'),
-      @TK::TkNL       .new(27, 1, 27, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_square_bracket_as_method
-    tokens = RDoc::RubyLex.tokenize "Array.[](1, 2)", nil
-
-    expected = [
-      @TK::TkCONSTANT  .new(0,  1,  0, "Array"),
-      @TK::TkDOT       .new(5,  1,  5, "."),
-      @TK::TkIDENTIFIER.new(6,  1,  6, "[]"),
-      @TK::TkfLPAREN   .new(8,  1,  8, "("),
-      @TK::TkINTEGER   .new(9,  1,  9, "1"),
-      @TK::TkCOMMA     .new(10, 1, 10, ","),
-      @TK::TkSPACE     .new(11, 1, 11, " "),
-      @TK::TkINTEGER   .new(12, 1, 12, "2"),
-      @TK::TkRPAREN    .new(13, 1, 13, ")"),
-      @TK::TkNL        .new(14, 1, 14, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_constant_with_exclamation
-    tokens = RDoc::RubyLex.tokenize "Hello there, Dave!", nil
-
-    expected = [
-      @TK::TkCONSTANT  .new( 0, 1,  0, "Hello"),
-      @TK::TkSPACE     .new( 5, 1,  5, " "),
-      @TK::TkIDENTIFIER.new( 6, 1,  6, "there"),
-      @TK::TkCOMMA     .new(11, 1, 11, ","),
-      @TK::TkSPACE     .new(12, 1, 12, " "),
-      @TK::TkIDENTIFIER.new(13, 1, 13, "Dave!"),
-      @TK::TkNL        .new(18, 1, 18, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_identifer_not_equal
-    tokens = RDoc::RubyLex.tokenize "foo!=bar\nfoo?=bar", nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1, 0, "foo"),
-      @TK::TkNEQ       .new( 3, 1, 3, "!="),
-      @TK::TkIDENTIFIER.new( 5, 1, 5, "bar"),
-      @TK::TkNL        .new( 8, 1, 8, "\n"),
-      @TK::TkFID       .new( 9, 2, 0, "foo?"),
-      @TK::TkASSIGN    .new(13, 2, 4, "="),
-      @TK::TkIDENTIFIER.new(14, 2, 5, "bar"),
-      @TK::TkNL        .new(17, 2, 9, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-end
-
diff --git a/test/rdoc/test_rdoc_token_stream.rb b/test/rdoc/test_rdoc_token_stream.rb
index 5ed7b1d1a9..95c3ffd13e 100644
--- a/test/rdoc/test_rdoc_token_stream.rb
+++ b/test/rdoc/test_rdoc_token_stream.rb
@@ -5,17 +5,17 @@ class TestRDocTokenStream < RDoc::TestCase
 
   def test_class_to_html
     tokens = [
-      RDoc::RubyToken::TkCONSTANT. new(0, 0, 0, 'CONSTANT'),
-      RDoc::RubyToken::TkDEF.      new(0, 0, 0, 'KW'),
-      RDoc::RubyToken::TkIVAR.     new(0, 0, 0, 'IVAR'),
-      RDoc::RubyToken::TkOp.       new(0, 0, 0, 'Op'),
-      RDoc::RubyToken::TkId.       new(0, 0, 0, 'Id'),
-      RDoc::RubyToken::TkNode.     new(0, 0, 0, 'Node'),
-      RDoc::RubyToken::TkCOMMENT.  new(0, 0, 0, 'COMMENT'),
-      RDoc::RubyToken::TkREGEXP.   new(0, 0, 0, 'REGEXP'),
-      RDoc::RubyToken::TkSTRING.   new(0, 0, 0, 'STRING'),
-      RDoc::RubyToken::TkVal.      new(0, 0, 0, 'Val'),
-      RDoc::RubyToken::TkBACKSLASH.new(0, 0, 0, '\\'),
+      { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' },
+      { :line_no => 0, :char_no => 0, :kind => :on_kw, :text => 'KW' },
+      { :line_no => 0, :char_no => 0, :kind => :on_ivar, :text => 'IVAR' },
+      { :line_no => 0, :char_no => 0, :kind => :on_op, :text => 'Op' },
+      { :line_no => 0, :char_no => 0, :kind => :on_ident, :text => 'Id' },
+      { :line_no => 0, :char_no => 0, :kind => :on_backref, :text => 'Node' },
+      { :line_no => 0, :char_no => 0, :kind => :on_comment, :text => 'COMMENT' },
+      { :line_no => 0, :char_no => 0, :kind => :on_regexp, :text => 'REGEXP' },
+      { :line_no => 0, :char_no => 0, :kind => :on_tstring, :text => 'STRING' },
+      { :line_no => 0, :char_no => 0, :kind => :on_int, :text => 'Val' },
+      { :line_no => 0, :char_no => 0, :kind => :on_unknown, :text => '\\' }
     ]
 
     expected = [