* lib/rdoc/ruby_lex.rb: Return a TkHEREDOC instead of a TkSTRING when

  the heredoc identifier is followed by a line-end.  This allows proper
  display of some HEREDOCs in source view.
* lib/rdoc/ruby_token.rb:  Added TkHEREDOC
* test/rdoc/test_rdoc_ruby_lex.rb:  Test for above.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@38443 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 51 insertions, 5 deletions

diff --git a/test/rdoc/test_rdoc_ruby_lex.rb b/test/rdoc/test_rdoc_ruby_lex.rb
index dfa350e018..0dcb42565d 100644
--- a/test/rdoc/test_rdoc_ruby_lex.rb
+++ b/test/rdoc/test_rdoc_ruby_lex.rb
@@ -70,7 +70,8 @@ end
       @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
       @TK::TkNL        .new( 5, 1,  5, "\n"),
       @TK::TkSPACE     .new( 6, 2,  0, '  '),
-      @TK::TkSTRING    .new( 8, 2,  2, %Q{"Line 1\nLine 2\n"}),
+      @TK::TkHEREDOC   .new( 8, 2,  2,
+                            %Q{<<E\nLine 1\nLine 2\nE}),
       @TK::TkNL        .new(27, 5, 28, "\n"),
       @TK::TkEND       .new(28, 6,  0, 'end'),
       @TK::TkNL        .new(31, 6, 28, "\n"),
@@ -96,12 +97,56 @@ end
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_heredoc_CR_NL
+    tokens = RDoc::RubyLex.tokenize <<-RUBY, nil
+string = <<-STRING\r
+Line 1\r
+Line 2\r
+  STRING\r
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkASSIGN    .new( 7, 1,  7, '='),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkHEREDOC   .new( 9, 1,  9,
+                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
+      @TK::TkSPACE     .new(44, 4, 45, "\r"),
+      @TK::TkNL        .new(45, 4, 46, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_heredoc_call
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+string = <<-STRING.chomp
+Line 1
+Line 2
+  STRING
+    RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkASSIGN    .new( 7, 1,  7, '='),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkSTRING    .new( 9, 1,  9, %Q{"Line 1\nLine 2\n"}),
+      @TK::TkDOT       .new(41, 4, 42, '.'),
+      @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'),
+      @TK::TkNL        .new(47, 4, 48, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_heredoc_indent
     tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
 string = <<-STRING
 Line 1
 Line 2
-STRING
+  STRING
     RUBY
 
     expected = [
@@ -109,8 +154,9 @@ STRING
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkSTRING    .new( 9, 1,  9, %Q{"Line 1\nLine 2\n"}),
-      @TK::TkNL        .new(39, 4, 40, "\n"),
+      @TK::TkHEREDOC   .new( 9, 1,  9,
+                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
+      @TK::TkNL        .new(41, 4, 42, "\n"),
     ]
 
     assert_equal expected, tokens
@@ -128,7 +174,7 @@ U
       @TK::TkSPACE     .new( 1, 1,  1, ' '),
       @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
       @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkSTRING    .new( 4, 1,  4, %Q{"%N\n"}),
+      @TK::TkHEREDOC   .new( 4, 1,  4, %Q{<<-U\n%N\nU}),
       @TK::TkNL        .new(13, 3, 14, "\n"),
     ]