summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authordrbrain <drbrain@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2012-12-18 07:31:36 +0000
committerdrbrain <drbrain@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2012-12-18 07:31:36 +0000
commit2e881bd59e12cb33e868be93af34cb1e2d5475d1 (patch)
treec611d13eff9582d3268a9df837119f5d682d2b66
parent84e3e8441cdc0dd4121e06f0ff4c31e0f4637247 (diff)
* lib/rdoc/ruby_lex.rb: Return a TkHEREDOC instead of a TkSTRING when
the heredoc identifier is followed by a line-end. This allows proper display of some HEREDOCs in source view. * lib/rdoc/ruby_token.rb: Added TkHEREDOC * test/rdoc/test_rdoc_ruby_lex.rb: Test for above. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@38443 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r--ChangeLog8
-rw-r--r--lib/rdoc/ruby_lex.rb24
-rw-r--r--lib/rdoc/ruby_token.rb1
-rw-r--r--test/rdoc/test_rdoc_ruby_lex.rb56
4 files changed, 80 insertions, 9 deletions
diff --git a/ChangeLog b/ChangeLog
index 906a1cd23e2..2332c997277 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+Tue Dec 18 16:31:20 2012 Eric Hodel <drbrain@segment7.net>
+
+ * lib/rdoc/ruby_lex.rb: Return a TkHEREDOC instead of a TkSTRING when
+ the heredoc identifier is followed by a line-end. This allows proper
+ display of some HEREDOCs in source view.
+ * lib/rdoc/ruby_token.rb: Added TkHEREDOC
+ * test/rdoc/test_rdoc_ruby_lex.rb: Test for above.
+
Tue Dec 18 09:45:14 2012 CHIKANAGA Tomoyuki <nagachika@ruby-lang.org>
* vm.c (rb_vm_make_jump_tag_but_local_jump): take care of the case
diff --git a/lib/rdoc/ruby_lex.rb b/lib/rdoc/ruby_lex.rb
index 313e69ea476..e6e0b41aab7 100644
--- a/lib/rdoc/ruby_lex.rb
+++ b/lib/rdoc/ruby_lex.rb
@@ -982,12 +982,13 @@ class RDoc::RubyLex
indent = true
end
if /['"`]/ =~ ch
- lt = ch
+ user_quote = lt = ch
quoted = ""
while (c = getc) && c != lt
quoted.concat c
end
else
+ user_quote = nil
lt = '"'
quoted = ch.dup
while (c = getc) && c =~ /\w/
@@ -1007,8 +1008,17 @@ class RDoc::RubyLex
end
end
+ output_heredoc = reserve.join =~ /\A\r?\n\z/
+
+ if output_heredoc then
+ doc = '<<'
+ doc << '-' if indent
+ doc << "#{user_quote}#{quoted}#{user_quote}\n"
+ else
+ doc = '"'
+ end
+
@here_header = false
- doc = '"'
while l = gets
l = l.sub(/(:?\r)?\n\z/, "\n")
if (indent ? l.strip : l.chomp) == quoted
@@ -1016,7 +1026,12 @@ class RDoc::RubyLex
end
doc << l
end
- doc << '"'
+
+ if output_heredoc then
+ doc << l.chomp
+ else
+ doc << '"'
+ end
@here_header = true
@here_readed.concat reserve
@@ -1024,9 +1039,10 @@ class RDoc::RubyLex
ungetc ch
end
+ token_class = output_heredoc ? RDoc::RubyLex::TkHEREDOC : Ltype2Token[lt]
@ltype = ltback
@lex_state = EXPR_END
- Token(Ltype2Token[lt], doc)
+ Token(token_class, doc)
end
def identify_quotation
diff --git a/lib/rdoc/ruby_token.rb b/lib/rdoc/ruby_token.rb
index 571ea704041..7878e4c22a7 100644
--- a/lib/rdoc/ruby_token.rb
+++ b/lib/rdoc/ruby_token.rb
@@ -331,6 +331,7 @@ module RDoc::RubyToken
[:TkINTEGER, TkVal],
[:TkFLOAT, TkVal],
[:TkSTRING, TkVal],
+ [:TkHEREDOC, TkVal],
[:TkXSTRING, TkVal],
[:TkREGEXP, TkVal],
[:TkSYMBOL, TkVal],
diff --git a/test/rdoc/test_rdoc_ruby_lex.rb b/test/rdoc/test_rdoc_ruby_lex.rb
index dfa350e0187..0dcb42565d1 100644
--- a/test/rdoc/test_rdoc_ruby_lex.rb
+++ b/test/rdoc/test_rdoc_ruby_lex.rb
@@ -70,7 +70,8 @@ end
@TK::TkIDENTIFIER.new( 4, 1, 4, 'x'),
@TK::TkNL .new( 5, 1, 5, "\n"),
@TK::TkSPACE .new( 6, 2, 0, ' '),
- @TK::TkSTRING .new( 8, 2, 2, %Q{"Line 1\nLine 2\n"}),
+ @TK::TkHEREDOC .new( 8, 2, 2,
+ %Q{<<E\nLine 1\nLine 2\nE}),
@TK::TkNL .new(27, 5, 28, "\n"),
@TK::TkEND .new(28, 6, 0, 'end'),
@TK::TkNL .new(31, 6, 28, "\n"),
@@ -96,12 +97,56 @@ end
assert_equal expected, tokens
end
+ def test_class_tokenize_heredoc_CR_NL
+ tokens = RDoc::RubyLex.tokenize <<-RUBY, nil
+string = <<-STRING\r
+Line 1\r
+Line 2\r
+ STRING\r
+ RUBY
+
+ expected = [
+ @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'),
+ @TK::TkSPACE .new( 6, 1, 6, ' '),
+ @TK::TkASSIGN .new( 7, 1, 7, '='),
+ @TK::TkSPACE .new( 8, 1, 8, ' '),
+ @TK::TkHEREDOC .new( 9, 1, 9,
+ %Q{<<-STRING\nLine 1\nLine 2\n STRING}),
+ @TK::TkSPACE .new(44, 4, 45, "\r"),
+ @TK::TkNL .new(45, 4, 46, "\n"),
+ ]
+
+ assert_equal expected, tokens
+ end
+
+ def test_class_tokenize_heredoc_call
+ tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+string = <<-STRING.chomp
+Line 1
+Line 2
+ STRING
+ RUBY
+
+ expected = [
+ @TK::TkIDENTIFIER.new( 0, 1, 0, 'string'),
+ @TK::TkSPACE .new( 6, 1, 6, ' '),
+ @TK::TkASSIGN .new( 7, 1, 7, '='),
+ @TK::TkSPACE .new( 8, 1, 8, ' '),
+ @TK::TkSTRING .new( 9, 1, 9, %Q{"Line 1\nLine 2\n"}),
+ @TK::TkDOT .new(41, 4, 42, '.'),
+ @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'),
+ @TK::TkNL .new(47, 4, 48, "\n"),
+ ]
+
+ assert_equal expected, tokens
+ end
+
def test_class_tokenize_heredoc_indent
tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
string = <<-STRING
Line 1
Line 2
-STRING
+ STRING
RUBY
expected = [
@@ -109,8 +154,9 @@ STRING
@TK::TkSPACE .new( 6, 1, 6, ' '),
@TK::TkASSIGN .new( 7, 1, 7, '='),
@TK::TkSPACE .new( 8, 1, 8, ' '),
- @TK::TkSTRING .new( 9, 1, 9, %Q{"Line 1\nLine 2\n"}),
- @TK::TkNL .new(39, 4, 40, "\n"),
+ @TK::TkHEREDOC .new( 9, 1, 9,
+ %Q{<<-STRING\nLine 1\nLine 2\n STRING}),
+ @TK::TkNL .new(41, 4, 42, "\n"),
]
assert_equal expected, tokens
@@ -128,7 +174,7 @@ U
@TK::TkSPACE .new( 1, 1, 1, ' '),
@TK::TkIDENTIFIER.new( 2, 1, 2, 'b'),
@TK::TkSPACE .new( 3, 1, 3, ' '),
- @TK::TkSTRING .new( 4, 1, 4, %Q{"%N\n"}),
+ @TK::TkHEREDOC .new( 4, 1, 4, %Q{<<-U\n%N\nU}),
@TK::TkNL .new(13, 3, 14, "\n"),
]