summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authornobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2017-07-06 04:12:51 +0000
committernobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2017-07-06 04:12:51 +0000
commitd9d8ca0b8f70415d4d35a2e86929de8f6b9feeeb (patch)
tree53d840e5effe78daf37773991992d5297a6ed20f
parent650e7580601327fab48eccde794cee2d467214af (diff)
lexer.rb: ignore dedented space
* ext/ripper/lib/ripper/lexer.rb (on_heredoc_dedent): replace an empty string content because of dedentation with :on_ignored_sp. an empty token makes the sorted order unstable. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59269 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r--ext/ripper/lib/ripper/lexer.rb5
-rw-r--r--test/ripper/test_lexer.rb19
2 files changed, 24 insertions, 0 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 63dcf851329..300c8eef518 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -72,6 +72,11 @@ class Ripper
if Elem === e and e.event == :on_tstring_content
tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok
if (n = dedent_string(e.tok, w)) > 0
+ if e.tok.empty?
+ e.tok = tok[0, n]
+ e.event = :on_ignored_sp
+ next
+ end
ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])]
e.pos[1] += n
end
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 72b7f1b5fd6..30e5d788844 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -32,4 +32,23 @@ class TestRipper::Lexer < Test::Unit::TestCase
E
assert_equal(str, Ripper.tokenize(str).join(""), bug)
end
+
+ def test_embedded_expr_in_heredoc
+ src = <<~'E'
+ <<~B
+ #{1}
+ B
+ E
+ expect = %I[
+ on_heredoc_beg
+ on_nl
+ on_ignored_sp
+ on_embexpr_beg
+ on_int
+ on_embexpr_end
+ on_tstring_content
+ on_heredoc_end
+ ]
+ assert_equal expect, Ripper.lex(src).map {|e| e[1]}
+ end
end