diff options
author | nagachika <nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-07-01 11:51:13 +0000 |
---|---|---|
committer | nagachika <nagachika@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-07-01 11:51:13 +0000 |
commit | 86bb8de8ab7b0e1bfeda1dfaf8e3770152365748 (patch) | |
tree | 6cc80fd43f19369f5d97f718651d7686315bf363 | |
parent | b5b1a2131ec5cbde6adaf0d37953ff05c393218b (diff) |
merge revision(s) 58545,58584: [Backport #13536]
ripper/lexer.rb: nested indented heredoc
* ext/ripper/lib/ripper/lexer.rb (on_heredoc_dedent): fix for
nested indedented here documents, where `Elem`s are nested too.
[ruby-core:80977] [Bug #13536]
ripper/lexer.rb: nested indented heredoc
* ext/ripper/lib/ripper/lexer.rb (on_heredoc_dedent): insert
stripped leading spaces as `on_ignored_sp` elements, so that the
original source can be reconsructed.
[ruby-core:80977] [Bug #13536]
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_4@59247 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r-- | ext/ripper/lib/ripper/lexer.rb | 11 | ||||
-rw-r--r-- | test/ripper/test_lexer.rb | 35 | ||||
-rw-r--r-- | test/ripper/test_scanner_events.rb | 1 | ||||
-rw-r--r-- | version.h | 8 |
4 files changed, 49 insertions, 6 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb index 69526340a7..e70db0b1d1 100644 --- a/ext/ripper/lib/ripper/lexer.rb +++ b/ext/ripper/lib/ripper/lexer.rb @@ -66,13 +66,20 @@ class Ripper private def on_heredoc_dedent(v, w) - @buf.last.each do |e| - if e.event == :on_tstring_content + ignored_sp = [] + heredoc = @buf.last + heredoc.each_with_index do |e, i| + if Elem === e and e.event == :on_tstring_content + tok = e.tok.dup if w > 0 and /\A\s/ =~ e.tok if (n = dedent_string(e.tok, w)) > 0 + ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])] e.pos[1] += n end end end + ignored_sp.reverse_each do |i, e| + heredoc[i, 0] = [e] + end v end diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb new file mode 100644 index 0000000000..72b7f1b5fd --- /dev/null +++ b/test/ripper/test_lexer.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true +begin + require_relative 'dummyparser' + require 'test/unit' + ripper_test = true + module TestRipper; end +rescue LoadError +end + +class TestRipper::Lexer < Test::Unit::TestCase + def test_nested_dedent_heredoc + bug = '[ruby-core:80977] [Bug #13536]' + str = <<~'E' + <<~"D" + #{ + <<~"B" + this must be a valid ruby + B + } + D + E + assert_equal(str, Ripper.tokenize(str).join(""), bug) + + str = <<~'E' + <<~"D" + #{ + <<~"B" + this must be a valid ruby + B + } + D + E + assert_equal(str, Ripper.tokenize(str).join(""), bug) + end +end diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb index 1b7b56ebfe..893e212ee7 100644 --- a/test/ripper/test_scanner_events.rb +++ b/test/ripper/test_scanner_events.rb @@ -105,6 +105,7 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase Ripper.lex("1r\n2i\n3ri\n4.2r\n5.6ri") assert_equal [[[1, 0], :on_heredoc_beg, "<<~EOS"], [[1, 6], :on_nl, "\n"], + [[2, 0], :on_ignored_sp, " "], [[2, 2], :on_tstring_content, "heredoc\n"], [[3, 0], :on_heredoc_end, "EOS"] ], @@ -1,10 +1,10 @@ #define RUBY_VERSION "2.4.2" -#define RUBY_RELEASE_DATE "2017-05-10" -#define RUBY_PATCHLEVEL 131 +#define RUBY_RELEASE_DATE "2017-07-01" +#define RUBY_PATCHLEVEL 132 #define RUBY_RELEASE_YEAR 2017 -#define RUBY_RELEASE_MONTH 5 -#define RUBY_RELEASE_DAY 10 +#define RUBY_RELEASE_MONTH 7 +#define RUBY_RELEASE_DAY 1 #include "ruby/version.h" |