diff options
Diffstat (limited to 'test/ripper/test_lexer.rb')
| -rw-r--r-- | test/ripper/test_lexer.rb | 108 |
1 files changed, 107 insertions, 1 deletions
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb index 83130668be..7d62a7ee28 100644 --- a/test/ripper/test_lexer.rb +++ b/test/ripper/test_lexer.rb @@ -100,6 +100,20 @@ class TestRipper::Lexer < Test::Unit::TestCase assert_equal expect, Ripper.lex(src).map {|e| e[1]} end + def test_end_of_script_char + all_assertions do |all| + ["a", %w"[a ]", %w"{, }", "if"].each do |src, append| + expected = Ripper.lex(src).map {|e| e[1]} + ["\0b", "\4b", "\32b"].each do |eof| + c = "#{src}#{eof}#{append}" + all.for(c) do + assert_equal expected, Ripper.lex(c).map {|e| e[1]} + end + end + end + end + end + def test_slice assert_equal "string\#{nil}\n", Ripper.slice(%(<<HERE\nstring\#{nil}\nHERE), "heredoc_beg .*? nl $(.*?) heredoc_end", 1) @@ -157,7 +171,7 @@ class TestRipper::Lexer < Test::Unit::TestCase end BAD_CODE = [ - [:parse_error, 'def req(true) end', %r[unexpected `true'], 'true'], + [:parse_error, 'def req(true) end', %r[unexpected 'true'], 'true'], [:parse_error, 'def req(a, a) end', %r[duplicated argument name], 'a'], [:assign_error, 'begin; nil = 1; end', %r[assign to nil], 'nil'], [:alias_error, 'begin; alias $x $1; end', %r[number variables], '$1'], @@ -228,4 +242,96 @@ class TestRipper::Lexer < Test::Unit::TestCase EOF assert_equal([[5, 0], :on_heredoc_end, "EOS\n", state(:EXPR_BEG)], Ripper.lex(s).last, bug) end + + def test_tokenize_with_here_document + bug = '[Bug #18963]' + code = %[ +<<A + "hello +A +world" +] + assert_equal(code, Ripper.tokenize(code).join(""), bug) + end + + def test_heredoc_inside_block_param + bug = '[Bug #19399]' + code = <<~CODE + a do |b + <<-C + C + | + end + CODE + assert_equal(code, Ripper.tokenize(code).join(""), bug) + end + + def test_heredoc_unterminated_interpolation + code = <<~'HEREDOC' + <<A+1 + #{ + HEREDOC + + assert_include(Ripper.tokenize(code).join(""), "+1") + end + + def test_nested_heredoc + code = <<~'HEREDOC' + <<~H1 + 1 + #{<<~H2} + 2 + H2 + 3 + H1 + HEREDOC + + expected = [ + [[1, 0], :on_heredoc_beg, "<<~H1", state(:EXPR_BEG)], + [[1, 5], :on_nl, "\n", state(:EXPR_BEG)], + [[2, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[2, 2], :on_tstring_content, "1\n", state(:EXPR_BEG)], + [[3, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[3, 2], :on_embexpr_beg, "\#{", state(:EXPR_BEG)], + [[3, 4], :on_heredoc_beg, "<<~H2", state(:EXPR_BEG)], + [[3, 9], :on_embexpr_end, "}", state(:EXPR_END)], + [[3, 10], :on_tstring_content, "\n", state(:EXPR_BEG)], + [[4, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[4, 4], :on_tstring_content, "2\n", state(:EXPR_BEG)], + [[5, 0], :on_heredoc_end, " H2\n", state(:EXPR_BEG)], + [[6, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[6, 2], :on_tstring_content, "3\n", state(:EXPR_BEG)], + [[7, 0], :on_heredoc_end, "H1\n", state(:EXPR_BEG)], + ] + assert_equal(code, Ripper.tokenize(code).join("")) + assert_equal(expected, result = Ripper.lex(code), + proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}}) + + code = <<~'HEREDOC' + <<-H1 + 1 + #{<<~H2} + 2 + H2 + 3 + H1 + HEREDOC + + expected = [ + [[1, 0], :on_heredoc_beg, "<<-H1", state(:EXPR_BEG)], + [[1, 5], :on_nl, "\n", state(:EXPR_BEG)], + [[2, 0], :on_tstring_content, " 1\n ", state(:EXPR_BEG)], + [[3, 2], :on_embexpr_beg, "\#{", state(:EXPR_BEG)], + [[3, 4], :on_heredoc_beg, "<<~H2", state(:EXPR_BEG)], + [[3, 9], :on_embexpr_end, "}", state(:EXPR_END)], + [[3, 10], :on_tstring_content, "\n", state(:EXPR_BEG)], + [[4, 0], :on_ignored_sp, " ", state(:EXPR_BEG)], + [[4, 4], :on_tstring_content, "2\n", state(:EXPR_BEG)], + [[5, 0], :on_heredoc_end, " H2\n", state(:EXPR_BEG)], + [[6, 0], :on_tstring_content, " 3\n", state(:EXPR_BEG)], + [[7, 0], :on_heredoc_end, "H1\n", state(:EXPR_BEG)], + ] + assert_equal(code, Ripper.tokenize(code).join("")) + assert_equal(expected, result = Ripper.lex(code), + proc {expected.zip(result) {|e, r| break diff(e, r) unless e == r}}) + end end |