# frozen_string_literal: true begin require 'ripper' require 'test/unit' module TestRipper; end rescue LoadError end class TestRipper::Lexer < Test::Unit::TestCase def test_nested_dedent_heredoc bug = '[ruby-core:80977] [Bug #13536]' str = <<~'E' <<~"D" #{ <<~"B" this must be a valid ruby B } D E assert_equal(str, Ripper.tokenize(str).join(""), bug) str = <<~'E' <<~"D" #{ <<~"B" this must be a valid ruby B } D E assert_equal(str, Ripper.tokenize(str).join(""), bug) end def test_embedded_expr_in_heredoc src = <<~'E' <<~B #{1} B E expect = %I[ on_heredoc_beg on_nl on_ignored_sp on_embexpr_beg on_int on_embexpr_end on_tstring_content on_heredoc_end ] assert_equal expect, Ripper.lex(src).map {|e| e[1]} end def test_space_after_expr_in_heredoc src = <<~'E' <<~B #{1} a B E expect = %I[ on_heredoc_beg on_nl on_ignored_sp on_embexpr_beg on_int on_embexpr_end on_tstring_content on_heredoc_end ] assert_equal expect, Ripper.lex(src).map {|e| e[1]} end def test_expr_at_beginning_in_heredoc src = <<~'E' <<~B a #{1} B E expect = %I[ on_heredoc_beg on_nl on_tstring_content on_embexpr_beg on_int on_embexpr_end on_tstring_content on_heredoc_end ] assert_equal expect, Ripper.lex(src).map {|e| e[1]} end def test_stack_at_on_heredoc_beg src = "a < s.pos[1] assert_equal pos, s.pos[1] + s.tok.bytesize, message else assert_equal pos, s.pos[1], message pos += s.tok.bytesize end end assert_equal pos, code.bytesize assert_equal expected.size, result.size end def test_trailing_on_embexpr_end # This is useful for scanning a template engine literal `{ foo, bar: baz }` # whose body inside brackes works like trailing method arguments, like Haml. token = Ripper.lex("a( foo, bar: baz }").last assert_equal [[1, 17], :on_embexpr_end, "}", state(:EXPR_ARG)], token end BAD_CODE = [ [:parse_error, 'def req(true) end', %r[unexpected `true'], 'true'], [:parse_error, 'def req(a, a) end', %r[duplicated argument name], 'a'], [:assign_error, 'begin; nil = 1; end', %r[assign to nil], 'nil'], [:alias_error, 'begin; alias $x $1; end', %r[number variables], '$1'], [:class_name_error, 'class bad; end', %r[class/module name], 'bad'], [:param_error, 'def req(@a) end', %r[formal argument.*instance], '@a'], [:param_error, 'def req(a?:) end', %r[formal argument must.*local], 'a?'], ] def test_raise_errors_keyword all_assertions do |all| BAD_CODE.each do |(err, code, message)| all.for([err, code]) do assert_raise_with_message(SyntaxError, message) { Ripper.tokenize(code, raise_errors: true) } end end end end def test_tokenize_with_syntax_error all_assertions do |all| BAD_CODE.each do |(err, code)| all.for([err, code]) do assert_equal "end", Ripper.tokenize(code).last end end end end def test_lex_with_syntax_error all_assertions do |all| BAD_CODE.each do |(err, code)| all.for([err, code]) do assert_equal [[1, code.size-3], :on_kw, "end", state(:EXPR_END)], Ripper.lex(code).last end end end end def test_lexer_scan_with_syntax_error all_assertions do |all| BAD_CODE.each do |(err, code, message, token)| all.for([err, code]) do lexer = Ripper::Lexer.new(code) elems = lexer.scan assert_predicate lexer, :error? error = lexer.errors.first assert_match message, error.message i = elems.find_index{|e| e == error} assert_operator 0...elems.size, :include?, i elem = elems[i + 1] assert_not_equal error.event, elem.event assert_equal error.pos, elem.pos assert_equal error.tok, elem.tok assert_equal error.state, elem.state end end end end end