diff options
-rw-r--r-- | ext/ripper/lib/ripper/lexer.rb | 8 | ||||
-rw-r--r-- | test/ripper/test_lexer.rb | 8 |
2 files changed, 16 insertions, 0 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb index 052aa52de9..636a87fdb2 100644 --- a/ext/ripper/lib/ripper/lexer.rb +++ b/ext/ripper/lib/ripper/lexer.rb @@ -31,6 +31,10 @@ class Ripper raise SyntaxError, r.errors.map(&:message).join(' ;') end + until (tokens = r.tokenize).empty? + ret.concat(tokens) + end + ret end @@ -65,6 +69,10 @@ class Ripper raise SyntaxError, r.errors.map(&:message).join(' ;') end + until (tokens = r.lex).empty? + ret.concat(tokens) + end + ret end diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb index 542db2fba8..3d546c2c94 100644 --- a/test/ripper/test_lexer.rb +++ b/test/ripper/test_lexer.rb @@ -150,4 +150,12 @@ class TestRipper::Lexer < Test::Unit::TestCase assert_raise(SyntaxError) { Ripper.tokenize('def req(true) end', raise_errors: true) } assert_raise(SyntaxError) { Ripper.tokenize('def req(true) end', raise_errors: true) } end + + def test_tokenize_with_syntax_error + assert_equal "end", Ripper.tokenize("def req(true) end").last + end + + def test_lex_with_syntax_error + assert_equal [[1, 14], :on_kw, "end", state(:EXPR_END)], Ripper.lex("def req(true) end").last + end end |