summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorNobuhiro IMAI <nov@yo.rim.or.jp>2020-11-19 23:19:08 +0900
committerNobuyoshi Nakada <nobu@ruby-lang.org>2020-11-20 11:44:57 +0900
commit1800f3fa5c29515113ecdcc5695e8a96f462b74c (patch)
tree91affdf983246ba6da885448e0c23b80eef1494b
parent80d3f21994653186bc1bb2401b16cacebd8f9efb (diff)
Ripper.{lex,tokenize} return full tokens even if syntax error
yet another implements [Feature #17276]
-rw-r--r--ext/ripper/lib/ripper/lexer.rb8
-rw-r--r--test/ripper/test_lexer.rb8
2 files changed, 16 insertions, 0 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 052aa52..636a87f 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -31,6 +31,10 @@ class Ripper
raise SyntaxError, r.errors.map(&:message).join(' ;')
end
+ until (tokens = r.tokenize).empty?
+ ret.concat(tokens)
+ end
+
ret
end
@@ -65,6 +69,10 @@ class Ripper
raise SyntaxError, r.errors.map(&:message).join(' ;')
end
+ until (tokens = r.lex).empty?
+ ret.concat(tokens)
+ end
+
ret
end
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 542db2f..3d546c2 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -150,4 +150,12 @@ class TestRipper::Lexer < Test::Unit::TestCase
assert_raise(SyntaxError) { Ripper.tokenize('def req(true) end', raise_errors: true) }
assert_raise(SyntaxError) { Ripper.tokenize('def req(true) end', raise_errors: true) }
end
+
+ def test_tokenize_with_syntax_error
+ assert_equal "end", Ripper.tokenize("def req(true) end").last
+ end
+
+ def test_lex_with_syntax_error
+ assert_equal [[1, 14], :on_kw, "end", state(:EXPR_END)], Ripper.lex("def req(true) end").last
+ end
end