Ripper.{lex,tokenize} return full tokens even if syntax error

yet another implements [Feature #17276]

1 files changed, 8 insertions, 0 deletions

diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 052aa52de9..636a87fdb2 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -31,6 +31,10 @@ class Ripper
       raise SyntaxError, r.errors.map(&:message).join(' ;')
     end
 
+    until (tokens = r.tokenize).empty?
+      ret.concat(tokens)
+    end
+
     ret
   end
 
@@ -65,6 +69,10 @@ class Ripper
       raise SyntaxError, r.errors.map(&:message).join(' ;')
     end
 
+    until (tokens = r.lex).empty?
+      ret.concat(tokens)
+    end
+
     ret
   end