[ruby/prism] Add `Ripper.tokenize` to translation layer

It's public API and trivial to implement.

https://github.com/ruby/prism/commit/e77545f8b5

1 files changed, 13 insertions, 0 deletions

diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb
index a901a72692..6552d2dbb8 100644
--- a/lib/prism/translation/ripper.rb
+++ b/lib/prism/translation/ripper.rb
@@ -78,6 +78,19 @@ module Prism
         end
       end
 
+      # Tokenizes the Ruby program and returns an array of strings.
+      # The +filename+ and +lineno+ arguments are mostly ignored, since the
+      # return value is just the tokenized input.
+      # By default, this method does not handle syntax errors in +src+,
+      # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
+      #
+      #   p Ripper.tokenize("def m(a) nil end")
+      #      # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"]
+      #
+      def self.tokenize(...)
+        lex(...).map(&:value)
+      end
+
       # This contains a table of all of the parser events and their
       # corresponding arity.
       PARSER_EVENT_TABLE = {