[ruby/prism] Add `Ripper.tokenize` to translation layer

It's public API and trivial to implement.

https://github.com/ruby/prism/commit/e77545f8b5

2 files changed, 18 insertions, 0 deletions

diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb
index a901a72692..6552d2dbb8 100644
--- a/lib/prism/translation/ripper.rb
+++ b/lib/prism/translation/ripper.rb
@@ -78,6 +78,19 @@ module Prism
         end
       end
 
+      # Tokenizes the Ruby program and returns an array of strings.
+      # The +filename+ and +lineno+ arguments are mostly ignored, since the
+      # return value is just the tokenized input.
+      # By default, this method does not handle syntax errors in +src+,
+      # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+.
+      #
+      #   p Ripper.tokenize("def m(a) nil end")
+      #      # => ["def", " ", "m", "(", "a", ")", " ", "nil", " ", "end"]
+      #
+      def self.tokenize(...)
+        lex(...).map(&:value)
+      end
+
       # This contains a table of all of the parser events and their
       # corresponding arity.
       PARSER_EVENT_TABLE = {
diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb
index 2bd9c2fe4a..cac20a073d 100644
--- a/test/prism/ruby/ripper_test.rb
+++ b/test/prism/ruby/ripper_test.rb
@@ -84,6 +84,11 @@ module Prism
       define_method("#{fixture.test_name}_lexer_parse") { assert_ripper_lexer_parse(fixture.read) }
     end
 
+    def test_tokenize
+      source = "foo;1;BAZ"
+      assert_equal(Ripper.tokenize(source), Translation::Ripper.tokenize(source))
+    end
+
     # Check that the hardcoded values don't change without us noticing.
     def test_internals
       actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort