8 files changed, 709 insertions, 66 deletions

diff --git a/test/prism/ruby/find_fixtures.rb b/test/prism/ruby/find_fixtures.rb
new file mode 100644
index 0000000000..c1bef0d0e6
--- /dev/null
+++ b/test/prism/ruby/find_fixtures.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+# Test fixtures for Prism.find. These must be in a separate file because
+# source_location returns the file path and Prism.find re-parses the file.
+
+module Prism
+  module FindFixtures
+    module Methods
+      def simple_method
+        42
+      end
+
+      def method_with_params(a, b, c)
+        a + b + c
+      end
+
+      def method_with_block(&block)
+        block.call
+      end
+
+      def self.singleton_method_fixture
+        :singleton
+      end
+
+      def été
+        :utf8
+      end
+
+      def inline_method; :inline; end
+    end
+
+    module Procs
+      SIMPLE_PROC = proc { 42 }
+      SIMPLE_LAMBDA = ->(x) { x * 2 }
+      MULTI_LINE_LAMBDA = lambda do |x|
+        x + 1
+      end
+      DO_BLOCK_PROC = proc do |x|
+        x - 1
+      end
+    end
+
+    module DefineMethod
+      define_method(:dynamic) { |x| x + 1 }
+    end
+
+    module ForLoop
+      for_proc = nil
+      o = Object.new
+      def o.each(&block) = block.call(block)
+      for for_proc in o; end
+      FOR_PROC = for_proc
+    end
+
+    module MultipleOnLine
+      def self.first; end; def self.second; end
+    end
+
+    module Errors
+      def self.divide(a, b)
+        a / b
+      end
+
+      def self.call_undefined
+        undefined_method_call
+      end
+    end
+  end
+end
diff --git a/test/prism/ruby/find_test.rb b/test/prism/ruby/find_test.rb
new file mode 100644
index 0000000000..5b59113d30
--- /dev/null
+++ b/test/prism/ruby/find_test.rb
@@ -0,0 +1,242 @@
+# frozen_string_literal: true
+
+return if RUBY_ENGINE == "ruby" && RUBY_VERSION < "3.4"
+return if defined?(RubyVM::InstructionSequence) && RubyVM::InstructionSequence.compile("").to_a[4][:parser] != :prism
+
+require_relative "../test_helper"
+require_relative "find_fixtures"
+
+module Prism
+  class FindTest < TestCase
+    Fixtures = FindFixtures
+    FIXTURES_PATH = File.expand_path("find_fixtures.rb", __dir__)
+
+    # === Method / UnboundMethod tests ===
+
+    def test_simple_method
+      assert_def_node Prism.find(Fixtures::Methods.instance_method(:simple_method)), :simple_method
+    end
+
+    def test_method_with_params
+      node = Prism.find(Fixtures::Methods.instance_method(:method_with_params))
+      assert_def_node node, :method_with_params
+      assert_equal 3, node.parameters.requireds.length
+    end
+
+    def test_method_with_block_param
+      assert_def_node Prism.find(Fixtures::Methods.instance_method(:method_with_block)), :method_with_block
+    end
+
+    def test_singleton_method
+      assert_def_node Prism.find(Fixtures::Methods.method(:singleton_method_fixture)), :singleton_method_fixture
+    end
+
+    def test_utf8_method_name
+      assert_def_node Prism.find(Fixtures::Methods.instance_method(:été)), :été
+    end
+
+    def test_inline_method
+      assert_def_node Prism.find(Fixtures::Methods.instance_method(:inline_method)), :inline_method
+    end
+
+    def test_bound_method
+      obj = Object.new
+      obj.extend(Fixtures::Methods)
+      assert_def_node Prism.find(obj.method(:simple_method)), :simple_method
+    end
+
+    # === Proc / Lambda tests ===
+
+    def test_simple_proc
+      assert_not_nil Prism.find(Fixtures::Procs::SIMPLE_PROC)
+    end
+
+    def test_simple_lambda
+      assert_not_nil Prism.find(Fixtures::Procs::SIMPLE_LAMBDA)
+    end
+
+    def test_multi_line_lambda
+      assert_not_nil Prism.find(Fixtures::Procs::MULTI_LINE_LAMBDA)
+    end
+
+    def test_do_block_proc
+      assert_not_nil Prism.find(Fixtures::Procs::DO_BLOCK_PROC)
+    end
+
+    # === define_method tests ===
+
+    def test_define_method
+      assert_not_nil Prism.find(Fixtures::DefineMethod.instance_method(:dynamic))
+    end
+
+    def test_define_method_bound
+      obj = Object.new
+      obj.extend(Fixtures::DefineMethod)
+      assert_not_nil Prism.find(obj.method(:dynamic))
+    end
+
+    # === for loop test ===
+
+    def test_for_loop_proc
+      node = Prism.find(Fixtures::ForLoop::FOR_PROC)
+      assert_instance_of ForNode, node
+    end
+
+    # === Thread::Backtrace::Location tests ===
+
+    def test_backtrace_location_zero_division
+      location = zero_division_location
+      assert_not_nil location, "could not find backtrace location in fixtures file"
+      assert_not_nil Prism.find(location)
+    end
+
+    def test_backtrace_location_name_error
+      location = begin
+        Fixtures::Errors.call_undefined
+      rescue NameError => e
+        fixture_backtrace_location(e)
+      end
+
+      assert_not_nil location, "could not find backtrace location in fixtures file"
+      assert_not_nil Prism.find(location)
+    end
+
+    def test_backtrace_location_from_caller
+      # caller_locations returns locations for the current call stack
+      location = caller_locations(0, 1).first
+      node = Prism.find(location)
+      assert_not_nil node
+    end
+
+    def test_backtrace_location_eval_returns_nil
+      location = begin
+        eval("raise 'eval error'")
+      rescue RuntimeError => e
+        e.backtrace_locations.find { |loc| loc.path == "(eval)" || loc.label&.include?("eval") }
+      end
+
+      # eval locations have no file on disk
+      assert_nil Prism.find(location) if location
+    end
+
+    # === Edge cases ===
+
+    def test_nil_source_location
+      # Built-in methods have nil source_location
+      assert_nil Prism.find(method(:puts))
+    end
+
+    def test_argument_error_on_wrong_type
+      assert_raise(ArgumentError) { Prism.find("not a callable") }
+      assert_raise(ArgumentError) { Prism.find(42) }
+      assert_raise(ArgumentError) { Prism.find(nil) }
+    end
+
+    def test_eval_returns_nil
+      # eval'd code has no file on disk
+      m = eval("proc { 1 }")
+      assert_nil Prism.find(m)
+    end
+
+    def test_multiple_methods_on_same_line
+      assert_def_node Prism.find(Fixtures::MultipleOnLine.method(:first)), :first
+      assert_def_node Prism.find(Fixtures::MultipleOnLine.method(:second)), :second
+    end
+
+    # === Fallback (line-based) tests via rubyvm: false ===
+
+    def test_fallback_simple_method
+      assert_def_node Prism.find(Fixtures::Methods.instance_method(:simple_method), rubyvm: false), :simple_method
+    end
+
+    def test_fallback_singleton_method
+      assert_def_node Prism.find(Fixtures::Methods.method(:singleton_method_fixture), rubyvm: false), :singleton_method_fixture
+    end
+
+    def test_fallback_lambda
+      node = Prism.find(Fixtures::Procs::SIMPLE_LAMBDA, rubyvm: false)
+      assert_instance_of LambdaNode, node
+    end
+
+    def test_fallback_proc
+      node = Prism.find(Fixtures::Procs::SIMPLE_PROC, rubyvm: false)
+      assert_instance_of CallNode, node
+      assert node.block.is_a?(BlockNode)
+    end
+
+    def test_fallback_define_method
+      node = Prism.find(Fixtures::DefineMethod.instance_method(:dynamic), rubyvm: false)
+      assert_instance_of CallNode, node
+      assert node.block.is_a?(BlockNode)
+    end
+
+    def test_fallback_for_loop
+      node = Prism.find(Fixtures::ForLoop::FOR_PROC, rubyvm: false)
+      assert_instance_of ForNode, node
+    end
+
+    def test_fallback_backtrace_location
+      location = zero_division_location
+      assert_not_nil location
+      node = Prism.find(location, rubyvm: false)
+      assert_not_nil node
+      assert_equal location.lineno, node.location.start_line
+    end
+
+    # === Node identity with node_id (CRuby only) ===
+
+    if defined?(RubyVM::InstructionSequence)
+      def test_node_id_matches_iseq
+        m = Fixtures::Methods.instance_method(:simple_method)
+        node = Prism.find(m)
+        assert_equal node_id_of(m), node.node_id
+      end
+
+      def test_node_id_for_lambda
+        node = Prism.find(Fixtures::Procs::SIMPLE_LAMBDA)
+        assert_equal node_id_of(Fixtures::Procs::SIMPLE_LAMBDA), node.node_id
+      end
+
+      def test_node_id_for_proc
+        node = Prism.find(Fixtures::Procs::SIMPLE_PROC)
+        assert_equal node_id_of(Fixtures::Procs::SIMPLE_PROC), node.node_id
+      end
+
+      def test_node_id_for_define_method
+        m = Fixtures::DefineMethod.instance_method(:dynamic)
+        node = Prism.find(m)
+        assert_equal node_id_of(m), node.node_id
+      end
+
+      def test_node_id_for_backtrace_location
+        location = zero_division_location
+        assert_not_nil location
+        expected_node_id = RubyVM::AbstractSyntaxTree.node_id_for_backtrace_location(location)
+
+        node = Prism.find(location)
+        assert_equal expected_node_id, node.node_id
+      end
+    end
+
+    private
+
+    def assert_def_node(node, expected_name)
+      assert_instance_of DefNode, node
+      assert_equal expected_name, node.name
+    end
+
+    def fixture_backtrace_location(exception)
+      exception.backtrace_locations.find { |loc| loc.path == FIXTURES_PATH }
+    end
+
+    def zero_division_location
+      Fixtures::Errors.divide(1, 0)
+    rescue ZeroDivisionError => e
+      fixture_backtrace_location(e)
+    end
+
+    def node_id_of(callable)
+      RubyVM::InstructionSequence.of(callable).to_a[4][:node_id]
+    end
+  end
+end
diff --git a/test/prism/ruby/location_test.rb b/test/prism/ruby/location_test.rb
index 33f844243c..12c4258cde 100644
--- a/test/prism/ruby/location_test.rb
+++ b/test/prism/ruby/location_test.rb
@@ -13,19 +13,22 @@ module Prism
       assert_equal 0, joined.start_offset
       assert_equal 10, joined.length
 
-      assert_raise(RuntimeError, "Incompatible locations") do
+      e = assert_raise(RuntimeError) do
         argument.location.join(receiver.location)
       end
+      assert_equal "Incompatible locations", e.message
 
       other_argument = Prism.parse_statement("1234 + 567").arguments.arguments.first
 
-      assert_raise(RuntimeError, "Incompatible sources") do
+      e = assert_raise(RuntimeError) do
         other_argument.location.join(receiver.location)
       end
+      assert_equal "Incompatible sources", e.message
 
-      assert_raise(RuntimeError, "Incompatible sources") do
+      e = assert_raise(RuntimeError) do
         receiver.location.join(other_argument.location)
       end
+      assert_equal "Incompatible sources", e.message
     end
 
     def test_character_offsets
@@ -70,7 +73,7 @@ module Prism
       assert_equal 0, location.start_code_units_offset(Encoding::UTF_16LE)
       assert_equal 0, location.start_code_units_offset(Encoding::UTF_32LE)
 
-      assert_equal 1, location.end_code_units_offset(Encoding::UTF_8)
+      assert_equal 4, location.end_code_units_offset(Encoding::UTF_8)
       assert_equal 2, location.end_code_units_offset(Encoding::UTF_16LE)
       assert_equal 1, location.end_code_units_offset(Encoding::UTF_32LE)
 
@@ -78,37 +81,37 @@ module Prism
       assert_equal 0, location.start_code_units_column(Encoding::UTF_16LE)
       assert_equal 0, location.start_code_units_column(Encoding::UTF_32LE)
 
-      assert_equal 1, location.end_code_units_column(Encoding::UTF_8)
+      assert_equal 4, location.end_code_units_column(Encoding::UTF_8)
       assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE)
       assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE)
 
       # second 😀
       location = program.statements.body.first.arguments.arguments.first.location
 
-      assert_equal 4, location.start_code_units_offset(Encoding::UTF_8)
+      assert_equal 7, location.start_code_units_offset(Encoding::UTF_8)
       assert_equal 5, location.start_code_units_offset(Encoding::UTF_16LE)
       assert_equal 4, location.start_code_units_offset(Encoding::UTF_32LE)
 
-      assert_equal 5, location.end_code_units_offset(Encoding::UTF_8)
+      assert_equal 11, location.end_code_units_offset(Encoding::UTF_8)
       assert_equal 7, location.end_code_units_offset(Encoding::UTF_16LE)
       assert_equal 5, location.end_code_units_offset(Encoding::UTF_32LE)
 
-      assert_equal 4, location.start_code_units_column(Encoding::UTF_8)
+      assert_equal 7, location.start_code_units_column(Encoding::UTF_8)
       assert_equal 5, location.start_code_units_column(Encoding::UTF_16LE)
       assert_equal 4, location.start_code_units_column(Encoding::UTF_32LE)
 
-      assert_equal 5, location.end_code_units_column(Encoding::UTF_8)
+      assert_equal 11, location.end_code_units_column(Encoding::UTF_8)
       assert_equal 7, location.end_code_units_column(Encoding::UTF_16LE)
       assert_equal 5, location.end_code_units_column(Encoding::UTF_32LE)
 
       # first 😍
       location = program.statements.body.last.name_loc
 
-      assert_equal 6, location.start_code_units_offset(Encoding::UTF_8)
+      assert_equal 12, location.start_code_units_offset(Encoding::UTF_8)
       assert_equal 8, location.start_code_units_offset(Encoding::UTF_16LE)
       assert_equal 6, location.start_code_units_offset(Encoding::UTF_32LE)
 
-      assert_equal 7, location.end_code_units_offset(Encoding::UTF_8)
+      assert_equal 16, location.end_code_units_offset(Encoding::UTF_8)
       assert_equal 10, location.end_code_units_offset(Encoding::UTF_16LE)
       assert_equal 7, location.end_code_units_offset(Encoding::UTF_32LE)
 
@@ -116,26 +119,26 @@ module Prism
       assert_equal 0, location.start_code_units_column(Encoding::UTF_16LE)
       assert_equal 0, location.start_code_units_column(Encoding::UTF_32LE)
 
-      assert_equal 1, location.end_code_units_column(Encoding::UTF_8)
+      assert_equal 4, location.end_code_units_column(Encoding::UTF_8)
       assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE)
       assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE)
 
       # second 😍
       location = program.statements.body.last.value.location
 
-      assert_equal 12, location.start_code_units_offset(Encoding::UTF_8)
+      assert_equal 21, location.start_code_units_offset(Encoding::UTF_8)
       assert_equal 15, location.start_code_units_offset(Encoding::UTF_16LE)
       assert_equal 12, location.start_code_units_offset(Encoding::UTF_32LE)
 
-      assert_equal 13, location.end_code_units_offset(Encoding::UTF_8)
+      assert_equal 25, location.end_code_units_offset(Encoding::UTF_8)
       assert_equal 17, location.end_code_units_offset(Encoding::UTF_16LE)
       assert_equal 13, location.end_code_units_offset(Encoding::UTF_32LE)
 
-      assert_equal 6, location.start_code_units_column(Encoding::UTF_8)
+      assert_equal 9, location.start_code_units_column(Encoding::UTF_8)
       assert_equal 7, location.start_code_units_column(Encoding::UTF_16LE)
       assert_equal 6, location.start_code_units_column(Encoding::UTF_32LE)
 
-      assert_equal 7, location.end_code_units_column(Encoding::UTF_8)
+      assert_equal 13, location.end_code_units_column(Encoding::UTF_8)
       assert_equal 9, location.end_code_units_column(Encoding::UTF_16LE)
       assert_equal 7, location.end_code_units_column(Encoding::UTF_32LE)
     end
@@ -154,7 +157,7 @@ module Prism
       assert_equal 0, location.cached_start_code_units_offset(utf16_cache)
       assert_equal 0, location.cached_start_code_units_offset(utf32_cache)
 
-      assert_equal 1, location.cached_end_code_units_offset(utf8_cache)
+      assert_equal 4, location.cached_end_code_units_offset(utf8_cache)
       assert_equal 2, location.cached_end_code_units_offset(utf16_cache)
       assert_equal 1, location.cached_end_code_units_offset(utf32_cache)
 
@@ -162,26 +165,26 @@ module Prism
       assert_equal 0, location.cached_start_code_units_column(utf16_cache)
       assert_equal 0, location.cached_start_code_units_column(utf32_cache)
 
-      assert_equal 1, location.cached_end_code_units_column(utf8_cache)
+      assert_equal 4, location.cached_end_code_units_column(utf8_cache)
       assert_equal 2, location.cached_end_code_units_column(utf16_cache)
       assert_equal 1, location.cached_end_code_units_column(utf32_cache)
 
       # second 😀
       location = result.value.statements.body.first.arguments.arguments.first.location
 
-      assert_equal 4, location.cached_start_code_units_offset(utf8_cache)
+      assert_equal 7, location.cached_start_code_units_offset(utf8_cache)
       assert_equal 5, location.cached_start_code_units_offset(utf16_cache)
       assert_equal 4, location.cached_start_code_units_offset(utf32_cache)
 
-      assert_equal 5, location.cached_end_code_units_offset(utf8_cache)
+      assert_equal 11, location.cached_end_code_units_offset(utf8_cache)
       assert_equal 7, location.cached_end_code_units_offset(utf16_cache)
       assert_equal 5, location.cached_end_code_units_offset(utf32_cache)
 
-      assert_equal 4, location.cached_start_code_units_column(utf8_cache)
+      assert_equal 7, location.cached_start_code_units_column(utf8_cache)
       assert_equal 5, location.cached_start_code_units_column(utf16_cache)
       assert_equal 4, location.cached_start_code_units_column(utf32_cache)
 
-      assert_equal 5, location.cached_end_code_units_column(utf8_cache)
+      assert_equal 11, location.cached_end_code_units_column(utf8_cache)
       assert_equal 7, location.cached_end_code_units_column(utf16_cache)
       assert_equal 5, location.cached_end_code_units_column(utf32_cache)
     end
@@ -197,7 +200,7 @@ module Prism
       assert_equal "😀".b.to_sym, receiver.name
 
       location = receiver.location
-      assert_equal 1, location.end_code_units_column(Encoding::UTF_8)
+      assert_equal 4, location.end_code_units_column(Encoding::UTF_8)
       assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE)
       assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE)
     end
diff --git a/test/prism/ruby/parameters_signature_test.rb b/test/prism/ruby/parameters_signature_test.rb
index ea1eea106b..1ca2b144a9 100644
--- a/test/prism/ruby/parameters_signature_test.rb
+++ b/test/prism/ruby/parameters_signature_test.rb
@@ -50,6 +50,11 @@ module Prism
       assert_parameters([[:nokey]], "**nil")
     end
 
+    def test_noblock
+      # FIXME: `compare: RUBY_VERSION >= "4.1"` once builds are available
+      assert_parameters([[:noblock]], "&nil", compare: false)
+    end
+
     def test_keyrest_anonymous
       assert_parameters([[:keyrest, :**]], "**")
     end
@@ -72,12 +77,20 @@ module Prism
       assert_parameters([[:rest, :*], [:keyrest, :**], [:block, :&]], "...")
     end
 
+    def test_invalid_syntax
+      e = assert_raise(RuntimeError) do
+        Prism.parse_statement("def f(**nil, ...); end").parameters.signature
+      end
+      assert_equal("Invalid syntax", e.message)
+    end
+
     private
 
-    def assert_parameters(expected, source)
+    def assert_parameters(expected, source, compare: true)
       # Compare against our expectation.
       assert_equal(expected, signature(source))
 
+      return unless compare
       # Compare against Ruby's expectation.
       object = Object.new
       eval("def object.m(#{source}); end")
diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb
index 98740f0973..ad9fa0c92c 100644
--- a/test/prism/ruby/parser_test.rb
+++ b/test/prism/ruby/parser_test.rb
@@ -5,8 +5,6 @@ require_relative "../test_helper"
 begin
   verbose, $VERBOSE = $VERBOSE, nil
   require "parser/ruby33"
-  require "prism/translation/parser33"
-  require "prism/translation/parser34"
 rescue LoadError
   # In CRuby's CI, we're not going to test against the parser gem because we
   # don't want to have to install it. So in this case we'll just skip this test.
@@ -62,14 +60,14 @@ module Prism
       "alias.txt",
       "seattlerb/bug_215.txt",
 
+      # %Q with newline delimiter and heredoc interpolation
+      "heredoc_percent_q_newline_delimiter.txt",
+
       # 1.. && 2
       "ranges.txt",
 
-      # Cannot yet handling leading logical operators.
-      "leading_logical.txt",
-
-      # Ruby >= 3.5 specific syntax
-      "endless_methods_command_call.txt",
+      # https://bugs.ruby-lang.org/issues/21168#note-5
+      "command_method_call_2.txt",
     ]
 
     # These files contain code that is being parsed incorrectly by the parser
@@ -103,6 +101,9 @@ module Prism
       # Regex with \c escape
       "unescaping.txt",
       "seattlerb/regexp_esc_C_slash.txt",
+
+      # https://github.com/whitequark/parser/issues/1084
+      "unary_method_calls.txt",
     ]
 
     # These files are failing to translate their lexer output into the lexer
@@ -139,7 +140,7 @@ module Prism
       "whitequark/space_args_block.txt"
     ]
 
-    Fixture.each(except: skip_syntax_error) do |fixture|
+    Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture|
       define_method(fixture.test_name) do
         assert_equal_parses(
           fixture,
@@ -162,9 +163,9 @@ module Prism
 
     if RUBY_VERSION >= "3.3"
       def test_current_parser_for_current_ruby
-        major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
+        major, minor = CURRENT_MAJOR_MINOR.split(".")
         # Let's just hope there never is a Ruby 3.10 or similar
-        expected = major * 10 + minor
+        expected = major.to_i * 10 + minor.to_i
         assert_equal(expected, Translation::ParserCurrent.new.version)
       end
     end
@@ -186,13 +187,7 @@ module Prism
     end
 
     def test_it_block_parameter_syntax
-      it_fixture_path = Pathname(__dir__).join("../../../test/prism/fixtures/it.txt")
-
-      buffer = Parser::Source::Buffer.new(it_fixture_path)
-      buffer.source = it_fixture_path.read
-      actual_ast = Prism::Translation::Parser34.new.tokenize(buffer)[0]
-
-      it_block_parameter_sexp = parse_sexp {
+      assert_new_syntax("3.4/it.txt", Prism::Translation::Parser34) do
         s(:begin,
         s(:itblock,
           s(:send, nil, :x), :it,
@@ -200,9 +195,20 @@ module Prism
         s(:itblock,
           s(:lambda), :it,
           s(:lvar, :it)))
-      }
+      end
+    end
 
-      assert_equal(it_block_parameter_sexp, actual_ast.to_sexp)
+    def test_nil_block_parameter_syntax
+      assert_new_syntax("4.1/noblock.txt", Prism::Translation::Parser41) do
+        s(:begin,
+        s(:def, :foo,
+          s(:args,
+            s(:blocknilarg)), nil),
+        s(:block,
+          s(:lambda),
+            s(:args,
+              s(:blocknilarg)), nil))
+      end
     end
 
     private
@@ -300,6 +306,16 @@ module Prism
       }
     end
 
+    def assert_new_syntax(path, parser, &sexp)
+      fixture_path = Pathname(__dir__).join("../../../test/prism/fixtures", path)
+
+      buffer = Parser::Source::Buffer.new(fixture_path)
+      buffer.source = fixture_path.read
+      actual_ast = parser.new.tokenize(buffer)[0]
+
+      assert_equal(parse_sexp(&sexp), actual_ast.to_sexp)
+    end
+
     def parse_sexp(&block)
       Class.new { extend AST::Sexp }.instance_eval(&block).to_sexp
     end
diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb
index 39325137ba..4fff630561 100644
--- a/test/prism/ruby/ripper_test.rb
+++ b/test/prism/ruby/ripper_test.rb
@@ -3,40 +3,56 @@
 return if RUBY_VERSION < "3.3" || RUBY_ENGINE != "ruby"
 
 require_relative "../test_helper"
+require "ripper"
 
 module Prism
   class RipperTest < TestCase
     # Skip these tests that Ripper is reporting the wrong results for.
     incorrect = [
-      # Not yet supported.
-      "leading_logical.txt",
-
       # Ripper incorrectly attributes the block to the keyword.
-      "seattlerb/block_break.txt",
-      "seattlerb/block_next.txt",
       "seattlerb/block_return.txt",
-      "whitequark/break_block.txt",
-      "whitequark/next_block.txt",
       "whitequark/return_block.txt",
 
-      # Ripper is not accounting for locals created by patterns using the **
-      # operator within an `in` clause.
-      "seattlerb/parse_pattern_058.txt",
-
       # Ripper cannot handle named capture groups in regular expressions.
       "regex.txt",
-      "regex_char_width.txt",
-      "whitequark/lvar_injecting_match.txt",
 
       # Ripper fails to understand some structures that span across heredocs.
       "spanning_heredoc.txt",
 
-      # https://bugs.ruby-lang.org/issues/17398#note-12
-      "endless_methods_command_call.txt",
+      # Ripper interprets circular keyword arguments as method calls.
+      "3.4/circular_parameters.txt",
+
+      # Ripper doesn't emit `args_add_block` when endless method is prefixed by modifier.
+      "4.0/endless_methods_command_call.txt",
+
+      # https://bugs.ruby-lang.org/issues/21168#note-5
+      "command_method_call_2.txt",
     ]
 
+    if RUBY_VERSION.start_with?("3.3.")
+      incorrect += [
+        "whitequark/lvar_injecting_match.txt",
+        "seattlerb/parse_pattern_058.txt",
+        "regex_char_width.txt",
+      ]
+    end
+
+    if RUBY_VERSION.start_with?("4.")
+      incorrect += [
+        # https://bugs.ruby-lang.org/issues/21945
+        "and_or_with_suffix.txt",
+      ]
+    end
+
+    # https://bugs.ruby-lang.org/issues/21669
+    incorrect << "4.1/void_value.txt"
+    # https://bugs.ruby-lang.org/issues/19107
+    incorrect << "4.1/trailing_comma_after_method_arguments.txt"
+
     # Skip these tests that we haven't implemented yet.
-    omitted = [
+    omitted_sexp_raw = [
+      "bom_leading_space.txt",
+      "bom_spaces.txt",
       "dos_endings.txt",
       "heredocs_with_fake_newlines.txt",
       "heredocs_with_ignored_newlines.txt",
@@ -57,14 +73,237 @@ module Prism
       "whitequark/slash_newline_in_heredocs.txt"
     ]
 
-    Fixture.each(except: incorrect | omitted) do |fixture|
-      define_method(fixture.test_name) { assert_ripper(fixture.read) }
+    omitted_lex = [
+      "heredoc_with_escaped_newline_at_start.txt",
+      "heredocs_with_fake_newlines.txt",
+      "indented_file_end.txt",
+      "spanning_heredoc_newlines.txt",
+      "whitequark/dedenting_heredoc.txt",
+      "whitequark/procarg0.txt",
+    ]
+
+    omitted_scan = [
+      "bom_leading_space.txt",
+      "bom_spaces.txt",
+      "dos_endings.txt",
+      "heredocs_with_fake_newlines.txt",
+      "rescue_modifier.txt",
+      "seattlerb/block_call_dot_op2_brace_block.txt",
+      "seattlerb/block_command_operation_colon.txt",
+      "seattlerb/block_command_operation_dot.txt",
+      "seattlerb/case_in.txt",
+      "seattlerb/heredoc__backslash_dos_format.txt",
+      "seattlerb/heredoc_backslash_nl.txt",
+      "seattlerb/heredoc_nested.txt",
+      "seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt",
+      "seattlerb/heredoc_squiggly_empty.txt",
+      "seattlerb/masgn_command_call.txt",
+      "seattlerb/messy_op_asgn_lineno.txt",
+      "seattlerb/op_asgn_primary_colon_const_command_call.txt",
+      "seattlerb/parse_pattern_076.txt",
+      "seattlerb/pct_w_heredoc_interp_nested.txt",
+      "tilde_heredocs.txt",
+      "unparser/corpus/literal/assignment.txt",
+      "unparser/corpus/literal/pattern.txt",
+      "unparser/corpus/semantic/dstr.txt",
+      "variables.txt",
+      "whitequark/dedenting_heredoc.txt",
+      "whitequark/masgn_nested.txt",
+      "whitequark/newline_in_hash_argument.txt",
+      "whitequark/numparam_ruby_bug_19025.txt",
+      "whitequark/op_asgn_cmd.txt",
+      "whitequark/parser_drops_truncated_parts_of_squiggly_heredoc.txt",
+      "whitequark/parser_slash_slash_n_escaping_in_literals.txt",
+      "whitequark/pattern_matching_nil_pattern.txt",
+      "whitequark/ruby_bug_12402.txt",
+      "whitequark/ruby_bug_18878.txt",
+      "whitequark/send_block_chain_cmd.txt",
+      "whitequark/slash_newline_in_heredocs.txt",
+    ]
+
+    Fixture.each_for_current_ruby(except: incorrect | omitted_sexp_raw) do |fixture|
+      define_method("#{fixture.test_name}_sexp_raw") { assert_ripper_sexp_raw(fixture.read) }
+    end
+
+    Fixture.each_for_current_ruby(except: incorrect | omitted_lex) do |fixture|
+      define_method("#{fixture.test_name}_lex") { assert_ripper_lex(fixture.read) }
+    end
+
+    def test_lex_ignored_missing_heredoc_end
+      ["", "-", "~"].each do |type|
+        source = "<<#{type}FOO\n"
+        assert_ripper_lex(source)
+
+        source = "<<#{type}'FOO'\n"
+        assert_ripper_lex(source)
+      end
+    end
+
+    UNSUPPORTED_EVENTS = %i[comma ignored_nl nl semicolon sp ignored_sp]
+    # Events that are currently not emitted
+    SUPPORTED_EVENTS = Translation::Ripper::EVENTS - UNSUPPORTED_EVENTS
+    # Events that assert against their line/column
+    CHECK_LOCATION_EVENTS = %i[kw op lbrace rbrace lbracket rbracket lparen rparen words_sep label_end]
+
+    module Events
+      attr_reader :events
+
+      def initialize(...)
+        super
+        @events = []
+      end
+
+      SUPPORTED_EVENTS.each do |event|
+        define_method(:"on_#{event}") do |*args|
+          if CHECK_LOCATION_EVENTS.include?(event)
+            @events << [event, lineno, column, *args]
+          else
+            @events << [event, *args]
+          end
+          super(*args)
+        end
+      end
+    end
+
+    class RipperEvents < Ripper
+      include Events
+    end
+
+    class PrismEvents < Translation::Ripper
+      include Events
+    end
+
+    class ObjectEvents < Translation::Ripper
+      OBJECT = BasicObject.new
+      SUPPORTED_EVENTS.each do |event|
+        define_method(:"on_#{event}") { |*args| OBJECT }
+      end
+    end
+
+    Fixture.each_for_current_ruby(except: incorrect | omitted_scan) do |fixture|
+      define_method("#{fixture.test_name}_events") do
+        source = fixture.read
+        # Similar to test/ripper/assert_parse_files.rb in CRuby
+        object_events = ObjectEvents.new(source)
+        assert_nothing_raised { object_events.parse }
+
+        ripper = RipperEvents.new(source, fixture.path)
+        prism = PrismEvents.new(source, fixture.path)
+        ripper.parse
+        prism.parse
+        # Check that the same events are emitted, regardless of order
+        assert_equal(ripper.events.sort_by(&:inspect), prism.events.sort_by(&:inspect))
+      end
+    end
+
+    def test_lexer
+      lexer = Translation::Ripper::Lexer.new("foo")
+      expected = [[1, 0], :on_ident, "foo", Translation::Ripper::EXPR_CMDARG]
+
+      assert_equal([expected], lexer.lex)
+      assert_equal(expected, lexer.parse[0].to_a)
+      assert_equal(lexer.parse[0].to_a, lexer.scan[0].to_a)
+
+      assert_equal(%i[on_int on_sp on_op], Translation::Ripper::Lexer.new("1 +").lex.map { |token| token[1] })
+      assert_raise(SyntaxError) { Translation::Ripper::Lexer.new("1 +").lex(raise_errors: true) }
+    end
+
+
+    # On syntax invalid code the output doesn't always match up
+    # In these cases we just want to make sure that it doesn't raise.
+    def test_lex_invalid_syntax
+      assert_nothing_raised do
+        Translation::Ripper.lex('scan/\p{alpha}/')
+      end
+
+      assert_equal(Ripper.lex('if;)'), Translation::Ripper.lex('if;)'))
+    end
+
+    def test_tokenize
+      source = "foo;1;BAZ"
+      assert_equal(Ripper.tokenize(source), Translation::Ripper.tokenize(source))
+    end
+
+    def test_encoding
+      source = '"わたし"'.encode(Encoding::Windows_31J)
+      assert_equal(Ripper.tokenize(source), Translation::Ripper.tokenize(source))
+      assert_equal(Ripper.sexp(source), Translation::Ripper.sexp(source))
+    end
+
+    def test_sexp_coercion
+      string_like = Object.new
+      def string_like.to_str
+        "a"
+      end
+      assert_equal Ripper.sexp(string_like), Translation::Ripper.sexp(string_like)
+
+      File.open(__FILE__) do |file1|
+        File.open(__FILE__) do |file2|
+          assert_equal Ripper.sexp(file1), Translation::Ripper.sexp(file2)
+        end
+      end
+
+      File.open(__FILE__) do |file1|
+        File.open(__FILE__) do |file2|
+          object1_with_gets = Object.new
+          object1_with_gets.define_singleton_method(:gets) do
+            file1.gets
+          end
+
+          object2_with_gets = Object.new
+          object2_with_gets.define_singleton_method(:gets) do
+            file2.gets
+          end
+
+          assert_equal Ripper.sexp(object1_with_gets), Translation::Ripper.sexp(object2_with_gets)
+        end
+      end
+    end
+
+    def test_lex_coersion
+      string_like = Object.new
+      def string_like.to_str
+        "a"
+      end
+      assert_equal Ripper.lex(string_like), Translation::Ripper.lex(string_like)
+    end
+
+    # Check that the hardcoded values don't change without us noticing.
+    def test_internals
+      actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
+      expected = Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
+
+      assert_equal(expected, actual)
+      expected.zip(actual).each do |ripper, prism|
+        assert_equal(Ripper.const_get(ripper), Translation::Ripper.const_get(prism))
+      end
     end
 
     private
 
-    def assert_ripper(source)
+    def assert_ripper_sexp_raw(source)
       assert_equal Ripper.sexp_raw(source), Prism::Translation::Ripper.sexp_raw(source)
     end
+
+    def assert_ripper_lex(source)
+      prism = Translation::Ripper.lex(source)
+      ripper = Ripper.lex(source)
+
+      # Prism emits tokens by their order in the code, not in parse order
+      ripper.sort_by! { |elem| elem[0] }
+
+      [prism.size, ripper.size].max.times do |index|
+        expected = ripper[index]
+        actual = prism[index]
+
+        # There are some tokens that have slightly different state that do not
+        # effect the parse tree, so they may not match.
+        if expected && actual && expected[1] == actual[1] && %i[on_comment on_heredoc_end on_embexpr_end on_sp].include?(expected[1])
+          expected[3] = actual[3] = nil
+        end
+
+        assert_equal(expected, actual)
+      end
+    end
   end
 end
diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb
index bcaed79791..bc89bdae72 100644
--- a/test/prism/ruby/ruby_parser_test.rb
+++ b/test/prism/ruby/ruby_parser_test.rb
@@ -16,6 +16,7 @@ end
 module Prism
   class RubyParserTest < TestCase
     todos = [
+      "character_literal.txt",
       "encoding_euc_jp.txt",
       "regex_char_width.txt",
       "seattlerb/masgn_colon3.txt",
@@ -36,9 +37,10 @@ module Prism
       "alias.txt",
       "dsym_str.txt",
       "dos_endings.txt",
+      "heredoc_dedent_line_continuation.txt",
+      "heredoc_percent_q_newline_delimiter.txt",
       "heredocs_with_fake_newlines.txt",
       "heredocs_with_ignored_newlines.txt",
-      "leading_logical.txt",
       "method_calls.txt",
       "methods.txt",
       "multi_write.txt",
@@ -57,6 +59,7 @@ module Prism
       "spanning_heredoc.txt",
       "symbols.txt",
       "tilde_heredocs.txt",
+      "unary_method_calls.txt",
       "unparser/corpus/literal/literal.txt",
       "while.txt",
       "whitequark/cond_eflipflop.txt",
@@ -76,11 +79,18 @@ module Prism
       "whitequark/ruby_bug_19281.txt",
       "whitequark/slash_newline_in_heredocs.txt",
 
-      # Ruby >= 3.5 specific syntax
-      "endless_methods_command_call.txt",
+      "3.3-3.3/block_args_in_array_assignment.txt",
+      "3.3-3.3/it_with_ordinary_parameter.txt",
+      "3.3-3.3/keyword_args_in_array_assignment.txt",
+      "3.3-3.3/return_in_sclass.txt",
+
+      "3.3-4.0/void_value.txt",
+
+      # https://bugs.ruby-lang.org/issues/21168#note-5
+      "command_method_call_2.txt",
     ]
 
-    Fixture.each(except: failures) do |fixture|
+    Fixture.each_for_version(version: "3.3", except: failures) do |fixture|
       define_method(fixture.test_name) do
         assert_ruby_parser(fixture, todos.include?(fixture.path))
       end
diff --git a/test/prism/ruby/source_test.rb b/test/prism/ruby/source_test.rb
new file mode 100644
index 0000000000..f7cf4fe83a
--- /dev/null
+++ b/test/prism/ruby/source_test.rb
@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require_relative "../test_helper"
+
+module Prism
+  class SourceTest < TestCase
+    def test_byte_offset
+      source = Prism.parse(<<~SRC).source
+      abcd
+      efgh
+      ijkl
+      SRC
+
+      assert_equal 0, source.byte_offset(1, 0)
+      assert_equal 5, source.byte_offset(2, 0)
+      assert_equal 10, source.byte_offset(3, 0)
+      assert_equal 15, source.byte_offset(4, 0)
+
+      error = assert_raise(ArgumentError) { source.byte_offset(5, 0) }
+      assert_equal "line 5 is out of range", error.message
+
+      error = assert_raise(ArgumentError) { source.byte_offset(0, 0) }
+      assert_equal "line 0 is out of range", error.message
+
+      error = assert_raise(ArgumentError) { source.byte_offset(-1, 0) }
+      assert_equal "line -1 is out of range", error.message
+    end
+
+    def test_byte_offset_with_start_line
+      source = Prism.parse(<<~SRC, line: 11).source
+      abcd
+      efgh
+      ijkl
+      SRC
+
+      assert_equal 0, source.byte_offset(11, 0)
+      assert_equal 5, source.byte_offset(12, 0)
+      assert_equal 10, source.byte_offset(13, 0)
+      assert_equal 15, source.byte_offset(14, 0)
+
+      error = assert_raise(ArgumentError) { source.byte_offset(15, 0) }
+      assert_equal "line 15 is out of range", error.message
+
+      error = assert_raise(ArgumentError) { source.byte_offset(10, 0) }
+      assert_equal "line 10 is out of range", error.message
+
+      error = assert_raise(ArgumentError) { source.byte_offset(9, 0) }
+      assert_equal "line 9 is out of range", error.message
+    end
+  end
+end