[ruby/prism] Make error and snapshot tests multi-version aware

This one has been on my mind for a while now.

Currently, there are only tests against the latest syntax version.

This changes the snapshot structure as follows:
* Snapshots at their current location are tested against all syntax versions
* Snapshots inside a version folder like "3.3" are tested against all versions starting from that version
* Snapshots inside a version folder like "3.3-4.2" are tested against all versions in the given range.

This makes sure that as new syntax is added, older versions still work as expected.
I also added a few tests for now valid syntax that should be invalid in older versions (and the other way around as well)

These tests run really fast. So even though it does 3x the work for these, I am still able to run the whole test suite in just 11 seconds.

https://github.com/ruby/prism/commit/5191b1aa68

30 files changed, 191 insertions, 84 deletions

diff --git a/prism/prism.c b/prism/prism.c
index 0ebcae62f9..95e7d09050 100644
--- a/prism/prism.c
+++ b/prism/prism.c
@@ -15764,7 +15764,7 @@ parse_return(pm_parser_t *parser, pm_node_t *node) {
                 break;
         }
     }
-    if (in_sclass) {
+    if (in_sclass && parser->version >= PM_OPTIONS_VERSION_CRUBY_3_4) {
         pm_parser_err_node(parser, node, PM_ERR_RETURN_INVALID);
     }
 }
diff --git a/test/prism/errors/3.3-3.3/circular_parameters.txt b/test/prism/errors/3.3-3.3/circular_parameters.txt
new file mode 100644
index 0000000000..ef9642b075
--- /dev/null
+++ b/test/prism/errors/3.3-3.3/circular_parameters.txt
@@ -0,0 +1,12 @@
+def foo(bar = bar) = 42
+        ^~~ circular argument reference - bar
+
+def foo(bar: bar) = 42
+        ^~~ circular argument reference - bar
+
+proc { |foo = foo| }
+        ^~~ circular argument reference - foo
+
+proc { |foo: foo| }
+        ^~~ circular argument reference - foo
+
diff --git a/test/prism/errors/3.3-3.4/leading_logical.txt b/test/prism/errors/3.3-3.4/leading_logical.txt
new file mode 100644
index 0000000000..2a702e281d
--- /dev/null
+++ b/test/prism/errors/3.3-3.4/leading_logical.txt
@@ -0,0 +1,34 @@
+1
+&& 2
+^~ unexpected '&&', ignoring it
+&& 3
+^~ unexpected '&&', ignoring it
+
+1
+|| 2
+^ unexpected '|', ignoring it
+ ^ unexpected '|', ignoring it
+|| 3
+^ unexpected '|', ignoring it
+ ^ unexpected '|', ignoring it
+
+1
+and 2
+^~~ unexpected 'and', ignoring it
+and 3
+^~~ unexpected 'and', ignoring it
+
+1
+or 2
+^~ unexpected 'or', ignoring it
+or 3
+^~ unexpected 'or', ignoring it
+
+1
+and foo
+^~~ unexpected 'and', ignoring it
+
+2
+or foo
+^~ unexpected 'or', ignoring it
+
diff --git a/test/prism/errors/3.3-3.4/private_endless_method.txt b/test/prism/errors/3.3-3.4/private_endless_method.txt
new file mode 100644
index 0000000000..8aae5e0cd3
--- /dev/null
+++ b/test/prism/errors/3.3-3.4/private_endless_method.txt
@@ -0,0 +1,3 @@
+private def foo = puts "Hello"
+                       ^ unexpected string literal, expecting end-of-input
+
diff --git a/test/prism/errors/block_args_in_array_assignment.txt b/test/prism/errors/3.4/block_args_in_array_assignment.txt
index 71dca8452b..71dca8452b 100644
--- a/test/prism/errors/block_args_in_array_assignment.txt
+++ b/test/prism/errors/3.4/block_args_in_array_assignment.txt
diff --git a/test/prism/errors/dont_allow_return_inside_sclass_body.txt b/test/prism/errors/3.4/dont_allow_return_inside_sclass_body.txt
index c29fe01728..c29fe01728 100644
--- a/test/prism/errors/dont_allow_return_inside_sclass_body.txt
+++ b/test/prism/errors/3.4/dont_allow_return_inside_sclass_body.txt
diff --git a/test/prism/errors/it_with_ordinary_parameter.txt b/test/prism/errors/3.4/it_with_ordinary_parameter.txt
index ff9c4276ca..ff9c4276ca 100644
--- a/test/prism/errors/it_with_ordinary_parameter.txt
+++ b/test/prism/errors/3.4/it_with_ordinary_parameter.txt
diff --git a/test/prism/errors/keyword_args_in_array_assignment.txt b/test/prism/errors/3.4/keyword_args_in_array_assignment.txt
index e379ec0ef4..e379ec0ef4 100644
--- a/test/prism/errors/keyword_args_in_array_assignment.txt
+++ b/test/prism/errors/3.4/keyword_args_in_array_assignment.txt
diff --git a/test/prism/errors_test.rb b/test/prism/errors_test.rb
index 9dd4aea728..9abed92652 100644
--- a/test/prism/errors_test.rb
+++ b/test/prism/errors_test.rb
@@ -1,41 +1,19 @@
 # frozen_string_literal: true
 
+return if RUBY_VERSION < "3.3.0"
+
 require_relative "test_helper"
 
 module Prism
   class ErrorsTest < TestCase
     base = File.expand_path("errors", __dir__)
-    filepaths = Dir["*.txt", base: base]
-
-    if RUBY_VERSION < "3.0"
-      filepaths -= [
-        "cannot_assign_to_a_reserved_numbered_parameter.txt",
-        "writing_numbered_parameter.txt",
-        "targeting_numbered_parameter.txt",
-        "defining_numbered_parameter.txt",
-        "defining_numbered_parameter_2.txt",
-        "numbered_parameters_in_block_arguments.txt",
-        "numbered_and_write.txt",
-        "numbered_or_write.txt",
-        "numbered_operator_write.txt"
-      ]
-    end
-
-    if RUBY_VERSION < "3.4"
-      filepaths -= [
-        "it_with_ordinary_parameter.txt",
-        "block_args_in_array_assignment.txt",
-        "keyword_args_in_array_assignment.txt"
-      ]
-    end
-
-    if RUBY_VERSION < "3.4" || RUBY_RELEASE_DATE < "2024-07-24"
-      filepaths -= ["dont_allow_return_inside_sclass_body.txt"]
-    end
+    filepaths = Dir["**/*.txt", base: base]
 
     filepaths.each do |filepath|
-      define_method(:"test_#{File.basename(filepath, ".txt")}") do
-        assert_errors(File.join(base, filepath))
+      ruby_versions_for(filepath).each do |version|
+        define_method(:"test_#{version}_#{File.basename(filepath, ".txt")}") do
+          assert_errors(File.join(base, filepath), version)
+        end
       end
     end
 
@@ -86,38 +64,15 @@ module Prism
       assert_equal :"", Prism.parse_statement("+.@foo,+=foo").write_name
     end
 
-    def test_circular_parameters
-      source = <<~RUBY
-        def foo(bar = bar) = 42
-        def foo(bar: bar) = 42
-        proc { |foo = foo| }
-        proc { |foo: foo| }
-      RUBY
-
-      source.each_line do |line|
-        assert_predicate Prism.parse(line, version: "3.3.0"), :failure?
-        assert_predicate Prism.parse(line), :success?
-      end
-    end
-
-    def test_private_endless_method
-      source = <<~RUBY
-        private def foo = puts "Hello"
-      RUBY
-
-      assert_predicate Prism.parse(source, version: "3.4"), :failure?
-      assert_predicate Prism.parse(source), :success?
-    end
-
     private
 
-    def assert_errors(filepath)
+    def assert_errors(filepath, version)
       expected = File.read(filepath, binmode: true, external_encoding: Encoding::UTF_8)
 
       source = expected.lines.grep_v(/^\s*\^/).join.gsub(/\n*\z/, "")
-      refute_valid_syntax(source)
+      refute_valid_syntax(source) if current_major_minor == version
 
-      result = Prism.parse(source)
+      result = Prism.parse(source, version: version)
       errors = result.errors
       refute_empty errors, "Expected errors in #{filepath}"
 
diff --git a/test/prism/fixtures/3.3-3.3/block_args_in_array_assignment.txt b/test/prism/fixtures/3.3-3.3/block_args_in_array_assignment.txt
new file mode 100644
index 0000000000..6d6b052681
--- /dev/null
+++ b/test/prism/fixtures/3.3-3.3/block_args_in_array_assignment.txt
@@ -0,0 +1 @@
+matrix[5, &block] = 8
diff --git a/test/prism/fixtures/it.txt b/test/prism/fixtures/3.3-3.3/it.txt
index 5410b01e71..5410b01e71 100644
--- a/test/prism/fixtures/it.txt
+++ b/test/prism/fixtures/3.3-3.3/it.txt
diff --git a/test/prism/fixtures/it_indirect_writes.txt b/test/prism/fixtures/3.3-3.3/it_indirect_writes.txt
index bb87e9483e..bb87e9483e 100644
--- a/test/prism/fixtures/it_indirect_writes.txt
+++ b/test/prism/fixtures/3.3-3.3/it_indirect_writes.txt
diff --git a/test/prism/fixtures/it_read_and_assignment.txt b/test/prism/fixtures/3.3-3.3/it_read_and_assignment.txt
index 2cceeb2a54..2cceeb2a54 100644
--- a/test/prism/fixtures/it_read_and_assignment.txt
+++ b/test/prism/fixtures/3.3-3.3/it_read_and_assignment.txt
diff --git a/test/prism/fixtures/3.3-3.3/it_with_ordinary_parameter.txt b/test/prism/fixtures/3.3-3.3/it_with_ordinary_parameter.txt
new file mode 100644
index 0000000000..178b641e6b
--- /dev/null
+++ b/test/prism/fixtures/3.3-3.3/it_with_ordinary_parameter.txt
@@ -0,0 +1 @@
+proc { || it }
diff --git a/test/prism/fixtures/3.3-3.3/keyword_args_in_array_assignment.txt b/test/prism/fixtures/3.3-3.3/keyword_args_in_array_assignment.txt
new file mode 100644
index 0000000000..88016c2afe
--- /dev/null
+++ b/test/prism/fixtures/3.3-3.3/keyword_args_in_array_assignment.txt
@@ -0,0 +1 @@
+matrix[5, axis: :y] = 8
diff --git a/test/prism/fixtures/3.3-3.3/return_in_sclass.txt b/test/prism/fixtures/3.3-3.3/return_in_sclass.txt
new file mode 100644
index 0000000000..f1fde5771a
--- /dev/null
+++ b/test/prism/fixtures/3.3-3.3/return_in_sclass.txt
@@ -0,0 +1 @@
+class << A; return; end
diff --git a/test/prism/fixtures/3.4/circular_parameters.txt b/test/prism/fixtures/3.4/circular_parameters.txt
new file mode 100644
index 0000000000..11537023ad
--- /dev/null
+++ b/test/prism/fixtures/3.4/circular_parameters.txt
@@ -0,0 +1,4 @@
+def foo(bar = bar) = 42
+def foo(bar: bar) = 42
+proc { |foo = foo| }
+proc { |foo: foo| }
diff --git a/test/prism/fixtures/3.4/it.txt b/test/prism/fixtures/3.4/it.txt
new file mode 100644
index 0000000000..5410b01e71
--- /dev/null
+++ b/test/prism/fixtures/3.4/it.txt
@@ -0,0 +1,5 @@
+x do
+  it
+end
+
+-> { it }
diff --git a/test/prism/fixtures/3.4/it_indirect_writes.txt b/test/prism/fixtures/3.4/it_indirect_writes.txt
new file mode 100644
index 0000000000..bb87e9483e
--- /dev/null
+++ b/test/prism/fixtures/3.4/it_indirect_writes.txt
@@ -0,0 +1,23 @@
+tap { it += 1 }
+
+tap { it ||= 1 }
+
+tap { it &&= 1 }
+
+tap { it; it += 1 }
+
+tap { it; it ||= 1 }
+
+tap { it; it &&= 1 }
+
+tap { it += 1; it }
+
+tap { it ||= 1; it }
+
+tap { it &&= 1; it }
+
+tap { it; it += 1; it }
+
+tap { it; it ||= 1; it }
+
+tap { it; it &&= 1; it }
diff --git a/test/prism/fixtures/3.4/it_read_and_assignment.txt b/test/prism/fixtures/3.4/it_read_and_assignment.txt
new file mode 100644
index 0000000000..2cceeb2a54
--- /dev/null
+++ b/test/prism/fixtures/3.4/it_read_and_assignment.txt
@@ -0,0 +1 @@
+42.tap { p it; it = it; p it }
diff --git a/test/prism/fixtures/endless_methods_command_call.txt b/test/prism/fixtures/3.5/endless_methods_command_call.txt
index 91a9d156d5..91a9d156d5 100644
--- a/test/prism/fixtures/endless_methods_command_call.txt
+++ b/test/prism/fixtures/3.5/endless_methods_command_call.txt
diff --git a/test/prism/fixtures/leading_logical.txt b/test/prism/fixtures/3.5/leading_logical.txt
index feb5ee245c..feb5ee245c 100644
--- a/test/prism/fixtures/leading_logical.txt
+++ b/test/prism/fixtures/3.5/leading_logical.txt
diff --git a/test/prism/fixtures_test.rb b/test/prism/fixtures_test.rb
index ddb6ffb40c..9d2acfdc1b 100644
--- a/test/prism/fixtures_test.rb
+++ b/test/prism/fixtures_test.rb
@@ -24,9 +24,13 @@ module Prism
       except << "whitequark/ruby_bug_19281.txt"
     end
 
+    if RUBY_VERSION < "3.4.0"
+      except << "3.4/circular_parameters.txt"
+    end
+
     # Leaving these out until they are supported by parse.y.
-    except << "leading_logical.txt"
-    except << "endless_methods_command_call.txt"
+    except << "3.5/leading_logical.txt"
+    except << "3.5/endless_methods_command_call.txt"
     # https://bugs.ruby-lang.org/issues/21168#note-5
     except << "command_method_call_2.txt"
 
diff --git a/test/prism/lex_test.rb b/test/prism/lex_test.rb
index 3a0da1a2d8..9682bf8a32 100644
--- a/test/prism/lex_test.rb
+++ b/test/prism/lex_test.rb
@@ -43,16 +43,16 @@ module Prism
     end
 
     # https://bugs.ruby-lang.org/issues/20925
-    except << "leading_logical.txt"
+    except << "3.5/leading_logical.txt"
 
     # https://bugs.ruby-lang.org/issues/17398#note-12
-    except << "endless_methods_command_call.txt"
+    except << "3.5/endless_methods_command_call.txt"
 
     # https://bugs.ruby-lang.org/issues/21168#note-5
     except << "command_method_call_2.txt"
 
-    Fixture.each(except: except) do |fixture|
-      define_method(fixture.test_name) { assert_lex(fixture) }
+    Fixture.each_with_version(except: except) do |fixture, version|
+      define_method(fixture.test_name(version)) { assert_lex(fixture, version) }
     end
 
     def test_lex_file
@@ -97,10 +97,12 @@ module Prism
 
     private
 
-    def assert_lex(fixture)
+    def assert_lex(fixture, version)
+      return unless current_major_minor == version
+
       source = fixture.read
 
-      result = Prism.lex_compat(source)
+      result = Prism.lex_compat(source, version: version)
       assert_equal [], result.errors
 
       Prism.lex_ripper(source).zip(result.value).each do |(ripper, prism)|
diff --git a/test/prism/locals_test.rb b/test/prism/locals_test.rb
index 9a3224e8ef..d5def0d18f 100644
--- a/test/prism/locals_test.rb
+++ b/test/prism/locals_test.rb
@@ -32,8 +32,8 @@ module Prism
       "whitequark/ruby_bug_10653.txt",
 
       # Leaving these out until they are supported by parse.y.
-      "leading_logical.txt",
-      "endless_methods_command_call.txt",
+      "3.5/leading_logical.txt",
+      "3.5/endless_methods_command_call.txt",
       "command_method_call_2.txt"
     ]
 
diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb
index 10b5fca5ea..016fda91f0 100644
--- a/test/prism/ruby/parser_test.rb
+++ b/test/prism/ruby/parser_test.rb
@@ -65,11 +65,14 @@ module Prism
       # 1.. && 2
       "ranges.txt",
 
+      # https://bugs.ruby-lang.org/issues/20478
+      "3.4/circular_parameters.txt",
+
       # Cannot yet handling leading logical operators.
-      "leading_logical.txt",
+      "3.5/leading_logical.txt",
 
       # Ruby >= 3.5 specific syntax
-      "endless_methods_command_call.txt",
+      "3.5/endless_methods_command_call.txt",
 
       # https://bugs.ruby-lang.org/issues/21168#note-5
       "command_method_call_2.txt",
@@ -165,9 +168,9 @@ module Prism
 
     if RUBY_VERSION >= "3.3"
       def test_current_parser_for_current_ruby
-        major, minor, _patch = Gem::Version.new(RUBY_VERSION).segments
+        major, minor = current_major_minor.split(".")
         # Let's just hope there never is a Ruby 3.10 or similar
-        expected = major * 10 + minor
+        expected = major.to_i * 10 + minor.to_i
         assert_equal(expected, Translation::ParserCurrent.new.version)
       end
     end
@@ -189,7 +192,7 @@ module Prism
     end
 
     def test_it_block_parameter_syntax
-      it_fixture_path = Pathname(__dir__).join("../../../test/prism/fixtures/it.txt")
+      it_fixture_path = Pathname(__dir__).join("../../../test/prism/fixtures/3.4/it.txt")
 
       buffer = Parser::Source::Buffer.new(it_fixture_path)
       buffer.source = it_fixture_path.read
diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb
index 4916ec8d9d..12c854aea6 100644
--- a/test/prism/ruby/ripper_test.rb
+++ b/test/prism/ruby/ripper_test.rb
@@ -9,7 +9,7 @@ module Prism
     # Skip these tests that Ripper is reporting the wrong results for.
     incorrect = [
       # Not yet supported.
-      "leading_logical.txt",
+      "3.5/leading_logical.txt",
 
       # Ripper incorrectly attributes the block to the keyword.
       "seattlerb/block_break.txt",
@@ -31,8 +31,16 @@ module Prism
       # Ripper fails to understand some structures that span across heredocs.
       "spanning_heredoc.txt",
 
+      "3.3-3.3/block_args_in_array_assignment.txt",
+      "3.3-3.3/it_with_ordinary_parameter.txt",
+      "3.3-3.3/keyword_args_in_array_assignment.txt",
+      "3.3-3.3/return_in_sclass.txt",
+
+      # https://bugs.ruby-lang.org/issues/20478
+      "3.4/circular_parameters.txt",
+
       # https://bugs.ruby-lang.org/issues/17398#note-12
-      "endless_methods_command_call.txt",
+      "3.5/endless_methods_command_call.txt",
 
       # https://bugs.ruby-lang.org/issues/21168#note-5
       "command_method_call_2.txt",
diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb
index ec55e41967..7640ddaf1c 100644
--- a/test/prism/ruby/ruby_parser_test.rb
+++ b/test/prism/ruby/ruby_parser_test.rb
@@ -39,7 +39,6 @@ module Prism
       "dos_endings.txt",
       "heredocs_with_fake_newlines.txt",
       "heredocs_with_ignored_newlines.txt",
-      "leading_logical.txt",
       "method_calls.txt",
       "methods.txt",
       "multi_write.txt",
@@ -77,8 +76,15 @@ module Prism
       "whitequark/ruby_bug_19281.txt",
       "whitequark/slash_newline_in_heredocs.txt",
 
-      # Ruby >= 3.5 specific syntax
-      "endless_methods_command_call.txt",
+      "3.3-3.3/block_args_in_array_assignment.txt",
+      "3.3-3.3/it_with_ordinary_parameter.txt",
+      "3.3-3.3/keyword_args_in_array_assignment.txt",
+      "3.3-3.3/return_in_sclass.txt",
+
+      "3.4/circular_parameters.txt",
+
+      "3.5/endless_methods_command_call.txt",
+      "3.5/leading_logical.txt",
 
       # https://bugs.ruby-lang.org/issues/21168#note-5
       "command_method_call_2.txt",
diff --git a/test/prism/snippets_test.rb b/test/prism/snippets_test.rb
index 66802c5dc3..3160442cc0 100644
--- a/test/prism/snippets_test.rb
+++ b/test/prism/snippets_test.rb
@@ -18,24 +18,24 @@ module Prism
       "whitequark/multiple_pattern_matches.txt"
     ]
 
-    Fixture.each(except: except) do |fixture|
-      define_method(fixture.test_name) { assert_snippets(fixture) }
+    Fixture.each_with_version(except: except) do |fixture, version|
+      define_method(fixture.test_name(version)) { assert_snippets(fixture, version) }
     end
 
     private
 
     # We test every snippet (separated by \n\n) in isolation to ensure the
     # parser does not try to read bytes further than the end of each snippet.
-    def assert_snippets(fixture)
+    def assert_snippets(fixture, version)
       fixture.read.split(/(?<=\S)\n\n(?=\S)/).each do |snippet|
         snippet = snippet.rstrip
 
-        result = Prism.parse(snippet, filepath: fixture.path)
+        result = Prism.parse(snippet, filepath: fixture.path, version: version)
         assert result.success?
 
         if !ENV["PRISM_BUILD_MINIMAL"]
-          dumped = Prism.dump(snippet, filepath: fixture.path)
-          assert_equal_nodes(result.value, Prism.load(snippet, dumped).value)
+          dumped = Prism.dump(snippet, filepath: fixture.path, version: version)
+          assert_equal_nodes(result.value, Prism.load(snippet, dumped, version: version).value)
         end
       end
     end
diff --git a/test/prism/test_helper.rb b/test/prism/test_helper.rb
index 0be9d1e7da..84871722c9 100644
--- a/test/prism/test_helper.rb
+++ b/test/prism/test_helper.rb
@@ -58,8 +58,12 @@ module Prism
         File.join(File.expand_path("../..", __dir__), "snapshots", path)
       end
 
-      def test_name
-        :"test_#{path}"
+      def test_name(version = nil)
+        if version
+          :"test_#{version}_#{path}"
+        else
+          :"test_#{path}"
+        end
       end
 
       def self.each(except: [], &block)
@@ -68,6 +72,14 @@ module Prism
         paths.each { |path| yield Fixture.new(path) }
       end
 
+      def self.each_with_version(except: [], &block)
+        each(except: except) do |fixture|
+          TestCase.ruby_versions_for(fixture.path).each do |version|
+            yield fixture, version
+          end
+        end
+      end
+
       def self.custom_base_path?
         ENV.key?("FIXTURE_BASE")
       end
@@ -217,6 +229,37 @@ module Prism
       RbConfig::CONFIG["host_os"].match?(/bccwin|cygwin|djgpp|mingw|mswin|wince/i)
     end
 
+    # All versions that prism can parse
+    SYNTAX_VERSIONS = %w[3.3 3.4 3.5]
+
+    # Returns an array of ruby versions that a given filepath should test against:
+    # test.txt         # => all available versions
+    # 3.4/test.txt     # => versions since 3.4 (inclusive)
+    # 3.4-4.2/test.txt # => verisions since 3.4 (inclusive) up to 4.2 (inclusive)
+    def self.ruby_versions_for(filepath)
+      return [ENV['SYNTAX_VERSION']] if ENV['SYNTAX_VERSION']
+
+      parts = filepath.split("/")
+      return SYNTAX_VERSIONS if parts.size == 1
+
+      version_start, version_stop = parts[0].split("-")
+      if version_stop
+        SYNTAX_VERSIONS[SYNTAX_VERSIONS.index(version_start)..SYNTAX_VERSIONS.index(version_stop)]
+      else
+        SYNTAX_VERSIONS[SYNTAX_VERSIONS.index(version_start)..]
+      end
+    end
+
+    def current_major_minor
+      RUBY_VERSION.split(".")[0, 2].join(".")
+    end
+
+    if RUBY_VERSION >= "3.3.0"
+      def test_all_syntax_versions_present
+        assert_include(SYNTAX_VERSIONS, current_major_minor)
+      end
+    end
+
     private
 
     if RUBY_ENGINE == "ruby" && RubyVM::InstructionSequence.compile("").to_a[4][:parser] != :prism