diff options
Diffstat (limited to 'test/prism/ruby')
| -rw-r--r-- | test/prism/ruby/dispatcher_test.rb | 19 | ||||
| -rw-r--r-- | test/prism/ruby/find_fixtures.rb | 69 | ||||
| -rw-r--r-- | test/prism/ruby/find_test.rb | 242 | ||||
| -rw-r--r-- | test/prism/ruby/location_test.rb | 49 | ||||
| -rw-r--r-- | test/prism/ruby/parameters_signature_test.rb | 22 | ||||
| -rw-r--r-- | test/prism/ruby/parser_test.rb | 130 | ||||
| -rw-r--r-- | test/prism/ruby/ripper_test.rb | 278 | ||||
| -rw-r--r-- | test/prism/ruby/ruby_parser_test.rb | 48 | ||||
| -rw-r--r-- | test/prism/ruby/source_test.rb | 51 |
9 files changed, 811 insertions, 97 deletions
diff --git a/test/prism/ruby/dispatcher_test.rb b/test/prism/ruby/dispatcher_test.rb index 1b6d7f4117..83eb29e1f3 100644 --- a/test/prism/ruby/dispatcher_test.rb +++ b/test/prism/ruby/dispatcher_test.rb @@ -25,9 +25,12 @@ module Prism end def test_dispatching_events - listener = TestListener.new + listener_manual = TestListener.new + listener_public = TestListener.new + dispatcher = Dispatcher.new - dispatcher.register(listener, :on_call_node_enter, :on_call_node_leave, :on_integer_node_enter) + dispatcher.register(listener_manual, :on_call_node_enter, :on_call_node_leave, :on_integer_node_enter) + dispatcher.register_public_methods(listener_public) root = Prism.parse(<<~RUBY).value def foo @@ -36,11 +39,17 @@ module Prism RUBY dispatcher.dispatch(root) - assert_equal([:on_call_node_enter, :on_integer_node_enter, :on_integer_node_enter, :on_integer_node_enter, :on_call_node_leave], listener.events_received) - listener.events_received.clear + [listener_manual, listener_public].each do |listener| + assert_equal([:on_call_node_enter, :on_integer_node_enter, :on_integer_node_enter, :on_integer_node_enter, :on_call_node_leave], listener.events_received) + listener.events_received.clear + end + dispatcher.dispatch_once(root.statements.body.first.body.body.first) - assert_equal([:on_call_node_enter, :on_call_node_leave], listener.events_received) + + [listener_manual, listener_public].each do |listener| + assert_equal([:on_call_node_enter, :on_call_node_leave], listener.events_received) + end end end end diff --git a/test/prism/ruby/find_fixtures.rb b/test/prism/ruby/find_fixtures.rb new file mode 100644 index 0000000000..c1bef0d0e6 --- /dev/null +++ b/test/prism/ruby/find_fixtures.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +# Test fixtures for Prism.find. These must be in a separate file because +# source_location returns the file path and Prism.find re-parses the file. + +module Prism + module FindFixtures + module Methods + def simple_method + 42 + end + + def method_with_params(a, b, c) + a + b + c + end + + def method_with_block(&block) + block.call + end + + def self.singleton_method_fixture + :singleton + end + + def été + :utf8 + end + + def inline_method; :inline; end + end + + module Procs + SIMPLE_PROC = proc { 42 } + SIMPLE_LAMBDA = ->(x) { x * 2 } + MULTI_LINE_LAMBDA = lambda do |x| + x + 1 + end + DO_BLOCK_PROC = proc do |x| + x - 1 + end + end + + module DefineMethod + define_method(:dynamic) { |x| x + 1 } + end + + module ForLoop + for_proc = nil + o = Object.new + def o.each(&block) = block.call(block) + for for_proc in o; end + FOR_PROC = for_proc + end + + module MultipleOnLine + def self.first; end; def self.second; end + end + + module Errors + def self.divide(a, b) + a / b + end + + def self.call_undefined + undefined_method_call + end + end + end +end diff --git a/test/prism/ruby/find_test.rb b/test/prism/ruby/find_test.rb new file mode 100644 index 0000000000..5b59113d30 --- /dev/null +++ b/test/prism/ruby/find_test.rb @@ -0,0 +1,242 @@ +# frozen_string_literal: true + +return if RUBY_ENGINE == "ruby" && RUBY_VERSION < "3.4" +return if defined?(RubyVM::InstructionSequence) && RubyVM::InstructionSequence.compile("").to_a[4][:parser] != :prism + +require_relative "../test_helper" +require_relative "find_fixtures" + +module Prism + class FindTest < TestCase + Fixtures = FindFixtures + FIXTURES_PATH = File.expand_path("find_fixtures.rb", __dir__) + + # === Method / UnboundMethod tests === + + def test_simple_method + assert_def_node Prism.find(Fixtures::Methods.instance_method(:simple_method)), :simple_method + end + + def test_method_with_params + node = Prism.find(Fixtures::Methods.instance_method(:method_with_params)) + assert_def_node node, :method_with_params + assert_equal 3, node.parameters.requireds.length + end + + def test_method_with_block_param + assert_def_node Prism.find(Fixtures::Methods.instance_method(:method_with_block)), :method_with_block + end + + def test_singleton_method + assert_def_node Prism.find(Fixtures::Methods.method(:singleton_method_fixture)), :singleton_method_fixture + end + + def test_utf8_method_name + assert_def_node Prism.find(Fixtures::Methods.instance_method(:été)), :été + end + + def test_inline_method + assert_def_node Prism.find(Fixtures::Methods.instance_method(:inline_method)), :inline_method + end + + def test_bound_method + obj = Object.new + obj.extend(Fixtures::Methods) + assert_def_node Prism.find(obj.method(:simple_method)), :simple_method + end + + # === Proc / Lambda tests === + + def test_simple_proc + assert_not_nil Prism.find(Fixtures::Procs::SIMPLE_PROC) + end + + def test_simple_lambda + assert_not_nil Prism.find(Fixtures::Procs::SIMPLE_LAMBDA) + end + + def test_multi_line_lambda + assert_not_nil Prism.find(Fixtures::Procs::MULTI_LINE_LAMBDA) + end + + def test_do_block_proc + assert_not_nil Prism.find(Fixtures::Procs::DO_BLOCK_PROC) + end + + # === define_method tests === + + def test_define_method + assert_not_nil Prism.find(Fixtures::DefineMethod.instance_method(:dynamic)) + end + + def test_define_method_bound + obj = Object.new + obj.extend(Fixtures::DefineMethod) + assert_not_nil Prism.find(obj.method(:dynamic)) + end + + # === for loop test === + + def test_for_loop_proc + node = Prism.find(Fixtures::ForLoop::FOR_PROC) + assert_instance_of ForNode, node + end + + # === Thread::Backtrace::Location tests === + + def test_backtrace_location_zero_division + location = zero_division_location + assert_not_nil location, "could not find backtrace location in fixtures file" + assert_not_nil Prism.find(location) + end + + def test_backtrace_location_name_error + location = begin + Fixtures::Errors.call_undefined + rescue NameError => e + fixture_backtrace_location(e) + end + + assert_not_nil location, "could not find backtrace location in fixtures file" + assert_not_nil Prism.find(location) + end + + def test_backtrace_location_from_caller + # caller_locations returns locations for the current call stack + location = caller_locations(0, 1).first + node = Prism.find(location) + assert_not_nil node + end + + def test_backtrace_location_eval_returns_nil + location = begin + eval("raise 'eval error'") + rescue RuntimeError => e + e.backtrace_locations.find { |loc| loc.path == "(eval)" || loc.label&.include?("eval") } + end + + # eval locations have no file on disk + assert_nil Prism.find(location) if location + end + + # === Edge cases === + + def test_nil_source_location + # Built-in methods have nil source_location + assert_nil Prism.find(method(:puts)) + end + + def test_argument_error_on_wrong_type + assert_raise(ArgumentError) { Prism.find("not a callable") } + assert_raise(ArgumentError) { Prism.find(42) } + assert_raise(ArgumentError) { Prism.find(nil) } + end + + def test_eval_returns_nil + # eval'd code has no file on disk + m = eval("proc { 1 }") + assert_nil Prism.find(m) + end + + def test_multiple_methods_on_same_line + assert_def_node Prism.find(Fixtures::MultipleOnLine.method(:first)), :first + assert_def_node Prism.find(Fixtures::MultipleOnLine.method(:second)), :second + end + + # === Fallback (line-based) tests via rubyvm: false === + + def test_fallback_simple_method + assert_def_node Prism.find(Fixtures::Methods.instance_method(:simple_method), rubyvm: false), :simple_method + end + + def test_fallback_singleton_method + assert_def_node Prism.find(Fixtures::Methods.method(:singleton_method_fixture), rubyvm: false), :singleton_method_fixture + end + + def test_fallback_lambda + node = Prism.find(Fixtures::Procs::SIMPLE_LAMBDA, rubyvm: false) + assert_instance_of LambdaNode, node + end + + def test_fallback_proc + node = Prism.find(Fixtures::Procs::SIMPLE_PROC, rubyvm: false) + assert_instance_of CallNode, node + assert node.block.is_a?(BlockNode) + end + + def test_fallback_define_method + node = Prism.find(Fixtures::DefineMethod.instance_method(:dynamic), rubyvm: false) + assert_instance_of CallNode, node + assert node.block.is_a?(BlockNode) + end + + def test_fallback_for_loop + node = Prism.find(Fixtures::ForLoop::FOR_PROC, rubyvm: false) + assert_instance_of ForNode, node + end + + def test_fallback_backtrace_location + location = zero_division_location + assert_not_nil location + node = Prism.find(location, rubyvm: false) + assert_not_nil node + assert_equal location.lineno, node.location.start_line + end + + # === Node identity with node_id (CRuby only) === + + if defined?(RubyVM::InstructionSequence) + def test_node_id_matches_iseq + m = Fixtures::Methods.instance_method(:simple_method) + node = Prism.find(m) + assert_equal node_id_of(m), node.node_id + end + + def test_node_id_for_lambda + node = Prism.find(Fixtures::Procs::SIMPLE_LAMBDA) + assert_equal node_id_of(Fixtures::Procs::SIMPLE_LAMBDA), node.node_id + end + + def test_node_id_for_proc + node = Prism.find(Fixtures::Procs::SIMPLE_PROC) + assert_equal node_id_of(Fixtures::Procs::SIMPLE_PROC), node.node_id + end + + def test_node_id_for_define_method + m = Fixtures::DefineMethod.instance_method(:dynamic) + node = Prism.find(m) + assert_equal node_id_of(m), node.node_id + end + + def test_node_id_for_backtrace_location + location = zero_division_location + assert_not_nil location + expected_node_id = RubyVM::AbstractSyntaxTree.node_id_for_backtrace_location(location) + + node = Prism.find(location) + assert_equal expected_node_id, node.node_id + end + end + + private + + def assert_def_node(node, expected_name) + assert_instance_of DefNode, node + assert_equal expected_name, node.name + end + + def fixture_backtrace_location(exception) + exception.backtrace_locations.find { |loc| loc.path == FIXTURES_PATH } + end + + def zero_division_location + Fixtures::Errors.divide(1, 0) + rescue ZeroDivisionError => e + fixture_backtrace_location(e) + end + + def node_id_of(callable) + RubyVM::InstructionSequence.of(callable).to_a[4][:node_id] + end + end +end diff --git a/test/prism/ruby/location_test.rb b/test/prism/ruby/location_test.rb index 33f844243c..12c4258cde 100644 --- a/test/prism/ruby/location_test.rb +++ b/test/prism/ruby/location_test.rb @@ -13,19 +13,22 @@ module Prism assert_equal 0, joined.start_offset assert_equal 10, joined.length - assert_raise(RuntimeError, "Incompatible locations") do + e = assert_raise(RuntimeError) do argument.location.join(receiver.location) end + assert_equal "Incompatible locations", e.message other_argument = Prism.parse_statement("1234 + 567").arguments.arguments.first - assert_raise(RuntimeError, "Incompatible sources") do + e = assert_raise(RuntimeError) do other_argument.location.join(receiver.location) end + assert_equal "Incompatible sources", e.message - assert_raise(RuntimeError, "Incompatible sources") do + e = assert_raise(RuntimeError) do receiver.location.join(other_argument.location) end + assert_equal "Incompatible sources", e.message end def test_character_offsets @@ -70,7 +73,7 @@ module Prism assert_equal 0, location.start_code_units_offset(Encoding::UTF_16LE) assert_equal 0, location.start_code_units_offset(Encoding::UTF_32LE) - assert_equal 1, location.end_code_units_offset(Encoding::UTF_8) + assert_equal 4, location.end_code_units_offset(Encoding::UTF_8) assert_equal 2, location.end_code_units_offset(Encoding::UTF_16LE) assert_equal 1, location.end_code_units_offset(Encoding::UTF_32LE) @@ -78,37 +81,37 @@ module Prism assert_equal 0, location.start_code_units_column(Encoding::UTF_16LE) assert_equal 0, location.start_code_units_column(Encoding::UTF_32LE) - assert_equal 1, location.end_code_units_column(Encoding::UTF_8) + assert_equal 4, location.end_code_units_column(Encoding::UTF_8) assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE) assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE) # second 😀 location = program.statements.body.first.arguments.arguments.first.location - assert_equal 4, location.start_code_units_offset(Encoding::UTF_8) + assert_equal 7, location.start_code_units_offset(Encoding::UTF_8) assert_equal 5, location.start_code_units_offset(Encoding::UTF_16LE) assert_equal 4, location.start_code_units_offset(Encoding::UTF_32LE) - assert_equal 5, location.end_code_units_offset(Encoding::UTF_8) + assert_equal 11, location.end_code_units_offset(Encoding::UTF_8) assert_equal 7, location.end_code_units_offset(Encoding::UTF_16LE) assert_equal 5, location.end_code_units_offset(Encoding::UTF_32LE) - assert_equal 4, location.start_code_units_column(Encoding::UTF_8) + assert_equal 7, location.start_code_units_column(Encoding::UTF_8) assert_equal 5, location.start_code_units_column(Encoding::UTF_16LE) assert_equal 4, location.start_code_units_column(Encoding::UTF_32LE) - assert_equal 5, location.end_code_units_column(Encoding::UTF_8) + assert_equal 11, location.end_code_units_column(Encoding::UTF_8) assert_equal 7, location.end_code_units_column(Encoding::UTF_16LE) assert_equal 5, location.end_code_units_column(Encoding::UTF_32LE) # first 😍 location = program.statements.body.last.name_loc - assert_equal 6, location.start_code_units_offset(Encoding::UTF_8) + assert_equal 12, location.start_code_units_offset(Encoding::UTF_8) assert_equal 8, location.start_code_units_offset(Encoding::UTF_16LE) assert_equal 6, location.start_code_units_offset(Encoding::UTF_32LE) - assert_equal 7, location.end_code_units_offset(Encoding::UTF_8) + assert_equal 16, location.end_code_units_offset(Encoding::UTF_8) assert_equal 10, location.end_code_units_offset(Encoding::UTF_16LE) assert_equal 7, location.end_code_units_offset(Encoding::UTF_32LE) @@ -116,26 +119,26 @@ module Prism assert_equal 0, location.start_code_units_column(Encoding::UTF_16LE) assert_equal 0, location.start_code_units_column(Encoding::UTF_32LE) - assert_equal 1, location.end_code_units_column(Encoding::UTF_8) + assert_equal 4, location.end_code_units_column(Encoding::UTF_8) assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE) assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE) # second 😍 location = program.statements.body.last.value.location - assert_equal 12, location.start_code_units_offset(Encoding::UTF_8) + assert_equal 21, location.start_code_units_offset(Encoding::UTF_8) assert_equal 15, location.start_code_units_offset(Encoding::UTF_16LE) assert_equal 12, location.start_code_units_offset(Encoding::UTF_32LE) - assert_equal 13, location.end_code_units_offset(Encoding::UTF_8) + assert_equal 25, location.end_code_units_offset(Encoding::UTF_8) assert_equal 17, location.end_code_units_offset(Encoding::UTF_16LE) assert_equal 13, location.end_code_units_offset(Encoding::UTF_32LE) - assert_equal 6, location.start_code_units_column(Encoding::UTF_8) + assert_equal 9, location.start_code_units_column(Encoding::UTF_8) assert_equal 7, location.start_code_units_column(Encoding::UTF_16LE) assert_equal 6, location.start_code_units_column(Encoding::UTF_32LE) - assert_equal 7, location.end_code_units_column(Encoding::UTF_8) + assert_equal 13, location.end_code_units_column(Encoding::UTF_8) assert_equal 9, location.end_code_units_column(Encoding::UTF_16LE) assert_equal 7, location.end_code_units_column(Encoding::UTF_32LE) end @@ -154,7 +157,7 @@ module Prism assert_equal 0, location.cached_start_code_units_offset(utf16_cache) assert_equal 0, location.cached_start_code_units_offset(utf32_cache) - assert_equal 1, location.cached_end_code_units_offset(utf8_cache) + assert_equal 4, location.cached_end_code_units_offset(utf8_cache) assert_equal 2, location.cached_end_code_units_offset(utf16_cache) assert_equal 1, location.cached_end_code_units_offset(utf32_cache) @@ -162,26 +165,26 @@ module Prism assert_equal 0, location.cached_start_code_units_column(utf16_cache) assert_equal 0, location.cached_start_code_units_column(utf32_cache) - assert_equal 1, location.cached_end_code_units_column(utf8_cache) + assert_equal 4, location.cached_end_code_units_column(utf8_cache) assert_equal 2, location.cached_end_code_units_column(utf16_cache) assert_equal 1, location.cached_end_code_units_column(utf32_cache) # second 😀 location = result.value.statements.body.first.arguments.arguments.first.location - assert_equal 4, location.cached_start_code_units_offset(utf8_cache) + assert_equal 7, location.cached_start_code_units_offset(utf8_cache) assert_equal 5, location.cached_start_code_units_offset(utf16_cache) assert_equal 4, location.cached_start_code_units_offset(utf32_cache) - assert_equal 5, location.cached_end_code_units_offset(utf8_cache) + assert_equal 11, location.cached_end_code_units_offset(utf8_cache) assert_equal 7, location.cached_end_code_units_offset(utf16_cache) assert_equal 5, location.cached_end_code_units_offset(utf32_cache) - assert_equal 4, location.cached_start_code_units_column(utf8_cache) + assert_equal 7, location.cached_start_code_units_column(utf8_cache) assert_equal 5, location.cached_start_code_units_column(utf16_cache) assert_equal 4, location.cached_start_code_units_column(utf32_cache) - assert_equal 5, location.cached_end_code_units_column(utf8_cache) + assert_equal 11, location.cached_end_code_units_column(utf8_cache) assert_equal 7, location.cached_end_code_units_column(utf16_cache) assert_equal 5, location.cached_end_code_units_column(utf32_cache) end @@ -197,7 +200,7 @@ module Prism assert_equal "😀".b.to_sym, receiver.name location = receiver.location - assert_equal 1, location.end_code_units_column(Encoding::UTF_8) + assert_equal 4, location.end_code_units_column(Encoding::UTF_8) assert_equal 2, location.end_code_units_column(Encoding::UTF_16LE) assert_equal 1, location.end_code_units_column(Encoding::UTF_32LE) end diff --git a/test/prism/ruby/parameters_signature_test.rb b/test/prism/ruby/parameters_signature_test.rb index 9256bcc070..1ca2b144a9 100644 --- a/test/prism/ruby/parameters_signature_test.rb +++ b/test/prism/ruby/parameters_signature_test.rb @@ -50,13 +50,19 @@ module Prism assert_parameters([[:nokey]], "**nil") end + def test_noblock + # FIXME: `compare: RUBY_VERSION >= "4.1"` once builds are available + assert_parameters([[:noblock]], "&nil", compare: false) + end + def test_keyrest_anonymous assert_parameters([[:keyrest, :**]], "**") end - def test_key_ordering - omit("TruffleRuby returns keys in order they were declared") if RUBY_ENGINE == "truffleruby" - assert_parameters([[:keyreq, :a], [:keyreq, :b], [:key, :c], [:key, :d]], "a:, c: 1, b:, d: 2") + if RUBY_ENGINE == "ruby" + def test_key_ordering + assert_parameters([[:keyreq, :a], [:keyreq, :b], [:key, :c], [:key, :d]], "a:, c: 1, b:, d: 2") + end end def test_block @@ -71,12 +77,20 @@ module Prism assert_parameters([[:rest, :*], [:keyrest, :**], [:block, :&]], "...") end + def test_invalid_syntax + e = assert_raise(RuntimeError) do + Prism.parse_statement("def f(**nil, ...); end").parameters.signature + end + assert_equal("Invalid syntax", e.message) + end + private - def assert_parameters(expected, source) + def assert_parameters(expected, source, compare: true) # Compare against our expectation. assert_equal(expected, signature(source)) + return unless compare # Compare against Ruby's expectation. object = Object.new eval("def object.m(#{source}); end") diff --git a/test/prism/ruby/parser_test.rb b/test/prism/ruby/parser_test.rb index cff36f56b0..ad9fa0c92c 100644 --- a/test/prism/ruby/parser_test.rb +++ b/test/prism/ruby/parser_test.rb @@ -5,7 +5,6 @@ require_relative "../test_helper" begin verbose, $VERBOSE = $VERBOSE, nil require "parser/ruby33" - require "prism/translation/parser33" rescue LoadError # In CRuby's CI, we're not going to test against the parser gem because we # don't want to have to install it. So in this case we'll just skip this test. @@ -16,6 +15,7 @@ end # First, opt in to every AST feature. Parser::Builders::Default.modernize +Prism::Translation::Parser::Builder.modernize # The parser gem rejects some strings that would most likely lead to errors # in consumers due to encoding problems. RuboCop however monkey-patches this @@ -54,6 +54,22 @@ Parser::AST::Node.prepend( module Prism class ParserTest < TestCase + # These files contain code with valid syntax that can't be parsed. + skip_syntax_error = [ + # alias/undef with %s(abc) symbol literal + "alias.txt", + "seattlerb/bug_215.txt", + + # %Q with newline delimiter and heredoc interpolation + "heredoc_percent_q_newline_delimiter.txt", + + # 1.. && 2 + "ranges.txt", + + # https://bugs.ruby-lang.org/issues/21168#note-5 + "command_method_call_2.txt", + ] + # These files contain code that is being parsed incorrectly by the parser # gem, and therefore we don't want to compare against our translation. skip_incorrect = [ @@ -80,31 +96,22 @@ module Prism "seattlerb/heredoc_with_extra_carriage_returns_windows.txt", "seattlerb/heredoc_with_only_carriage_returns_windows.txt", "seattlerb/heredoc_with_only_carriage_returns.txt", - ] - # These files are either failing to parse or failing to translate, so we'll - # skip them for now. - skip_all = skip_incorrect | [ + # https://github.com/whitequark/parser/issues/1026 + # Regex with \c escape "unescaping.txt", - "seattlerb/pctW_lineno.txt", "seattlerb/regexp_esc_C_slash.txt", - "unparser/corpus/literal/literal.txt", - "whitequark/parser_slash_slash_n_escaping_in_literals.txt", - ] - # Not sure why these files are failing on JRuby, but skipping them for now. - if RUBY_ENGINE == "jruby" - skip_all.push("emoji_method_calls.txt", "symbols.txt") - end + # https://github.com/whitequark/parser/issues/1084 + "unary_method_calls.txt", + ] # These files are failing to translate their lexer output into the lexer # output expected by the parser gem, so we'll skip them for now. skip_tokens = [ "dash_heredocs.txt", "embdoc_no_newline_at_end.txt", - "heredocs_with_ignored_newlines.txt", "methods.txt", - "strings.txt", "seattlerb/bug169.txt", "seattlerb/case_in.txt", "seattlerb/difficult4__leading_dots2.txt", @@ -114,9 +121,9 @@ module Prism "seattlerb/parse_line_heredoc.txt", "seattlerb/pct_w_heredoc_interp_nested.txt", "seattlerb/required_kwarg_no_value.txt", - "seattlerb/slashy_newlines_within_string.txt", "seattlerb/TestRubyParserShared.txt", "unparser/corpus/literal/assignment.txt", + "unparser/corpus/literal/literal.txt", "whitequark/args.txt", "whitequark/beginless_erange_after_newline.txt", "whitequark/beginless_irange_after_newline.txt", @@ -125,28 +132,85 @@ module Prism "whitequark/lbrace_arg_after_command_args.txt", "whitequark/multiple_pattern_matches.txt", "whitequark/newline_in_hash_argument.txt", - "whitequark/parser_bug_640.txt", "whitequark/pattern_matching_expr_in_paren.txt", "whitequark/pattern_matching_hash.txt", - "whitequark/pin_expr.txt", "whitequark/ruby_bug_14690.txt", "whitequark/ruby_bug_9669.txt", - "whitequark/slash_newline_in_heredocs.txt", "whitequark/space_args_arg_block.txt", "whitequark/space_args_block.txt" ] - Fixture.each do |fixture| + Fixture.each_for_version(except: skip_syntax_error, version: "3.3") do |fixture| define_method(fixture.test_name) do assert_equal_parses( fixture, - compare_asts: !skip_all.include?(fixture.path), + compare_asts: !skip_incorrect.include?(fixture.path), compare_tokens: !skip_tokens.include?(fixture.path), compare_comments: fixture.path != "embdoc_no_newline_at_end.txt" ) end end + def test_non_prism_builder_class_deprecated + warnings = capture_warnings { Prism::Translation::Parser33.new(Parser::Builders::Default.new) } + + assert_include(warnings, "#{__FILE__}:#{__LINE__ - 2}") + assert_include(warnings, "is not a `Prism::Translation::Parser::Builder` subclass") + + warnings = capture_warnings { Prism::Translation::Parser33.new } + assert_empty(warnings) + end + + if RUBY_VERSION >= "3.3" + def test_current_parser_for_current_ruby + major, minor = CURRENT_MAJOR_MINOR.split(".") + # Let's just hope there never is a Ruby 3.10 or similar + expected = major.to_i * 10 + minor.to_i + assert_equal(expected, Translation::ParserCurrent.new.version) + end + end + + def test_invalid_syntax + code = <<~RUBY + foo do + case bar + when + end + end + RUBY + buffer = Parser::Source::Buffer.new("(string)") + buffer.source = code + + parser = Prism::Translation::Parser33.new + parser.diagnostics.all_errors_are_fatal = true + assert_raise(Parser::SyntaxError) { parser.tokenize(buffer) } + end + + def test_it_block_parameter_syntax + assert_new_syntax("3.4/it.txt", Prism::Translation::Parser34) do + s(:begin, + s(:itblock, + s(:send, nil, :x), :it, + s(:lvar, :it)), + s(:itblock, + s(:lambda), :it, + s(:lvar, :it))) + end + end + + def test_nil_block_parameter_syntax + assert_new_syntax("4.1/noblock.txt", Prism::Translation::Parser41) do + s(:begin, + s(:def, :foo, + s(:args, + s(:blocknilarg)), nil), + s(:block, + s(:lambda), + s(:args, + s(:blocknilarg)), nil)) + end + end + private def assert_equal_parses(fixture, compare_asts: true, compare_tokens: true, compare_comments: true) @@ -158,17 +222,13 @@ module Prism parser.diagnostics.all_errors_are_fatal = true expected_ast, expected_comments, expected_tokens = - begin - ignore_warnings { parser.tokenize(buffer) } - rescue ArgumentError, Parser::SyntaxError - return - end + ignore_warnings { parser.tokenize(buffer) } actual_ast, actual_comments, actual_tokens = ignore_warnings { Prism::Translation::Parser33.new.tokenize(buffer) } if expected_ast == actual_ast - if !compare_asts + if !compare_asts && !Fixture.custom_base_path? puts "#{fixture.path} is now passing" end @@ -179,7 +239,7 @@ module Prism rescue Test::Unit::AssertionFailedError raise if compare_tokens else - puts "#{fixture.path} is now passing" if !compare_tokens + puts "#{fixture.path} is now passing" if !compare_tokens && !Fixture.custom_base_path? end assert_equal_comments(expected_comments, actual_comments) if compare_comments @@ -245,5 +305,19 @@ module Prism "actual: #{actual_comments.inspect}" } end + + def assert_new_syntax(path, parser, &sexp) + fixture_path = Pathname(__dir__).join("../../../test/prism/fixtures", path) + + buffer = Parser::Source::Buffer.new(fixture_path) + buffer.source = fixture_path.read + actual_ast = parser.new.tokenize(buffer)[0] + + assert_equal(parse_sexp(&sexp), actual_ast.to_sexp) + end + + def parse_sexp(&block) + Class.new { extend AST::Sexp }.instance_eval(&block).to_sexp + end end end diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 7ed32ed216..4fff630561 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -1,37 +1,60 @@ # frozen_string_literal: true -return if RUBY_VERSION < "3.3" +return if RUBY_VERSION < "3.3" || RUBY_ENGINE != "ruby" require_relative "../test_helper" +require "ripper" module Prism class RipperTest < TestCase # Skip these tests that Ripper is reporting the wrong results for. incorrect = [ # Ripper incorrectly attributes the block to the keyword. - "seattlerb/block_break.txt", - "seattlerb/block_next.txt", "seattlerb/block_return.txt", - "whitequark/break_block.txt", - "whitequark/next_block.txt", "whitequark/return_block.txt", - # Ripper is not accounting for locals created by patterns using the ** - # operator within an `in` clause. - "seattlerb/parse_pattern_058.txt", - # Ripper cannot handle named capture groups in regular expressions. "regex.txt", - "regex_char_width.txt", - "whitequark/lvar_injecting_match.txt", # Ripper fails to understand some structures that span across heredocs. - "spanning_heredoc.txt" + "spanning_heredoc.txt", + + # Ripper interprets circular keyword arguments as method calls. + "3.4/circular_parameters.txt", + + # Ripper doesn't emit `args_add_block` when endless method is prefixed by modifier. + "4.0/endless_methods_command_call.txt", + + # https://bugs.ruby-lang.org/issues/21168#note-5 + "command_method_call_2.txt", ] + if RUBY_VERSION.start_with?("3.3.") + incorrect += [ + "whitequark/lvar_injecting_match.txt", + "seattlerb/parse_pattern_058.txt", + "regex_char_width.txt", + ] + end + + if RUBY_VERSION.start_with?("4.") + incorrect += [ + # https://bugs.ruby-lang.org/issues/21945 + "and_or_with_suffix.txt", + ] + end + + # https://bugs.ruby-lang.org/issues/21669 + incorrect << "4.1/void_value.txt" + # https://bugs.ruby-lang.org/issues/19107 + incorrect << "4.1/trailing_comma_after_method_arguments.txt" + # Skip these tests that we haven't implemented yet. - omitted = [ + omitted_sexp_raw = [ + "bom_leading_space.txt", + "bom_spaces.txt", "dos_endings.txt", + "heredocs_with_fake_newlines.txt", "heredocs_with_ignored_newlines.txt", "seattlerb/block_call_dot_op2_brace_block.txt", "seattlerb/block_command_operation_colon.txt", @@ -50,14 +73,237 @@ module Prism "whitequark/slash_newline_in_heredocs.txt" ] - Fixture.each(except: incorrect | omitted) do |fixture| - define_method(fixture.test_name) { assert_ripper(fixture.read) } + omitted_lex = [ + "heredoc_with_escaped_newline_at_start.txt", + "heredocs_with_fake_newlines.txt", + "indented_file_end.txt", + "spanning_heredoc_newlines.txt", + "whitequark/dedenting_heredoc.txt", + "whitequark/procarg0.txt", + ] + + omitted_scan = [ + "bom_leading_space.txt", + "bom_spaces.txt", + "dos_endings.txt", + "heredocs_with_fake_newlines.txt", + "rescue_modifier.txt", + "seattlerb/block_call_dot_op2_brace_block.txt", + "seattlerb/block_command_operation_colon.txt", + "seattlerb/block_command_operation_dot.txt", + "seattlerb/case_in.txt", + "seattlerb/heredoc__backslash_dos_format.txt", + "seattlerb/heredoc_backslash_nl.txt", + "seattlerb/heredoc_nested.txt", + "seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt", + "seattlerb/heredoc_squiggly_empty.txt", + "seattlerb/masgn_command_call.txt", + "seattlerb/messy_op_asgn_lineno.txt", + "seattlerb/op_asgn_primary_colon_const_command_call.txt", + "seattlerb/parse_pattern_076.txt", + "seattlerb/pct_w_heredoc_interp_nested.txt", + "tilde_heredocs.txt", + "unparser/corpus/literal/assignment.txt", + "unparser/corpus/literal/pattern.txt", + "unparser/corpus/semantic/dstr.txt", + "variables.txt", + "whitequark/dedenting_heredoc.txt", + "whitequark/masgn_nested.txt", + "whitequark/newline_in_hash_argument.txt", + "whitequark/numparam_ruby_bug_19025.txt", + "whitequark/op_asgn_cmd.txt", + "whitequark/parser_drops_truncated_parts_of_squiggly_heredoc.txt", + "whitequark/parser_slash_slash_n_escaping_in_literals.txt", + "whitequark/pattern_matching_nil_pattern.txt", + "whitequark/ruby_bug_12402.txt", + "whitequark/ruby_bug_18878.txt", + "whitequark/send_block_chain_cmd.txt", + "whitequark/slash_newline_in_heredocs.txt", + ] + + Fixture.each_for_current_ruby(except: incorrect | omitted_sexp_raw) do |fixture| + define_method("#{fixture.test_name}_sexp_raw") { assert_ripper_sexp_raw(fixture.read) } + end + + Fixture.each_for_current_ruby(except: incorrect | omitted_lex) do |fixture| + define_method("#{fixture.test_name}_lex") { assert_ripper_lex(fixture.read) } + end + + def test_lex_ignored_missing_heredoc_end + ["", "-", "~"].each do |type| + source = "<<#{type}FOO\n" + assert_ripper_lex(source) + + source = "<<#{type}'FOO'\n" + assert_ripper_lex(source) + end + end + + UNSUPPORTED_EVENTS = %i[comma ignored_nl nl semicolon sp ignored_sp] + # Events that are currently not emitted + SUPPORTED_EVENTS = Translation::Ripper::EVENTS - UNSUPPORTED_EVENTS + # Events that assert against their line/column + CHECK_LOCATION_EVENTS = %i[kw op lbrace rbrace lbracket rbracket lparen rparen words_sep label_end] + + module Events + attr_reader :events + + def initialize(...) + super + @events = [] + end + + SUPPORTED_EVENTS.each do |event| + define_method(:"on_#{event}") do |*args| + if CHECK_LOCATION_EVENTS.include?(event) + @events << [event, lineno, column, *args] + else + @events << [event, *args] + end + super(*args) + end + end + end + + class RipperEvents < Ripper + include Events + end + + class PrismEvents < Translation::Ripper + include Events + end + + class ObjectEvents < Translation::Ripper + OBJECT = BasicObject.new + SUPPORTED_EVENTS.each do |event| + define_method(:"on_#{event}") { |*args| OBJECT } + end + end + + Fixture.each_for_current_ruby(except: incorrect | omitted_scan) do |fixture| + define_method("#{fixture.test_name}_events") do + source = fixture.read + # Similar to test/ripper/assert_parse_files.rb in CRuby + object_events = ObjectEvents.new(source) + assert_nothing_raised { object_events.parse } + + ripper = RipperEvents.new(source, fixture.path) + prism = PrismEvents.new(source, fixture.path) + ripper.parse + prism.parse + # Check that the same events are emitted, regardless of order + assert_equal(ripper.events.sort_by(&:inspect), prism.events.sort_by(&:inspect)) + end + end + + def test_lexer + lexer = Translation::Ripper::Lexer.new("foo") + expected = [[1, 0], :on_ident, "foo", Translation::Ripper::EXPR_CMDARG] + + assert_equal([expected], lexer.lex) + assert_equal(expected, lexer.parse[0].to_a) + assert_equal(lexer.parse[0].to_a, lexer.scan[0].to_a) + + assert_equal(%i[on_int on_sp on_op], Translation::Ripper::Lexer.new("1 +").lex.map { |token| token[1] }) + assert_raise(SyntaxError) { Translation::Ripper::Lexer.new("1 +").lex(raise_errors: true) } + end + + + # On syntax invalid code the output doesn't always match up + # In these cases we just want to make sure that it doesn't raise. + def test_lex_invalid_syntax + assert_nothing_raised do + Translation::Ripper.lex('scan/\p{alpha}/') + end + + assert_equal(Ripper.lex('if;)'), Translation::Ripper.lex('if;)')) + end + + def test_tokenize + source = "foo;1;BAZ" + assert_equal(Ripper.tokenize(source), Translation::Ripper.tokenize(source)) + end + + def test_encoding + source = '"わたし"'.encode(Encoding::Windows_31J) + assert_equal(Ripper.tokenize(source), Translation::Ripper.tokenize(source)) + assert_equal(Ripper.sexp(source), Translation::Ripper.sexp(source)) + end + + def test_sexp_coercion + string_like = Object.new + def string_like.to_str + "a" + end + assert_equal Ripper.sexp(string_like), Translation::Ripper.sexp(string_like) + + File.open(__FILE__) do |file1| + File.open(__FILE__) do |file2| + assert_equal Ripper.sexp(file1), Translation::Ripper.sexp(file2) + end + end + + File.open(__FILE__) do |file1| + File.open(__FILE__) do |file2| + object1_with_gets = Object.new + object1_with_gets.define_singleton_method(:gets) do + file1.gets + end + + object2_with_gets = Object.new + object2_with_gets.define_singleton_method(:gets) do + file2.gets + end + + assert_equal Ripper.sexp(object1_with_gets), Translation::Ripper.sexp(object2_with_gets) + end + end + end + + def test_lex_coersion + string_like = Object.new + def string_like.to_str + "a" + end + assert_equal Ripper.lex(string_like), Translation::Ripper.lex(string_like) + end + + # Check that the hardcoded values don't change without us noticing. + def test_internals + actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort + expected = Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort + + assert_equal(expected, actual) + expected.zip(actual).each do |ripper, prism| + assert_equal(Ripper.const_get(ripper), Translation::Ripper.const_get(prism)) + end end private - def assert_ripper(source) + def assert_ripper_sexp_raw(source) assert_equal Ripper.sexp_raw(source), Prism::Translation::Ripper.sexp_raw(source) end + + def assert_ripper_lex(source) + prism = Translation::Ripper.lex(source) + ripper = Ripper.lex(source) + + # Prism emits tokens by their order in the code, not in parse order + ripper.sort_by! { |elem| elem[0] } + + [prism.size, ripper.size].max.times do |index| + expected = ripper[index] + actual = prism[index] + + # There are some tokens that have slightly different state that do not + # effect the parse tree, so they may not match. + if expected && actual && expected[1] == actual[1] && %i[on_comment on_heredoc_end on_embexpr_end on_sp].include?(expected[1]) + expected[3] = actual[3] = nil + end + + assert_equal(expected, actual) + end + end end end diff --git a/test/prism/ruby/ruby_parser_test.rb b/test/prism/ruby/ruby_parser_test.rb index 1d530dd13b..bc89bdae72 100644 --- a/test/prism/ruby/ruby_parser_test.rb +++ b/test/prism/ruby/ruby_parser_test.rb @@ -13,38 +13,23 @@ rescue LoadError return end -# We want to also compare lines and files to make sure we're setting them -# correctly. -Sexp.prepend( - Module.new do - def ==(other) - super && line == other.line && file == other.file # && line_max == other.line_max - end - end -) - module Prism class RubyParserTest < TestCase todos = [ + "character_literal.txt", "encoding_euc_jp.txt", - "newline_terminated.txt", "regex_char_width.txt", - "seattlerb/bug169.txt", "seattlerb/masgn_colon3.txt", "seattlerb/messy_op_asgn_lineno.txt", "seattlerb/op_asgn_primary_colon_const_command_call.txt", "seattlerb/regexp_esc_C_slash.txt", "seattlerb/str_lit_concat_bad_encodings.txt", + "strings.txt", "unescaping.txt", - "unparser/corpus/literal/kwbegin.txt", - "unparser/corpus/literal/send.txt", "whitequark/masgn_const.txt", "whitequark/pattern_matching_constants.txt", - "whitequark/pattern_matching_implicit_array_match.txt", "whitequark/pattern_matching_single_match.txt", "whitequark/ruby_bug_12402.txt", - "whitequark/ruby_bug_14690.txt", - "whitequark/space_args_block.txt" ] # https://github.com/seattlerb/ruby_parser/issues/344 @@ -52,6 +37,9 @@ module Prism "alias.txt", "dsym_str.txt", "dos_endings.txt", + "heredoc_dedent_line_continuation.txt", + "heredoc_percent_q_newline_delimiter.txt", + "heredocs_with_fake_newlines.txt", "heredocs_with_ignored_newlines.txt", "method_calls.txt", "methods.txt", @@ -69,7 +57,9 @@ module Prism "seattlerb/heredoc_with_only_carriage_returns.txt", "spanning_heredoc_newlines.txt", "spanning_heredoc.txt", + "symbols.txt", "tilde_heredocs.txt", + "unary_method_calls.txt", "unparser/corpus/literal/literal.txt", "while.txt", "whitequark/cond_eflipflop.txt", @@ -87,10 +77,20 @@ module Prism "whitequark/ruby_bug_11989.txt", "whitequark/ruby_bug_18878.txt", "whitequark/ruby_bug_19281.txt", - "whitequark/slash_newline_in_heredocs.txt" + "whitequark/slash_newline_in_heredocs.txt", + + "3.3-3.3/block_args_in_array_assignment.txt", + "3.3-3.3/it_with_ordinary_parameter.txt", + "3.3-3.3/keyword_args_in_array_assignment.txt", + "3.3-3.3/return_in_sclass.txt", + + "3.3-4.0/void_value.txt", + + # https://bugs.ruby-lang.org/issues/21168#note-5 + "command_method_call_2.txt", ] - Fixture.each(except: failures) do |fixture| + Fixture.each_for_version(version: "3.3", except: failures) do |fixture| define_method(fixture.test_name) do assert_ruby_parser(fixture, todos.include?(fixture.path)) end @@ -102,10 +102,16 @@ module Prism source = fixture.read expected = ignore_warnings { ::RubyParser.new.parse(source, fixture.path) } actual = Prism::Translation::RubyParser.new.parse(source, fixture.path) + on_failure = -> { message(expected, actual) } if !allowed_failure - assert_equal(expected, actual, -> { message(expected, actual) }) - elsif expected == actual + assert_equal(expected, actual, on_failure) + + unless actual.nil? + assert_equal(expected.line, actual.line, on_failure) + assert_equal(expected.file, actual.file, on_failure) + end + elsif expected == actual && expected.line && actual.line && expected.file == actual.file puts "#{name} now passes" end end diff --git a/test/prism/ruby/source_test.rb b/test/prism/ruby/source_test.rb new file mode 100644 index 0000000000..f7cf4fe83a --- /dev/null +++ b/test/prism/ruby/source_test.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require_relative "../test_helper" + +module Prism + class SourceTest < TestCase + def test_byte_offset + source = Prism.parse(<<~SRC).source + abcd + efgh + ijkl + SRC + + assert_equal 0, source.byte_offset(1, 0) + assert_equal 5, source.byte_offset(2, 0) + assert_equal 10, source.byte_offset(3, 0) + assert_equal 15, source.byte_offset(4, 0) + + error = assert_raise(ArgumentError) { source.byte_offset(5, 0) } + assert_equal "line 5 is out of range", error.message + + error = assert_raise(ArgumentError) { source.byte_offset(0, 0) } + assert_equal "line 0 is out of range", error.message + + error = assert_raise(ArgumentError) { source.byte_offset(-1, 0) } + assert_equal "line -1 is out of range", error.message + end + + def test_byte_offset_with_start_line + source = Prism.parse(<<~SRC, line: 11).source + abcd + efgh + ijkl + SRC + + assert_equal 0, source.byte_offset(11, 0) + assert_equal 5, source.byte_offset(12, 0) + assert_equal 10, source.byte_offset(13, 0) + assert_equal 15, source.byte_offset(14, 0) + + error = assert_raise(ArgumentError) { source.byte_offset(15, 0) } + assert_equal "line 15 is out of range", error.message + + error = assert_raise(ArgumentError) { source.byte_offset(10, 0) } + assert_equal "line 10 is out of range", error.message + + error = assert_raise(ArgumentError) { source.byte_offset(9, 0) } + assert_equal "line 9 is out of range", error.message + end + end +end |
