1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
|
# frozen_string_literal: true
return if RUBY_VERSION < "3.3" || RUBY_ENGINE != "ruby"
require_relative "../test_helper"
require "ripper"
module Prism
class RipperTest < TestCase
# Skip these tests that Ripper is reporting the wrong results for.
incorrect = [
# Ripper incorrectly attributes the block to the keyword.
"seattlerb/block_return.txt",
"whitequark/return_block.txt",
# Ripper cannot handle named capture groups in regular expressions.
"regex.txt",
# Ripper fails to understand some structures that span across heredocs.
"spanning_heredoc.txt",
# Ripper interprets circular keyword arguments as method calls.
"3.4/circular_parameters.txt",
# Ripper doesn't emit `args_add_block` when endless method is prefixed by modifier.
"4.0/endless_methods_command_call.txt",
# https://bugs.ruby-lang.org/issues/21168#note-5
"command_method_call_2.txt",
]
if RUBY_VERSION.start_with?("3.3.")
incorrect += [
"whitequark/lvar_injecting_match.txt",
"seattlerb/parse_pattern_058.txt",
"regex_char_width.txt",
]
end
# Skip these tests that we haven't implemented yet.
omitted_sexp_raw = [
"bom_leading_space.txt",
"bom_spaces.txt",
"dos_endings.txt",
"heredocs_with_fake_newlines.txt",
"heredocs_with_ignored_newlines.txt",
"seattlerb/block_call_dot_op2_brace_block.txt",
"seattlerb/block_command_operation_colon.txt",
"seattlerb/block_command_operation_dot.txt",
"seattlerb/heredoc__backslash_dos_format.txt",
"seattlerb/heredoc_backslash_nl.txt",
"seattlerb/heredoc_nested.txt",
"seattlerb/heredoc_squiggly_blank_line_plus_interpolation.txt",
"tilde_heredocs.txt",
"unparser/corpus/semantic/dstr.txt",
"whitequark/dedenting_heredoc.txt",
"whitequark/parser_drops_truncated_parts_of_squiggly_heredoc.txt",
"whitequark/parser_slash_slash_n_escaping_in_literals.txt",
"whitequark/ruby_bug_18878.txt",
"whitequark/send_block_chain_cmd.txt",
"whitequark/slash_newline_in_heredocs.txt"
]
omitted_lex = [
"comments.txt",
"heredoc_percent_q_newline_delimiter.txt",
"heredoc_with_escaped_newline_at_start.txt",
"heredocs_with_fake_newlines.txt",
"indented_file_end.txt",
"seattlerb/TestRubyParserShared.txt",
"seattlerb/class_comments.txt",
"seattlerb/module_comments.txt",
"seattlerb/parse_line_block_inline_comment_leading_newlines.txt",
"seattlerb/parse_line_block_inline_multiline_comment.txt",
"spanning_heredoc_newlines.txt",
"strings.txt",
"whitequark/dedenting_heredoc.txt",
"whitequark/procarg0.txt",
]
Fixture.each_for_current_ruby(except: incorrect | omitted_sexp_raw) do |fixture|
define_method("#{fixture.test_name}_sexp_raw") { assert_ripper_sexp_raw(fixture.read) }
end
Fixture.each_for_current_ruby(except: incorrect | omitted_lex) do |fixture|
define_method("#{fixture.test_name}_lex") { assert_ripper_lex(fixture.read) }
end
def test_lexer
lexer = Translation::Ripper::Lexer.new("foo")
expected = [[1, 0], :on_ident, "foo", Translation::Ripper::EXPR_CMDARG]
assert_equal([expected], lexer.lex)
assert_equal(expected, lexer.parse[0].to_a)
assert_equal(lexer.parse[0].to_a, lexer.scan[0].to_a)
assert_equal(%i[on_int on_sp on_op], Translation::Ripper::Lexer.new("1 +").lex.map(&:event))
assert_raise(SyntaxError) { Translation::Ripper::Lexer.new("1 +").lex(raise_errors: true) }
end
def test_tokenize
source = "foo;1;BAZ"
assert_equal(Ripper.tokenize(source), Translation::Ripper.tokenize(source))
end
# Check that the hardcoded values don't change without us noticing.
def test_internals
actual = Translation::Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
expected = Ripper.constants.select { |name| name.start_with?("EXPR_") }.sort
assert_equal(expected, actual)
expected.zip(actual).each do |ripper, prism|
assert_equal(Ripper.const_get(ripper), Translation::Ripper.const_get(prism))
end
end
private
def assert_ripper_sexp_raw(source)
assert_equal Ripper.sexp_raw(source), Prism::Translation::Ripper.sexp_raw(source)
end
def assert_ripper_lex(source)
prism = Translation::Ripper.lex(source)
ripper = Ripper.lex(source)
# Prism emits tokens by their order in the code, not in parse order
ripper.sort_by! { |elem| elem[0] }
[prism.size, ripper.size].max.times do |i|
expected = ripper[i]
actual = prism[i]
# Since tokens related to heredocs are not emitted in the same order,
# the state also doesn't line up.
if expected && actual && expected[1] == :on_heredoc_end && actual[1] == :on_heredoc_end
expected[3] = actual[3] = nil
end
assert_equal(expected, actual)
end
end
end
end
|
