1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
# frozen_string_literal: true
begin
require 'ripper'
require 'test/unit'
module TestRipper; end
rescue LoadError
end
class TestRipper::Lexer < Test::Unit::TestCase
def test_nested_dedent_heredoc
bug = '[ruby-core:80977] [Bug #13536]'
str = <<~'E'
<<~"D"
#{
<<~"B"
this must be a valid ruby
B
}
D
E
assert_equal(str, Ripper.tokenize(str).join(""), bug)
str = <<~'E'
<<~"D"
#{
<<~"B"
this must be a valid ruby
B
}
D
E
assert_equal(str, Ripper.tokenize(str).join(""), bug)
end
def test_embedded_expr_in_heredoc
src = <<~'E'
<<~B
#{1}
B
E
expect = %I[
on_heredoc_beg
on_nl
on_ignored_sp
on_embexpr_beg
on_int
on_embexpr_end
on_tstring_content
on_heredoc_end
]
assert_equal expect, Ripper.lex(src).map {|e| e[1]}
end
def test_space_after_expr_in_heredoc
src = <<~'E'
<<~B
#{1} a
B
E
expect = %I[
on_heredoc_beg
on_nl
on_ignored_sp
on_embexpr_beg
on_int
on_embexpr_end
on_tstring_content
on_heredoc_end
]
assert_equal expect, Ripper.lex(src).map {|e| e[1]}
end
def test_expr_at_beginning_in_heredoc
src = <<~'E'
<<~B
a
#{1}
B
E
expect = %I[
on_heredoc_beg
on_nl
on_tstring_content
on_embexpr_beg
on_int
on_embexpr_end
on_tstring_content
on_heredoc_end
]
assert_equal expect, Ripper.lex(src).map {|e| e[1]}
end
def test_slice
assert_equal "string\#{nil}\n",
Ripper.slice(%(<<HERE\nstring\#{nil}\nHERE), "heredoc_beg .*? nl $(.*?) heredoc_end", 1)
end
def state(name)
Ripper::Lexer::State.new(Ripper.const_get(name))
end
def test_state_after_ivar
assert_equal [[1,0],:on_ivar,"@a",state(:EXPR_END)], Ripper.lex("@a").last
assert_equal [[1,1],:on_ivar,"@a",state(:EXPR_ENDFN)], Ripper.lex(":@a").last
assert_equal [[1,1],:on_int,"1",state(:EXPR_END)], Ripper.lex("@1").last
assert_equal [[1,2],:on_int,"1",state(:EXPR_END)], Ripper.lex(":@1").last
end
def test_state_after_cvar
assert_equal [[1,0],:on_cvar,"@@a",state(:EXPR_END)], Ripper.lex("@@a").last
assert_equal [[1,1],:on_cvar,"@@a",state(:EXPR_ENDFN)], Ripper.lex(":@@a").last
assert_equal [[1,2],:on_int,"1",state(:EXPR_END)], Ripper.lex("@@1").last
assert_equal [[1,3],:on_int,"1",state(:EXPR_END)], Ripper.lex(":@@1").last
end
def test_token_aftr_error_heredoc
code = "<<A.upcase\n"
result = Ripper::Lexer.new(code).scan
message = proc {result.pretty_inspect}
expected = [
[[1, 0], :on_heredoc_beg, "<<A", state(:EXPR_BEG)],
[[1, 2], :compile_error, "A", state(:EXPR_BEG), "can't find string \"A\" anywhere before EOF"],
[[1, 3], :on_period, ".", state(:EXPR_DOT)],
[[1, 4], :on_ident, "upcase", state(:EXPR_ARG)],
[[1, 10], :on_nl, "\n", state(:EXPR_BEG)],
]
pos = 0
expected.each_with_index do |ex, i|
s = result[i]
assert_equal ex, s.to_a, message
if pos > s.pos[1]
assert_equal pos, s.pos[1] + s.tok.bytesize, message
else
assert_equal pos, s.pos[1], message
pos += s.tok.bytesize
end
end
assert_equal pos, code.bytesize
assert_equal expected.size, result.size
end
def test_trailing_on_embexpr_end
# This is useful for scanning a template engine literal `{ foo, bar: baz }`
# whose body inside brackes works like trailing method arguments, like Haml.
token = Ripper.lex("a( foo, bar: baz }").last
assert_equal [[1, 17], :on_embexpr_end, "}", state(:EXPR_ARG)], token
end
BAD_CODE = [
[:parse_error, 'def req(true) end', %r[unexpected `true'], 'true'],
[:assign_error, 'begin; nil = 1; end', %r[assign to nil], 'nil'],
[:alias_error, 'begin; alias $x $1; end', %r[number variables], '$1'],
[:class_name_error, 'class bad; end', %r[class/module name], 'bad'],
[:param_error, 'def req(@a) end', %r[formal argument.*instance], '@a'],
[:param_error, 'def req(a?:) end', %r[formal argument must.*local], 'a?'],
]
def test_raise_errors_keyword
all_assertions do |all|
BAD_CODE.each do |(err, code, message)|
all.for([err, code]) do
assert_raise_with_message(SyntaxError, message) { Ripper.tokenize(code, raise_errors: true) }
end
end
end
end
def test_tokenize_with_syntax_error
all_assertions do |all|
BAD_CODE.each do |(err, code)|
all.for([err, code]) do
assert_equal "end", Ripper.tokenize(code).last
end
end
end
end
def test_lex_with_syntax_error
all_assertions do |all|
BAD_CODE.each do |(err, code)|
all.for([err, code]) do
assert_equal [[1, code.size-3], :on_kw, "end", state(:EXPR_END)], Ripper.lex(code).last
end
end
end
end
def test_lexer_scan_with_syntax_error
all_assertions do |all|
BAD_CODE.each do |(err, code, message, token)|
all.for([err, code]) do
lexer = Ripper::Lexer.new(code)
elems = lexer.scan
assert_predicate lexer, :error?
error = lexer.errors.first
assert_match message, error.message
i = elems.find_index{|e| e == error}
assert_operator 0...elems.size, :include?, i
elem = elems[i + 1]
assert_not_equal error.event, elem.event
assert_equal error.pos, elem.pos
assert_equal error.tok, elem.tok
assert_equal error.state, elem.state
end
end
end
end
end
|
