1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
|
# frozen_string_literal: true
module IRB
module NestingParser
IGNORE_TOKENS = %i[on_sp on_ignored_nl on_comment on_embdoc_beg on_embdoc on_embdoc_end]
# Scan each token and call the given block with array of token and other information for parsing
def self.scan_opens(tokens)
opens = []
pending_heredocs = []
first_token_on_line = true
tokens.each do |t|
skip = false
last_tok, state, args = opens.last
case state
when :in_alias_undef
skip = t.event == :on_kw
when :in_unquoted_symbol
unless IGNORE_TOKENS.include?(t.event)
opens.pop
skip = true
end
when :in_lambda_head
opens.pop if t.event == :on_tlambeg || (t.event == :on_kw && t.tok == 'do')
when :in_method_head
unless IGNORE_TOKENS.include?(t.event)
next_args = []
body = nil
if args.include?(:receiver)
case t.event
when :on_lparen, :on_ivar, :on_gvar, :on_cvar
# def (receiver). | def @ivar. | def $gvar. | def @@cvar.
next_args << :dot
when :on_kw
case t.tok
when 'self', 'true', 'false', 'nil'
# def self(arg) | def self.
next_args.push(:arg, :dot)
else
# def if(arg)
skip = true
next_args << :arg
end
when :on_op, :on_backtick
# def +(arg)
skip = true
next_args << :arg
when :on_ident, :on_const
# def a(arg) | def a.
next_args.push(:arg, :dot)
end
end
if args.include?(:dot)
# def receiver.name
next_args << :name if t.event == :on_period || (t.event == :on_op && t.tok == '::')
end
if args.include?(:name)
if %i[on_ident on_const on_op on_kw on_backtick].include?(t.event)
# def name(arg) | def receiver.name(arg)
next_args << :arg
skip = true
end
end
if args.include?(:arg)
case t.event
when :on_nl, :on_semicolon
# def receiver.f;
body = :normal
when :on_lparen
# def receiver.f()
next_args << :eq
else
if t.event == :on_op && t.tok == '='
# def receiver.f =
body = :oneliner
else
# def receiver.f arg
next_args << :arg_without_paren
end
end
end
if args.include?(:eq)
if t.event == :on_op && t.tok == '='
body = :oneliner
else
body = :normal
end
end
if args.include?(:arg_without_paren)
if %i[on_semicolon on_nl].include?(t.event)
# def f a;
body = :normal
else
# def f a, b
next_args << :arg_without_paren
end
end
if body == :oneliner
opens.pop
elsif body
opens[-1] = [last_tok, nil]
else
opens[-1] = [last_tok, :in_method_head, next_args]
end
end
when :in_for_while_until_condition
if t.event == :on_semicolon || t.event == :on_nl || (t.event == :on_kw && t.tok == 'do')
skip = true if t.event == :on_kw && t.tok == 'do'
opens[-1] = [last_tok, nil]
end
end
unless skip
case t.event
when :on_kw
case t.tok
when 'begin', 'class', 'module', 'do', 'case'
opens << [t, nil]
when 'end'
opens.pop
when 'def'
opens << [t, :in_method_head, [:receiver, :name]]
when 'if', 'unless'
unless t.state.allbits?(Ripper::EXPR_LABEL)
opens << [t, nil]
end
when 'while', 'until'
unless t.state.allbits?(Ripper::EXPR_LABEL)
opens << [t, :in_for_while_until_condition]
end
when 'ensure', 'rescue'
unless t.state.allbits?(Ripper::EXPR_LABEL)
opens.pop
opens << [t, nil]
end
when 'alias'
opens << [t, :in_alias_undef, 2]
when 'undef'
opens << [t, :in_alias_undef, 1]
when 'elsif', 'else', 'when'
opens.pop
opens << [t, nil]
when 'for'
opens << [t, :in_for_while_until_condition]
when 'in'
if last_tok&.event == :on_kw && %w[case in].include?(last_tok.tok) && first_token_on_line
opens.pop
opens << [t, nil]
end
end
when :on_tlambda
opens << [t, :in_lambda_head]
when :on_lparen, :on_lbracket, :on_lbrace, :on_tlambeg, :on_embexpr_beg, :on_embdoc_beg
opens << [t, nil]
when :on_rparen, :on_rbracket, :on_rbrace, :on_embexpr_end, :on_embdoc_end
opens.pop
when :on_heredoc_beg
pending_heredocs << t
when :on_heredoc_end
opens.pop
when :on_backtick
opens << [t, nil] if t.state.allbits?(Ripper::EXPR_BEG)
when :on_tstring_beg, :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg, :on_regexp_beg
opens << [t, nil]
when :on_tstring_end, :on_regexp_end, :on_label_end
opens.pop
when :on_symbeg
if t.tok == ':'
opens << [t, :in_unquoted_symbol]
else
opens << [t, nil]
end
end
end
if t.event == :on_nl || t.event == :on_semicolon
first_token_on_line = true
elsif t.event != :on_sp
first_token_on_line = false
end
if pending_heredocs.any? && t.tok.include?("\n")
pending_heredocs.reverse_each { |t| opens << [t, nil] }
pending_heredocs = []
end
if opens.last && opens.last[1] == :in_alias_undef && !IGNORE_TOKENS.include?(t.event) && t.event != :on_heredoc_end
tok, state, arg = opens.pop
opens << [tok, state, arg - 1] if arg >= 1
end
yield t, opens if block_given?
end
opens.map(&:first) + pending_heredocs.reverse
end
def self.open_tokens(tokens)
# scan_opens without block will return a list of open tokens at last token position
scan_opens(tokens)
end
# Calculates token information [line_tokens, prev_opens, next_opens, min_depth] for each line.
# Example code
# ["hello
# world"+(
# First line
# line_tokens: [[lbracket, '['], [tstring_beg, '"'], [tstring_content("hello\nworld"), "hello\n"]]
# prev_opens: []
# next_tokens: [lbracket, tstring_beg]
# min_depth: 0 (minimum at beginning of line)
# Second line
# line_tokens: [[tstring_content("hello\nworld"), "world"], [tstring_end, '"'], [op, '+'], [lparen, '(']]
# prev_opens: [lbracket, tstring_beg]
# next_tokens: [lbracket, lparen]
# min_depth: 1 (minimum just after tstring_end)
def self.parse_by_line(tokens)
line_tokens = []
prev_opens = []
min_depth = 0
output = []
last_opens = scan_opens(tokens) do |t, opens|
depth = t == opens.last&.first ? opens.size - 1 : opens.size
min_depth = depth if depth < min_depth
if t.tok.include?("\n")
t.tok.each_line do |line|
line_tokens << [t, line]
next if line[-1] != "\n"
next_opens = opens.map(&:first)
output << [line_tokens, prev_opens, next_opens, min_depth]
prev_opens = next_opens
min_depth = prev_opens.size
line_tokens = []
end
else
line_tokens << [t, t.tok]
end
end
output << [line_tokens, prev_opens, last_opens, min_depth] if line_tokens.any?
output
end
end
end
|
