1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
|
#!/usr/bin/ruby
# tool/update-deps verify makefile dependencies.
# Requirements:
# gcc 4.5 (for -save-temps=obj option)
# GNU make (for -p option)
#
# Usage:
# 1. Compile ruby with -save-temps=obj option.
# Ex. ./configure debugflags='-save-temps=obj -g' && make all golf
# 2. Run tool/update-deps to show dependency problems.
# Ex. ruby tool/update-deps
# 3. Use --fix to fix makefiles.
# Ex. ruby tool/update-deps --fix
#
# Other usages:
# * Fix makefiles using previously detected dependency problems
# Ex. ruby tool/update-deps --actual-fix [file]
# "ruby tool/update-deps --fix" is same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix".
require 'optparse'
require 'stringio'
require 'pathname'
require 'pp'
ENV['LC_ALL'] = 'C'
$opt_fix = false
$opt_a = false
$opt_actual_fix = false
def optionparser
op = OptionParser.new
op.banner = 'Usage: ruby tool/update-deps'
op.def_option('-a', 'show valid dependencies') { $opt_a = true }
op.def_option('--fix') { $opt_fix = true }
op.def_option('--actual-fix') { $opt_actual_fix = true }
op
end
def read_make_deps(cwd)
dependencies = {}
make_p = `make -p all miniruby ruby golf 2> /dev/null`
dirstack = [cwd]
curdir = nil
make_p.scan(%r{Entering\ directory\ ['`](.*)'|
^\#\ (GNU\ Make)\ |
^CURDIR\ :=\ (.*)|
^([/0-9a-zA-Z._-]+):(.*)\n((?:\#.*\n)*)|
^\#\ (Finished\ Make\ data\ base\ on)\ |
Leaving\ directory\ ['`](.*)'}x) {
directory_enter = $1
data_base_start = $2
data_base_curdir = $3
rule_target = $4
rule_sources = $5
rule_desc = $6
data_base_end = $7
directory_leave = $8
#p $~
if directory_enter
enter_dir = Pathname(directory_enter)
#p [:enter, enter_dir]
dirstack.push enter_dir
elsif data_base_start
curdir = nil
elsif data_base_curdir
curdir = Pathname(data_base_curdir)
elsif rule_target && rule_sources && rule_desc &&
/Modification time never checked/ !~ rule_desc # This pattern match eliminates rules which VPATH is not expanded.
target = rule_target
deps = rule_sources
deps = deps.scan(%r{[/0-9a-zA-Z._-]+})
next if /\.o\z/ !~ target.to_s
next if /\A\./ =~ target.to_s # skip rules such as ".c.o"
#p [curdir, target, deps]
dir = curdir || dirstack.last
dependencies[dir + target] ||= []
dependencies[dir + target] |= deps.map {|dep| dir + dep }
elsif data_base_end
curdir = nil
elsif directory_leave
leave_dir = Pathname(directory_leave)
#p [:leave, leave_dir]
if leave_dir != dirstack.last
warn "unexpected leave_dir : #{dirstack.last.inspect} != #{leave_dir.inspect}"
end
dirstack.pop
end
}
dependencies
end
#def guess_compiler_wd(filename, hint0)
# hint = hint0
# begin
# guess = hint + filename
# if guess.file?
# return hint
# end
# hint = hint.parent
# end while hint.to_s != '.'
# raise ArgumentError, "can not find #{filename} (hint: #{hint0})"
#end
def read_single_actual_deps(path_i, cwd)
files = {}
path_i.each_line.with_index {|line, lineindex|
next if /\A\# \d+ "(.*)"/ !~ line
files[$1] = lineindex
}
# gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line.
compiler_wd = files.keys.find {|f| %r{\A/.*//\z} =~ f }
if compiler_wd
files.delete compiler_wd
compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, ''))
else
raise "compiler working directory not found"
end
deps = []
files.each_key {|dep|
next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc.
dep = Pathname(dep)
if dep.relative?
dep = compiler_wd + dep
end
if !dep.file?
warn "file not found: #{dep}"
next
end
next if !dep.to_s.start_with?(cwd.to_s) # omit system headers.
deps << dep
}
deps
end
def read_actual_deps(cwd)
deps = {}
Pathname.glob('**/*.o').sort.each {|fn_o|
fn_i = fn_o.sub_ext('.i')
if !fn_i.exist?
warn "not found: #{fn_i}"
next
end
path_o = cwd + fn_o
path_i = cwd + fn_i
deps[path_o] = read_single_actual_deps(path_i, cwd)
}
deps
end
def concentrate(dependencies, cwd)
deps = {}
dependencies.keys.sort.each {|target|
sources = dependencies[target]
target = target.relative_path_from(cwd)
sources = sources.map {|s|
rel = s.relative_path_from(cwd)
rel
}
if %r{\A\.\.(/|\z)} =~ target.to_s
warn "out of tree target: #{target}"
next
end
sources = sources.reject {|s|
if %r{\A\.\.(/|\z)} =~ s.to_s
warn "out of tree source: #{s}"
true
else
false
end
}
deps[target] = sources
}
deps
end
def sort_paths(paths)
paths.sort_by {|t|
ary = t.to_s.split(%r{/})
ary.map.with_index {|e, i| i == ary.length-1 ? [0, e] : [1, e] } # regular file first, directories last.
}
end
def in_makefile(target, source)
target = target.to_s
source = source.to_s
case target
when %r{\A[^/]*\z}
target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}"
case source
when 'newline.c', 'miniprelude.c', 'prelude.c' then source2 = source
when 'thread_pthread.c' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).c'
when 'thread_pthread.h' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).h'
when 'include/ruby.h' then source2 = '$(hdrdir)/ruby.h'
when 'include/ruby/ruby.h' then source2 = '$(hdrdir)/ruby/ruby.h'
when 'revision.h' then source2 = '$(srcdir)/revision.h'
when 'version.h' then source2 = '$(srcdir)/version.h'
when 'include/ruby/version.h' then source2 = '$(srcdir)/include/ruby/version.h'
when %r{\A[^/]*\z} then source2 = "{$(VPATH)}#{File.basename source}"
when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "{$(VPATH)}#{$'}"
when %r{\Ainclude/ruby/} then source2 = "{$(VPATH)}#{$'}"
when %r{\Aenc/} then source2 = "{$(VPATH)}#{$'}"
when %r{\Amissing/} then source2 = "{$(VPATH)}#{$'}"
when %r{\Accan/} then source2 = "$(CCAN_DIR)/#{$'}"
when %r{\Adefs/} then source2 = "{$(VPATH)}#{source}"
else source2 = "$(top_srcdir)/#{source}"
end
["common.mk", target2, source2]
when %r{\Aenc/}
target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}"
case source
when 'include/ruby.h' then source2 = '$(hdrdir)/ruby.h'
when 'include/ruby/ruby.h' then source2 = '$(hdrdir)/ruby/ruby.h'
when %r{\A\.ext/include/[^/]+/ruby/} then source2 = $'
when %r{\Ainclude/ruby/} then source2 = $'
when %r{\Aenc/} then source2 = source
else source2 = "$(top_srcdir)/#{source}"
end
["enc/depend", target2, source2]
when %r{\Aext/}
unless File.exist?("#{File.dirname(target)}/extconf.rb")
warn "not found: #{File.dirname(target)}/extconf.rb"
end
target2 = File.basename(target)
case source
when 'include/ruby.h' then source2 = '$(top_srcdir)/include/ruby.h'
when %r{\Ainclude/} then source2 = "$(hdrdir)/#{$'}"
when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "$(arch_hdrdir)/ruby/#{$'}"
when %r{\A#{Regexp.escape File.dirname(target)}/extconf\.h\z} then source2 = "$(RUBY_EXTCONF_H)"
when %r{\A#{Regexp.escape File.dirname(target)}/} then source2 = $'
when 'id.h' then source2 = '{$(VPATH)}id.h'
when 'parse.h' then source2 = '{$(VPATH)}parse.h'
when 'lex.c' then source2 = '{$(VPATH)}lex.c'
when 'probes.h' then source2 = '{$(VPATH)}probes.h'
else source2 = "$(top_srcdir)/#{source}"
end
["#{File.dirname(target)}/depend", target2, source2]
else
raise "unexpected target: #{target}"
end
end
def compare_deps(make_deps, actual_deps, out=$stdout)
targets = sort_paths(actual_deps.keys)
targets.each {|target|
actual_sources = actual_deps[target]
if !make_deps.has_key?(target)
warn "no makefile dependency for #{target}"
else
make_sources = make_deps[target]
sort_paths(actual_sources | make_sources).each {|source|
makefile, target2, source2 = in_makefile(target, source)
lines = begin
File.readlines(makefile)
rescue Errno::ENOENT
[]
end
#depline = "#{target2}: #{source2} \# #{target}: #{source}\n"
depline = "#{target2}: #{source2}\n"
if !make_sources.include?(source)
out.puts "add #{makefile} : #{depline}"
elsif !actual_sources.include?(source)
if lines.include? depline
out.puts "delL #{makefile} : #{depline}" # delL stands for del line
else
out.puts "delP #{makefile} : #{depline}" # delP stands for del prerequisite
end
else
if $opt_a
if lines.include? depline
out.puts "okL #{makefile} : #{depline}" # okL stands for ok line
else
out.puts "okP #{makefile} : #{depline}" # okP stands for ok prerequisite
end
end
end
}
end
}
end
def main_show(out=$stdout)
cwd = Pathname.pwd
make_deps = read_make_deps(cwd)
#pp make_deps
make_deps = concentrate(make_deps, cwd)
#pp make_deps
actual_deps = read_actual_deps(cwd)
#pp actual_deps
actual_deps = concentrate(actual_deps, cwd)
#pp actual_deps
compare_deps(make_deps, actual_deps, out)
end
def extract_deplines(problems)
adds = {}
dels = {}
problems.each_line {|line|
case line
when /\Aadd (\S+) : (\S.*\n)\z/
(adds[$1] ||= []) << $2
when /\AdelL (\S+) : (\S.*\n)\z/
(dels[$1] ||= []) << $2
when /\AdelP (\S+) : (\S.*\n)\z/
(dels[$1] ||= []) << $2
when /\AokL (\S+) : (\S.*\n)\z/
when /\AokP (\S+) : (\S.*\n)\z/
(adds[$1] ||= []) << $2
end
}
return adds, dels
end
DEPENDENCIES_SECTION_START_MARK = "\# AUTOGENERATED DEPENDENCIES START\n"
DEPENDENCIES_SECTION_END_MARK = "\# AUTOGENERATED DEPENDENCIES END\n"
def main_actual_fix(problems)
adds, dels = extract_deplines(problems)
(adds.keys | dels.keys).sort.each {|makefile|
content = begin
File.read(makefile)
rescue Errno::ENOENT
''
end
if /^#{Regexp.escape DEPENDENCIES_SECTION_START_MARK}((?:.*\n)*)#{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/ =~ content
pre_post_part = [$`, $']
lines = $1.lines.to_a
else
pre_post_part = nil
lines = []
end
lines_original = lines.dup
if dels[makefile]
lines -= dels[makefile]
end
if adds[makefile]
lines.concat(adds[makefile] - lines)
end
if lines == lines_original
next
end
if pre_post_part
new_content = [
pre_post_part.first,
DEPENDENCIES_SECTION_START_MARK,
*lines,
DEPENDENCIES_SECTION_END_MARK,
pre_post_part.last
].join
tmp_makefile = "#{makefile}.new#{$$}"
File.write(tmp_makefile, new_content)
File.rename tmp_makefile, makefile
puts "modified: #{makefile}"
else
new_content = [
DEPENDENCIES_SECTION_START_MARK,
*lines,
DEPENDENCIES_SECTION_END_MARK,
].join
if !File.exist?(makefile)
if !lines.empty?
File.open(makefile, 'w') {|f|
f.print new_content
}
puts "created: #{makefile}"
end
else
puts "no dependencies section: #{makefile}"
(lines_original - lines).each {|line|
puts " del: #{line}"
}
(lines - lines_original).each {|line|
puts " add: #{line}"
}
end
end
}
end
def main_fix
problems = StringIO.new
main_show(problems)
main_actual_fix(problems.string)
end
def run
op = optionparser
op.parse!(ARGV)
if $opt_actual_fix
main_actual_fix(ARGF.read)
elsif $opt_fix
main_fix
else
main_show
end
end
run
|