summaryrefslogtreecommitdiff
path: root/tool/update-deps
blob: 8e27f8a510f731b2c88873c77de96b8575a7bba6 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
#!/usr/bin/ruby

# tool/update-deps verify makefile dependencies.

# Requirements:
#   gcc 4.5 (for -save-temps=obj option)
#   GNU make (for -p option)
#
# Usage:
#   1. Compile ruby with -save-temps=obj option.
#      Ex.  ./configure debugflags='-save-temps=obj -g' && make all golf
#   2. Run tool/update-deps to show dependency problems.
#      Ex.  ruby tool/update-deps
#   3. Use --fix to fix makefiles.
#      Ex.  ruby tool/update-deps --fix
#
# Other usages:
#   *  Fix makefiles using previously detected dependency problems
#      Ex.  ruby tool/update-deps --actual-fix [file]
#      "ruby tool/update-deps --fix" is same as "ruby tool/update-deps | ruby tool/update-deps --actual-fix".

require 'optparse'
require 'stringio'
require 'pathname'
require 'pp'

ENV['LC_ALL'] = 'C'

$opt_fix = false
$opt_a = false
$opt_actual_fix = false

def optionparser
  op = OptionParser.new
  op.banner = 'Usage: ruby tool/update-deps'
  op.def_option('-a', 'show valid dependencies') { $opt_a = true }
  op.def_option('--fix') { $opt_fix = true }
  op.def_option('--actual-fix') { $opt_actual_fix = true }
  op
end

def read_make_deps(cwd)
  dependencies = {}
  make_p = `make -p all miniruby ruby golf 2> /dev/null`
  dirstack = [cwd]
  curdir = nil
  make_p.scan(%r{Entering\ directory\ ['`](.*)'|
                 ^\#\ (GNU\ Make)\ |
                 ^CURDIR\ :=\ (.*)|
                 ^([/0-9a-zA-Z._-]+):(.*)\n((?:\#.*\n)*)|
                 ^\#\ (Finished\ Make\ data\ base\ on)\ |
                 Leaving\ directory\ ['`](.*)'}x) {
    directory_enter = $1
    data_base_start = $2
    data_base_curdir = $3
    rule_target = $4
    rule_sources = $5
    rule_desc = $6
    data_base_end = $7
    directory_leave = $8
    #p $~
    if directory_enter
      enter_dir = Pathname(directory_enter)
      #p [:enter, enter_dir]
      dirstack.push enter_dir
    elsif data_base_start
      curdir = nil
    elsif data_base_curdir
      curdir = Pathname(data_base_curdir)
    elsif rule_target && rule_sources && rule_desc &&
          /Modification time never checked/ !~ rule_desc # This pattern match eliminates rules which VPATH is not expanded.
      target = rule_target
      deps = rule_sources
      deps = deps.scan(%r{[/0-9a-zA-Z._-]+})
      next if /\.o\z/ !~ target.to_s
      next if /\A\./ =~ target.to_s # skip rules such as ".c.o"
      #p [curdir, target, deps]
      dir = curdir || dirstack.last
      dependencies[dir + target] ||= []
      dependencies[dir + target] |= deps.map {|dep| dir + dep }
    elsif data_base_end
      curdir = nil
    elsif directory_leave
      leave_dir = Pathname(directory_leave)
      #p [:leave, leave_dir]
      if leave_dir != dirstack.last
        warn "unexpected leave_dir : #{dirstack.last.inspect} != #{leave_dir.inspect}"
      end
      dirstack.pop
    end
  }
  dependencies
end

#def guess_compiler_wd(filename, hint0)
#  hint = hint0
#  begin
#    guess = hint + filename
#    if guess.file?
#      return hint
#    end
#    hint = hint.parent
#  end while hint.to_s != '.'
#  raise ArgumentError, "can not find #{filename} (hint: #{hint0})"
#end

def read_single_actual_deps(path_i, cwd)
  files = {}
  path_i.each_line.with_index {|line, lineindex|
    next if /\A\# \d+ "(.*)"/ !~ line
    files[$1] = lineindex
  }
  # gcc emits {# 1 "/absolute/directory/of/the/source/file//"} at 2nd line.
  compiler_wd = files.keys.find {|f| %r{\A/.*//\z} =~ f }
  if compiler_wd
    files.delete compiler_wd
    compiler_wd = Pathname(compiler_wd.sub(%r{//\z}, ''))
  else
    raise "compiler working directory not found"
  end
  deps = []
  files.each_key {|dep|
    next if %r{\A<.*>\z} =~ dep # omit <command-line>, etc.
    dep = Pathname(dep)
    if dep.relative?
      dep = compiler_wd + dep
    end
    if !dep.file?
      warn "file not found: #{dep}"
      next
    end
    next if !dep.to_s.start_with?(cwd.to_s) # omit system headers.
    deps << dep
  }
  deps
end

def read_actual_deps(cwd)
  deps = {}
  Pathname.glob('**/*.o').sort.each {|fn_o|
    fn_i = fn_o.sub_ext('.i')
    if !fn_i.exist?
      warn "not found: #{fn_i}"
      next
    end
    path_o = cwd + fn_o
    path_i = cwd + fn_i
    deps[path_o] = read_single_actual_deps(path_i, cwd)
  }
  deps
end

def concentrate(dependencies, cwd)
  deps = {}
  dependencies.keys.sort.each {|target|
    sources = dependencies[target]
    target = target.relative_path_from(cwd)
    sources = sources.map {|s|
      rel = s.relative_path_from(cwd)
      rel
    }
    if %r{\A\.\.(/|\z)} =~ target.to_s
      warn "out of tree target: #{target}"
      next
    end
    sources = sources.reject {|s|
      if %r{\A\.\.(/|\z)} =~ s.to_s
        warn "out of tree source: #{s}"
        true
      else
        false
      end
    }
    deps[target] = sources
  }
  deps
end

def sort_paths(paths)
  paths.sort_by {|t|
    ary = t.to_s.split(%r{/})
    ary.map.with_index {|e, i| i == ary.length-1 ? [0, e] : [1, e] } # regular file first, directories last.
  }
end

def in_makefile(target, source)
  target = target.to_s
  source = source.to_s
  case target
  when %r{\A[^/]*\z}
    target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}"
    case source
    when 'newline.c', 'miniprelude.c', 'prelude.c' then source2 = source
    when 'thread_pthread.c' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).c'
    when 'thread_pthread.h' then source2 = '{$(VPATH)}thread_$(THREAD_MODEL).h'
    when 'include/ruby.h' then source2 = '$(hdrdir)/ruby.h'
    when 'include/ruby/ruby.h' then source2 = '$(hdrdir)/ruby/ruby.h'
    when 'revision.h' then source2 = '$(srcdir)/revision.h'
    when 'version.h' then source2 = '$(srcdir)/version.h'
    when 'include/ruby/version.h' then source2 = '$(srcdir)/include/ruby/version.h'
    when %r{\A[^/]*\z} then source2 = "{$(VPATH)}#{File.basename source}"
    when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "{$(VPATH)}#{$'}"
    when %r{\Ainclude/ruby/} then source2 = "{$(VPATH)}#{$'}"
    when %r{\Aenc/} then source2 = "{$(VPATH)}#{$'}"
    when %r{\Amissing/} then source2 = "{$(VPATH)}#{$'}"
    when %r{\Accan/} then source2 = "$(CCAN_DIR)/#{$'}"
    when %r{\Adefs/} then source2 = "{$(VPATH)}#{source}"
    else source2 = "$(top_srcdir)/#{source}"
    end
    ["common.mk", target2, source2]
  when %r{\Aenc/}
    target2 = "#{target.sub(/\.o\z/, '.$(OBJEXT)')}"
    case source
    when 'include/ruby.h' then source2 = '$(hdrdir)/ruby.h'
    when 'include/ruby/ruby.h' then source2 = '$(hdrdir)/ruby/ruby.h'
    when %r{\A\.ext/include/[^/]+/ruby/} then source2 = $'
    when %r{\Ainclude/ruby/} then source2 = $'
    when %r{\Aenc/} then source2 = source
    else source2 = "$(top_srcdir)/#{source}"
    end
    ["enc/depend", target2, source2]
  when %r{\Aext/}
    unless File.exist?("#{File.dirname(target)}/extconf.rb")
      warn "not found: #{File.dirname(target)}/extconf.rb"
    end
    target2 = File.basename(target)
    case source
    when 'include/ruby.h' then source2 = '$(top_srcdir)/include/ruby.h'
    when %r{\Ainclude/} then source2 = "$(hdrdir)/#{$'}"
    when %r{\A\.ext/include/[^/]+/ruby/} then source2 = "$(arch_hdrdir)/ruby/#{$'}"
    when %r{\A#{Regexp.escape File.dirname(target)}/extconf\.h\z} then source2 = "$(RUBY_EXTCONF_H)"
    when %r{\A#{Regexp.escape File.dirname(target)}/} then source2 = $'
    when 'id.h' then source2 = '{$(VPATH)}id.h'
    when 'parse.h' then source2 = '{$(VPATH)}parse.h'
    when 'lex.c' then source2 = '{$(VPATH)}lex.c'
    when 'probes.h' then source2 = '{$(VPATH)}probes.h'
    else source2 = "$(top_srcdir)/#{source}"
    end
    ["#{File.dirname(target)}/depend", target2, source2]
  else
    raise "unexpected target: #{target}"
  end
end

def compare_deps(make_deps, actual_deps, out=$stdout)
  targets = sort_paths(actual_deps.keys)
  targets.each {|target|
    actual_sources = actual_deps[target]
    if !make_deps.has_key?(target)
      warn "no makefile dependency for #{target}"
    else
      make_sources = make_deps[target]
      sort_paths(actual_sources | make_sources).each {|source|
        makefile, target2, source2 = in_makefile(target, source)
        lines = begin
          File.readlines(makefile)
        rescue Errno::ENOENT
          []
        end
        #depline = "#{target2}: #{source2} \# #{target}: #{source}\n"
        depline = "#{target2}: #{source2}\n"
        if !make_sources.include?(source)
          out.puts "add #{makefile} : #{depline}"
        elsif !actual_sources.include?(source)
          if lines.include? depline
            out.puts "delL #{makefile} : #{depline}" # delL stands for del line
          else
            out.puts "delP #{makefile} : #{depline}" # delP stands for del prerequisite
          end
        else
          if $opt_a
            if lines.include? depline
              out.puts "okL #{makefile} : #{depline}" # okL stands for ok line
            else
              out.puts "okP #{makefile} : #{depline}" # okP stands for ok prerequisite
            end
          end
        end
      }
    end
  }
end

def main_show(out=$stdout)
  cwd = Pathname.pwd
  make_deps = read_make_deps(cwd)
  #pp make_deps
  make_deps = concentrate(make_deps, cwd)
  #pp make_deps
  actual_deps = read_actual_deps(cwd)
  #pp actual_deps
  actual_deps = concentrate(actual_deps, cwd)
  #pp actual_deps
  compare_deps(make_deps, actual_deps, out)
end

def extract_deplines(problems)
  adds = {}
  dels = {}
  problems.each_line {|line|
    case line
    when /\Aadd (\S+) : (\S.*\n)\z/
      (adds[$1] ||= []) << $2
    when /\AdelL (\S+) : (\S.*\n)\z/
      (dels[$1] ||= []) << $2
    when /\AdelP (\S+) : (\S.*\n)\z/
      (dels[$1] ||= []) << $2
    when /\AokL (\S+) : (\S.*\n)\z/
    when /\AokP (\S+) : (\S.*\n)\z/
      (adds[$1] ||= []) << $2
    end
  }
  return adds, dels
end

DEPENDENCIES_SECTION_START_MARK = "\# AUTOGENERATED DEPENDENCIES START\n"
DEPENDENCIES_SECTION_END_MARK = "\# AUTOGENERATED DEPENDENCIES END\n"

def main_actual_fix(problems)
  adds, dels = extract_deplines(problems)
  (adds.keys | dels.keys).sort.each {|makefile|
    content = begin
      File.read(makefile)
    rescue Errno::ENOENT
      ''
    end
    if /^#{Regexp.escape DEPENDENCIES_SECTION_START_MARK}((?:.*\n)*)#{Regexp.escape DEPENDENCIES_SECTION_END_MARK}/ =~ content
      pre_post_part = [$`, $']
      lines = $1.lines.to_a
    else
      pre_post_part = nil
      lines = []
    end

    lines_original = lines.dup

    if dels[makefile]
      lines -= dels[makefile]
    end
    if adds[makefile]
      lines.concat(adds[makefile] - lines)
    end

    if lines == lines_original
      next
    end

    if pre_post_part
      new_content = [
        pre_post_part.first,
        DEPENDENCIES_SECTION_START_MARK,
        *lines,
        DEPENDENCIES_SECTION_END_MARK,
        pre_post_part.last
      ].join
      tmp_makefile = "#{makefile}.new#{$$}"
      File.write(tmp_makefile, new_content)
      File.rename tmp_makefile, makefile
      puts "modified: #{makefile}"
    else
      new_content = [
        DEPENDENCIES_SECTION_START_MARK,
        *lines,
        DEPENDENCIES_SECTION_END_MARK,
      ].join
      if !File.exist?(makefile)
        if !lines.empty?
          File.open(makefile, 'w') {|f|
            f.print new_content
          }
          puts "created: #{makefile}"
        end
      else
        puts "no dependencies section: #{makefile}"
        (lines_original - lines).each {|line|
          puts "  del: #{line}"
        }
        (lines - lines_original).each {|line|
          puts "  add: #{line}"
        }
      end
    end
  }
end

def main_fix
  problems = StringIO.new
  main_show(problems)
  main_actual_fix(problems.string)
end

def run
  op = optionparser
  op.parse!(ARGV)
  if $opt_actual_fix
    main_actual_fix(ARGF.read)
  elsif $opt_fix
    main_fix
  else
    main_show
  end
end

run