diff options
Diffstat (limited to 'benchmark/driver.rb')
| -rwxr-xr-x | benchmark/driver.rb | 397 |
1 files changed, 48 insertions, 349 deletions
diff --git a/benchmark/driver.rb b/benchmark/driver.rb index c166c80edb..3536d9abd4 100755 --- a/benchmark/driver.rb +++ b/benchmark/driver.rb @@ -10,256 +10,66 @@ rescue LoadError require 'optparse' end -require 'benchmark' -require 'pp' +require 'shellwords' require 'tempfile' class BenchmarkDriver - def self.benchmark(opt) - driver = self.new(opt[:execs], opt[:dir], opt) - begin - driver.run - ensure - driver.show_results + # Run benchmark-driver prepared by `make update-benchmark-driver` + def self.run(*args) + benchmark_driver = File.expand_path('./benchmark-driver/exe/benchmark-driver', __dir__) + command = [benchmark_driver, *args] + unless system(command.shelljoin) + abort "Failed to execute: #{command.shelljoin}" end end - def self.load(input, type, opt) - case type - when 'yaml' - require 'yaml' - h = YAML.load(input) - when 'json' - require 'json' - h = JSON.load(input) - else - h = eval(input.read) - end - results = h[:results] || h["results"] - obj = allocate - obj.instance_variable_set("@execs", h[:executables] || h["executables"]) - obj.instance_variable_set("@results", results) - obj.instance_variable_set("@opt", opt) - [1, 2].each do |i| - loop = results.assoc((n = "loop_whileloop#{i}").intern) || results.assoc(n) - obj.instance_variable_set("@loop_wl#{i}", loop ? loop[1].map {|t,*|t} : nil) - end - obj.instance_variable_set("@measure_target", opt[:measure_target] || opt["measure_target"]) - obj - end - - def output *args - puts(*args) - @output and @output.puts(*args) - end - - def message *args - output(*args) if @verbose - end - - def message_print *args - if @verbose - print(*args) - STDOUT.flush - @output and @output.print(*args) - end - end - - def progress_message *args - unless STDOUT.tty? - STDERR.print(*args) - STDERR.flush - end - end - - def initialize execs, dir, opt = {} - @execs = execs.map{|e| - e.strip! - next if e.empty? - - if /(.+)::(.+)/ =~ e - # ex) ruby-a::/path/to/ruby-a - label = $1.strip - path = $2 - version = `#{path} --version`.chomp - else - path = e - version = label = `#{path} --version`.chomp - end - [path, label, version] - }.compact - + def initialize(dir, opt = {}) @dir = dir - @repeat = opt[:repeat] || 1 - @repeat = 1 if @repeat < 1 @pattern = opt[:pattern] || nil @exclude = opt[:exclude] || nil - @verbose = opt[:quiet] ? false : (opt[:verbose] || false) - @output = opt[:output] ? open(opt[:output], 'w') : nil - @loop_wl1 = @loop_wl2 = nil - @ruby_arg = opt[:ruby_arg] || nil - @measure_target = opt[:measure_target] - @opt = opt - - # [[name, [[r-1-1, r-1-2, ...], [r-2-1, r-2-2, ...]]], ...] - @results = [] - - if @verbose - @start_time = Time.now - message @start_time - @execs.each_with_index{|(path, label, version), i| - message "target #{i}: " + (label == version ? "#{label}" : "#{label} (#{version})") + " at \"#{path}\"" - } - message "measure target: #{@measure_target}" - end end - def adjusted_results name, results - s = nil - results.each_with_index{|e, i| - r = e.min - case name - when /^vm1_/ - if @loop_wl1 - r -= @loop_wl1[i] - r = 0 if r < 0 - s = '*' + def with_yamls(&block) + ios = files.map do |file| + Tempfile.open.tap do |io| + if file.end_with?('.yml') + io.write(File.read(file)) + else + io.write(build_yaml(file)) end - when /^vm2_/ - if @loop_wl2 - r -= @loop_wl2[i] - r = 0 if r < 0 - s = '*' - end - end - yield r - } - s - end - - def show_results - case @opt[:format] - when :tsv - strformat = "\t%1$s" - numformat = "\t%1$*2$.3f" - minwidth = 0 - name_width = 0 - when :markdown - markdown = true - strformat = "|%1$-*2$s" - numformat = "|%1$*2$.3f" - when :plain - strformat = " %1$-*2$s" - numformat = " %1$*2$.3f" - end - - name_width ||= @results.map {|v, result| - v.size + (case v; when /^vm1_/; @loop_wl1; when /^vm2_/; @loop_wl2; end ? 1 : 0) - }.max - minwidth ||= 7 - width = @execs.map{|(_, v)| [v.size, minwidth].max} - - output - - if @verbose - message '-----------------------------------------------------------' - message 'raw data:' - message - message PP.pp(@results, "", 79) - message - message "Elapsed time: #{Time.now - @start_time} (sec)" - end - - if rawdata_output = @opt[:rawdata_output] - h = {} - h[:cpuinfo] = File.read('/proc/cpuinfo') if File.exist?('/proc/cpuinfo') - h[:executables] = @execs - h[:results] = @results - if (type = File.extname(rawdata_output)).empty? - type = rawdata_output - rawdata_output = @output.path.sub(/\.[^.\/]+\z/, '') << '.' << rawdata_output + io.close end - case type - when 'yaml' - require 'yaml' - h = YAML.dump(h) - when 'json' - require 'json' - h = JSON.pretty_generate(h) - else - require 'pp' - h = h.pretty_inspect - end - open(rawdata_output, 'w') {|f| f.puts h} - end - - output '-----------------------------------------------------------' - output 'benchmark results:' - - if @verbose and @repeat > 1 - output "minimum results in each #{@repeat} measurements." end + block.call(ios.map(&:path)) + ensure + ios.each(&:close) + end - output({ - real: "Execution time (sec)", - utime: "user CPU time", - stime: "system CPU time", - cutime: "user CPU time of children", - cstime: "system CPU time of children", - total: "all CPU time", - peak: "Memory usage (peak) (B)", - size: "Memory usage (last size) (B)", - }[@measure_target]) - output if markdown - output ["name".ljust(name_width), @execs.map.with_index{|(_, v), i| sprintf(strformat, v, width[i])}].join("").rstrip - output ["-"*name_width, width.map{|n|":".rjust(n, "-")}].join("|") if markdown - @results.each{|v, result| - rets = [] - s = adjusted_results(v, result){|r| - rets << sprintf(numformat, r, width[rets.size]) - } - v += s if s - output [v.ljust(name_width), rets].join("") - } + private - if @execs.size > 1 - output - output({ - real: "Speedup ratio: compare with the result of `#{@execs[0][1]}' (greater is better)", - peak: "Memory consuming ratio (peak) with the result of `#{@execs[0][1]}' (greater is better)", - size: "Memory consuming ratio (size) with the result of `#{@execs[0][1]}' (greater is better)", - }[@measure_target]) - output if markdown - output ["name".ljust(name_width), @execs[1..-1].map.with_index{|(_, v), i| sprintf(strformat, v, width[i])}].join("").rstrip - output ["-"*name_width, width[1..-1].map{|n|":".rjust(n, "-")}].join("|") if markdown - @results.each{|v, result| - rets = [] - first_value = nil - s = adjusted_results(v, result){|r| - if first_value - if r == 0 - rets << "Error" - else - rets << sprintf(numformat, first_value/Float(r), width[rets.size+1]) - end - else - first_value = r - end - } - v += s if s - output [v.ljust(name_width), rets].join("") - } + def build_yaml(file) + magic_comment = '# prelude' # bm_so_nsieve_bits hangs without magic comment + name = File.basename(file).sub(/\Abm_/, '').sub(/\.rb\z/, '') + script = File.read(file).sub(/^__END__\n(.+\n)*/m, '').sub(/\A(^#.+\n)+/m) do |comment| + magic_comment = comment + '' end - if @opt[:output] - output - output "Log file: #{@opt[:output]}" - end + <<-YAML +prelude: | +#{magic_comment.gsub(/^/, ' ')} +benchmark: + #{name}: | +#{script.gsub(/^/, ' ')} +loop_count: 1 + YAML end def files flag = {} - @files = Dir.glob(File.join(@dir, 'bm*.rb')).map{|file| + legacy_files = Dir.glob(File.join(@dir, 'bm*.rb')) + yaml_files = Dir.glob(File.join(@dir, '*.yml')) + files = (legacy_files + yaml_files).map{|file| next if @pattern && /#{@pattern}/ !~ File.basename(file) next if @exclude && /#{@exclude}/ =~ File.basename(file) case file @@ -270,90 +80,13 @@ class BenchmarkDriver }.compact if flag['vm1'] && !flag['whileloop'] - @files << File.join(@dir, 'bm_loop_whileloop.rb') + files << File.join(@dir, 'bm_loop_whileloop.rb') elsif flag['vm2'] && !flag['whileloop2'] - @files << File.join(@dir, 'bm_loop_whileloop2.rb') - end - - @files.sort! - progress_message "total: #{@files.size * @repeat} trial(s) (#{@repeat} trial(s) for #{@files.size} benchmark(s))\n" - @files - end - - def run - files.each_with_index{|file, i| - @i = i - r = measure_file(file) - - if /bm_loop_whileloop.rb/ =~ file - @loop_wl1 = r[1].map{|e| e.min} - elsif /bm_loop_whileloop2.rb/ =~ file - @loop_wl2 = r[1].map{|e| e.min} - end - } - end - - def measure_file file - name = File.basename(file, '.rb').sub(/^bm_/, '') - prepare_file = File.join(File.dirname(file), "prepare_#{name}.rb") - load prepare_file if FileTest.exist?(prepare_file) - - if @verbose - output - output '-----------------------------------------------------------' - output name - output - output File.read(file) - output - end - - result = [name] - result << @execs.map{|(e, v)| - (0...@repeat).map{ - message_print "#{v}\t" - progress_message '.' - - m = measure(e, file) - message "#{m}" - m - } - } - @results << result - result - end - - unless defined?(File::NULL) - if File.exist?('/dev/null') - File::NULL = '/dev/null' + files << File.join(@dir, 'bm_loop_whileloop2.rb') end - end - def measure executable, file - case @measure_target - when :real, :utime, :stime, :cutime, :cstime, :total - cmd = "#{executable} #{@ruby_arg} #{file}" - m = Benchmark.measure{ - system(cmd, out: File::NULL) - } - result = m.__send__(@measure_target) - when :peak, :size - tmp = Tempfile.new("benchmark-memory-wrapper-data") - wrapper = "#{File.join(__dir__, 'memory_wrapper.rb')} #{tmp.path} #{@measure_target}" - cmd = "#{executable} #{@ruby_arg} #{wrapper} #{file}" - system(cmd, out: File::NULL) - result = tmp.read.to_i - tmp.close - else - raise "unknown measure target" - end - - if $? != 0 - raise $?.inspect if $? && $?.signaled? - output "\`#{cmd}\' exited with abnormal status (#{$?})" - 0 - else - result - end + files.sort! + files end end @@ -362,15 +95,7 @@ if __FILE__ == $0 :execs => [], :dir => File.dirname(__FILE__), :repeat => 1, - :measure_target => :real, - :output => nil, - :raw_output => nil, - :format => :tsv, - } - formats = { - :tsv => ".tsv", - :markdown => ".md", - :plain => ".txt", + :verbose => 1, } parser = OptionParser.new{|o| @@ -397,44 +122,18 @@ if __FILE__ == $0 o.on('-r', '--repeat-count [NUM]', "Repeat count"){|n| opt[:repeat] = n.to_i } - o.on('-o', '--output-file [FILE]', "Output file"){|f| - opt[:output] = f - } - o.on('--ruby-arg [ARG]', "Optional argument for ruby"){|a| - opt[:ruby_arg] = a - } - o.on('--measure-target [TARGET]', - 'real (execution time), peak, size (memory), total'){|mt| - opt[:measure_target] = mt.to_sym - } - o.on('--rawdata-output [FILE]', 'output rawdata'){|r| - opt[:rawdata_output] = r - } - o.on('--load-rawdata=FILE', 'input rawdata'){|r| - opt[:rawdata_input] = r - } - o.on('-f', "--format=FORMAT", "output format (#{formats.keys.join(",")})", formats.keys){|r| - opt[:format] = r - } o.on('-v', '--verbose'){|v| - opt[:verbose] = v + opt[:verbose] = 2 } o.on('-q', '--quiet', "Run without notify information except result table."){|q| - opt[:quiet] = q - opt[:verbose] = false + opt[:verbose] = 0 } } parser.parse!(ARGV) - if input = opt[:rawdata_input] - b = open(input) {|f| - BenchmarkDriver.load(f, File.extname(input)[1..-1], opt) - } - b.show_results - else - opt[:output] ||= "bmlog-#{Time.now.strftime('%Y%m%d-%H%M%S')}.#{$$}#{formats[opt[:format]]}" - BenchmarkDriver.benchmark(opt) + execs = opt[:execs].map { |exec| ['--executables', exec.shellsplit.join(',')] }.flatten + BenchmarkDriver.new(opt[:dir], opt).with_yamls do |yamls| + BenchmarkDriver.run(*yamls, *execs, "--verbose=#{opt[:verbose]}", "--repeat-count=#{opt[:repeat]}") end end - |