summaryrefslogtreecommitdiff
path: root/benchmark
diff options
context:
space:
mode:
authork0kubun <k0kubun@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-07-10 12:14:04 (GMT)
committerk0kubun <k0kubun@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-07-10 12:14:04 (GMT)
commit12a38cbac0bc7ff7be4b05e2e7d4f0ca76df7bf4 (patch)
treee4472b4f4ff6ed5e32855616b5721574f27636fb /benchmark
parentd8583b132c4ce3c203c9f5a117fd581c8c7eb6fe (diff)
benchmark: resurrect peak / size metrics
by adding runner plugins for them. benchmark/lib/benchmark_driver/runner/peak.rb: added peak runner plugin benchmark/lib/benchmark_driver/runner/size.rb: added size runner plugin common.mk: allow using them benchmark/memory_wrapper.rb: deleted in favor of those runner plugins benchmark/README.md: document them git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@63924 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'benchmark')
-rw-r--r--benchmark/README.md4
-rw-r--r--benchmark/lib/benchmark_driver/runner/peak.rb146
-rw-r--r--benchmark/lib/benchmark_driver/runner/size.rb20
-rw-r--r--benchmark/memory_wrapper.rb16
4 files changed, 170 insertions, 16 deletions
diff --git a/benchmark/README.md b/benchmark/README.md
index 24f6637..013de79 100644
--- a/benchmark/README.md
+++ b/benchmark/README.md
@@ -43,4 +43,8 @@ make benchmark ARGS=../benchmark/erb_render.yml
# You can specify any option via $OPTS
make benchmark OPTS="--help"
+
+# With `make benchmark`, some special runner plugins are available:
+# -r peak, -r size
+make benchmark ITEM=vm2_bigarray OPTS="-r peak"
```
diff --git a/benchmark/lib/benchmark_driver/runner/peak.rb b/benchmark/lib/benchmark_driver/runner/peak.rb
new file mode 100644
index 0000000..e00d667
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/peak.rb
@@ -0,0 +1,146 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'benchmark_driver/default_job'
+require 'benchmark_driver/default_job_parser'
+require 'tempfile'
+
+class BenchmarkDriver::Runner::Peak
+ METRIC = BenchmarkDriver::Metric.new(
+ name: 'Peak memory usage', unit: 'bytes', larger_better: false, worse_word: 'larger',
+ )
+
+ # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+ Job = Class.new(BenchmarkDriver::DefaultJob)
+ # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+ JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+ # @param [BenchmarkDriver::Config::RunnerConfig] config
+ # @param [BenchmarkDriver::Output] output
+ # @param [BenchmarkDriver::Context] contexts
+ def initialize(config:, output:, contexts:)
+ @config = config
+ @output = output
+ @contexts = contexts
+ end
+
+ # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+ # @param [Array<BenchmarkDriver::Runner::Peak::Job>] jobs
+ def run(jobs)
+ if jobs.any? { |job| job.loop_count.nil? }
+ jobs = jobs.map do |job|
+ job.loop_count ? job : Job.new(job.to_h.merge(loop_count: 1))
+ end
+ end
+
+ @output.with_benchmark do
+ jobs.each do |job|
+ @output.with_job(name: job.name) do
+ job.runnable_contexts(@contexts).each do |context|
+ value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
+ run_benchmark(job, context: context)
+ end
+ @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
+ @output.report(values: { METRIC => value }, loop_count: job.loop_count)
+ end
+ end
+ end
+ end
+ end
+ end
+
+ private
+
+ # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+ # @param [BenchmarkDriver::Context] context
+ # @return [BenchmarkDriver::Metrics]
+ def run_benchmark(job, context:)
+ benchmark = BenchmarkScript.new(
+ preludes: [context.prelude, job.prelude],
+ script: job.script,
+ teardown: job.teardown,
+ loop_count: job.loop_count,
+ )
+
+ memory_status = File.expand_path('../../../../test/lib/memory_status', __dir__)
+ Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+ with_script(benchmark.render) do |path|
+ output = IO.popen([*context.executable.command, path, f.path, target, memory_status], &:read)
+ if $?.success?
+ Integer(f.read)
+ else
+ $stdout.print(output)
+ BenchmarkDriver::Result::ERROR
+ end
+ end
+ end
+ end
+
+ # Overridden by BenchmarkDriver::Runner::Size
+ def target
+ 'peak'
+ end
+
+ def with_script(script)
+ if @config.verbose >= 2
+ sep = '-' * 30
+ $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
+ end
+
+ Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+ f.puts script
+ f.close
+ return yield(f.path)
+ end
+ end
+
+ # @param [String] prelude
+ # @param [String] script
+ # @param [String] teardown
+ # @param [Integer] loop_count
+ BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do
+ def render
+ prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n")
+ <<-RUBY
+#{prelude}
+#{while_loop(script, loop_count)}
+#{teardown}
+
+result_file, target, memory_status = ARGV
+require_relative memory_status
+
+ms = Memory::Status.new
+case target.to_sym
+when :peak
+ key = ms.respond_to?(:hwm) ? :hwm : :peak
+when :size
+ key = ms.respond_to?(:rss) ? :rss : :size
+else
+ raise('unexpected target: ' + target)
+end
+
+File.write(result_file, ms[key])
+ RUBY
+ end
+
+ private
+
+ def while_loop(content, times)
+ if !times.is_a?(Integer) || times <= 0
+ raise ArgumentError.new("Unexpected times: #{times.inspect}")
+ end
+
+ if times > 1
+ <<-RUBY
+__bmdv_i = 0
+while __bmdv_i < #{times}
+ #{content}
+ __bmdv_i += 1
+end
+ RUBY
+ else
+ content
+ end
+ end
+ end
+ private_constant :BenchmarkScript
+end
diff --git a/benchmark/lib/benchmark_driver/runner/size.rb b/benchmark/lib/benchmark_driver/runner/size.rb
new file mode 100644
index 0000000..e3ec820
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/size.rb
@@ -0,0 +1,20 @@
+require 'benchmark_driver/runner/peak'
+
+# Actually the same as BenchmarkDriver::Runner::Memory
+class BenchmarkDriver::Runner::Size < BenchmarkDriver::Runner::Peak
+ METRIC = BenchmarkDriver::Metric.new(
+ name: 'Max resident set size', unit: 'bytes', larger_better: false, worse_word: 'larger',
+ )
+
+ # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+ Job = Class.new(BenchmarkDriver::DefaultJob)
+ # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+ JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+ private
+
+ # Overriding BenchmarkDriver::Runner::Peak#target
+ def target
+ 'size'
+ end
+end
diff --git a/benchmark/memory_wrapper.rb b/benchmark/memory_wrapper.rb
deleted file mode 100644
index 3f4451a..0000000
--- a/benchmark/memory_wrapper.rb
+++ /dev/null
@@ -1,16 +0,0 @@
-
-write_file, target, script_file = ARGV
-
-load(script_file)
-require_relative '../test/lib/memory_status'
-open(write_file, 'wb'){|f|
- ms = Memory::Status.new
- case target.to_sym
- when :peak
- key = ms.respond_to?(:hwm) ? :hwm : :peak
- when :size
- key = ms.respond_to?(:rss) ? :rss : :size
- end
-
- f.puts ms[key]
-}