summaryrefslogtreecommitdiff
path: root/benchmark
diff options
context:
space:
mode:
authork0kubun <k0kubun@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-07-10 15:01:27 (GMT)
committerk0kubun <k0kubun@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-07-10 15:01:27 (GMT)
commitf328fe40b3bb22ef60713cfc707ba9716ade6cd6 (patch)
tree37184e9deb0a476ae9970c3bb4a53b7511e647a3 /benchmark
parent79a8c10b0b48eedefc7ee96fd3514adfb1d6c8e0 (diff)
benchmark_driver/runner: add runners for metrics
supported by legacy benchmark/driver.rb. benchmark/README.md: document them common.mk: update benchmark_driver to correct 0.0 output and to fix spacing format of `-o simple` and `-o markdown`. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@63933 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'benchmark')
-rw-r--r--benchmark/README.md5
-rw-r--r--benchmark/lib/benchmark_driver/runner/cstime.rb22
-rw-r--r--benchmark/lib/benchmark_driver/runner/cutime.rb22
-rw-r--r--benchmark/lib/benchmark_driver/runner/stime.rb22
-rw-r--r--benchmark/lib/benchmark_driver/runner/total.rb137
-rw-r--r--benchmark/lib/benchmark_driver/runner/utime.rb22
6 files changed, 229 insertions, 1 deletions
diff --git a/benchmark/README.md b/benchmark/README.md
index a70aedf..d2c411e 100644
--- a/benchmark/README.md
+++ b/benchmark/README.md
@@ -16,6 +16,9 @@ benchmark-driver benchmark/*.yml -e /path/to/ruby -e '/path/to/ruby --jit'
# Or compare Ruby versions managed by rbenv
benchmark-driver benchmark/*.yml --rbenv '2.5.1;2.6.0-preview2 --jit'
+
+# You can collect many metrics in many ways
+benchmark-driver benchmark/*.yml --runner memory --output markdown
```
See also:
@@ -62,6 +65,6 @@ make benchmark ARGS=../benchmark/erb_render.yml
make benchmark OPTS="--help"
# With `make benchmark`, some special runner plugins are available:
-# -r peak, -r size
+# -r peak, -r size, -r total, -r utime, -r stime, -r cutime, -r cstime
make benchmark ITEM=vm2_bigarray OPTS="-r peak"
```
diff --git a/benchmark/lib/benchmark_driver/runner/cstime.rb b/benchmark/lib/benchmark_driver/runner/cstime.rb
new file mode 100644
index 0000000..3c3453e
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/cstime.rb
@@ -0,0 +1,22 @@
+require 'benchmark_driver/runner/total'
+
+class BenchmarkDriver::Runner::Cstime < BenchmarkDriver::Runner::Total
+ METRIC = BenchmarkDriver::Metric.new(name: 'cstime', unit: 's', larger_better: false)
+
+ # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+ Job = Class.new(BenchmarkDriver::DefaultJob)
+ # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+ JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+ private
+
+ # Overriding BenchmarkDriver::Runner::Total#metric
+ def metric
+ METRIC
+ end
+
+ # Overriding BenchmarkDriver::Runner::Total#target
+ def target
+ :cstime
+ end
+end
diff --git a/benchmark/lib/benchmark_driver/runner/cutime.rb b/benchmark/lib/benchmark_driver/runner/cutime.rb
new file mode 100644
index 0000000..e139962
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/cutime.rb
@@ -0,0 +1,22 @@
+require 'benchmark_driver/runner/total'
+
+class BenchmarkDriver::Runner::Cutime < BenchmarkDriver::Runner::Total
+ METRIC = BenchmarkDriver::Metric.new(name: 'cutime', unit: 's', larger_better: false)
+
+ # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+ Job = Class.new(BenchmarkDriver::DefaultJob)
+ # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+ JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+ private
+
+ # Overriding BenchmarkDriver::Runner::Total#metric
+ def metric
+ METRIC
+ end
+
+ # Overriding BenchmarkDriver::Runner::Total#target
+ def target
+ :cutime
+ end
+end
diff --git a/benchmark/lib/benchmark_driver/runner/stime.rb b/benchmark/lib/benchmark_driver/runner/stime.rb
new file mode 100644
index 0000000..4577fb0
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/stime.rb
@@ -0,0 +1,22 @@
+require 'benchmark_driver/runner/total'
+
+class BenchmarkDriver::Runner::Stime < BenchmarkDriver::Runner::Total
+ METRIC = BenchmarkDriver::Metric.new(name: 'stime', unit: 's', larger_better: false)
+
+ # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+ Job = Class.new(BenchmarkDriver::DefaultJob)
+ # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+ JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+ private
+
+ # Overriding BenchmarkDriver::Runner::Total#metric
+ def metric
+ METRIC
+ end
+
+ # Overriding BenchmarkDriver::Runner::Total#target
+ def target
+ :stime
+ end
+end
diff --git a/benchmark/lib/benchmark_driver/runner/total.rb b/benchmark/lib/benchmark_driver/runner/total.rb
new file mode 100644
index 0000000..64dc14f
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/total.rb
@@ -0,0 +1,137 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'benchmark_driver/default_job'
+require 'benchmark_driver/default_job_parser'
+require 'tempfile'
+
+class BenchmarkDriver::Runner::Total
+ METRIC = BenchmarkDriver::Metric.new(name: 'Total time', unit: 's', larger_better: false)
+
+ # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+ Job = Class.new(BenchmarkDriver::DefaultJob)
+ # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+ JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+ # @param [BenchmarkDriver::Config::RunnerConfig] config
+ # @param [BenchmarkDriver::Output] output
+ # @param [BenchmarkDriver::Context] contexts
+ def initialize(config:, output:, contexts:)
+ @config = config
+ @output = output
+ @contexts = contexts
+ end
+
+ # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+ # @param [Array<BenchmarkDriver::Runner::Total::Job>] jobs
+ def run(jobs)
+ if jobs.any? { |job| job.loop_count.nil? }
+ raise 'missing loop_count is not supported in Ruby repository'
+ end
+
+ @output.with_benchmark do
+ jobs.each do |job|
+ @output.with_job(name: job.name) do
+ job.runnable_contexts(@contexts).each do |context|
+ duration = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
+ run_benchmark(job, context: context)
+ end
+ @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
+ @output.report(values: { metric => duration }, duration: duration, loop_count: job.loop_count)
+ end
+ end
+ end
+ end
+ end
+ end
+
+ private
+
+ # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+ # @param [BenchmarkDriver::Context] context
+ # @return [BenchmarkDriver::Metrics]
+ def run_benchmark(job, context:)
+ benchmark = BenchmarkScript.new(
+ preludes: [context.prelude, job.prelude],
+ script: job.script,
+ teardown: job.teardown,
+ loop_count: job.loop_count,
+ )
+
+ Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+ with_script(benchmark.render(result: f.path, target: target)) do |path|
+ IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2
+ if $?.success?
+ Float(f.read)
+ else
+ BenchmarkDriver::Result::ERROR
+ end
+ end
+ end
+ end
+
+ # This method is overridden by some subclasses
+ def metric
+ METRIC
+ end
+
+ # This method is overridden by some subclasses
+ def target
+ :total
+ end
+
+ def with_script(script)
+ if @config.verbose >= 2
+ sep = '-' * 30
+ $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
+ end
+
+ Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+ f.puts script
+ f.close
+ return yield(f.path)
+ end
+ end
+
+ # @param [String] prelude
+ # @param [String] script
+ # @param [String] teardown
+ # @param [Integer] loop_count
+ BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do
+ # @param [String] result - A file to write result
+ def render(result:, target:)
+ prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n")
+ <<-RUBY
+#{prelude}
+
+require 'benchmark'
+__bmdv_result = Benchmark.measure {
+ #{while_loop(script, loop_count)}
+}
+
+#{teardown}
+
+File.write(#{result.dump}, __bmdv_result.#{target})
+ RUBY
+ end
+
+ private
+
+ def while_loop(content, times)
+ if !times.is_a?(Integer) || times <= 0
+ raise ArgumentError.new("Unexpected times: #{times.inspect}")
+ elsif times == 1
+ return content
+ end
+
+ # TODO: execute in batch
+ <<-RUBY
+__bmdv_i = 0
+while __bmdv_i < #{times}
+ #{content}
+ __bmdv_i += 1
+end
+ RUBY
+ end
+ end
+ private_constant :BenchmarkScript
+end
diff --git a/benchmark/lib/benchmark_driver/runner/utime.rb b/benchmark/lib/benchmark_driver/runner/utime.rb
new file mode 100644
index 0000000..b61d83a
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/utime.rb
@@ -0,0 +1,22 @@
+require 'benchmark_driver/runner/total'
+
+class BenchmarkDriver::Runner::Utime < BenchmarkDriver::Runner::Total
+ METRIC = BenchmarkDriver::Metric.new(name: 'utime', unit: 's', larger_better: false)
+
+ # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+ Job = Class.new(BenchmarkDriver::DefaultJob)
+ # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+ JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+ private
+
+ # Overriding BenchmarkDriver::Runner::Total#metric
+ def metric
+ METRIC
+ end
+
+ # Overriding BenchmarkDriver::Runner::Total#target
+ def target
+ :utime
+ end
+end