10 files changed, 577 insertions, 0 deletions
diff --git a/benchmark/lib/benchmark_driver/output/driver.rb b/benchmark/lib/benchmark_driver/output/driver.rb
new file mode 100644
index 0000000000..d22236e9fb
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/output/driver.rb
@@ -0,0 +1,36 @@
+require 'benchmark_driver/output/simple'
+
+# This replicates the legacy benchmark/driver.rb behavior.
+class BenchmarkDriver::Output::Driver < BenchmarkDriver::Output::Simple
+  def initialize(*)
+    super
+    @stdout = $stdout
+    @strio  = StringIO.new
+    $stdout = IOMultiplexer.new(@stdout, @strio)
+  end
+
+  def with_benchmark(*)
+    super
+  ensure
+    logfile = "bmlog-#{Time.now.strftime('%Y%m%d-%H%M%S')}.#{$$}.log"
+    puts "\nLog file: #{logfile}"
+
+    $stdout = @stdout
+    File.write(logfile, @strio.tap(&:rewind).read)
+  end
+
+  class IOMultiplexer
+    def initialize(io1, io2)
+      @io1 = io1
+      @io2 = io2
+    end
+
+    [:write, :sync, :sync=, :puts, :print, :flush].each do |method|
+      define_method(method) do |*args|
+        @io1.send(method, *args)
+        @io2.send(method, *args)
+      end
+    end
+  end
+  private_constant :IOMultiplexer
+end
diff --git a/benchmark/lib/benchmark_driver/runner/cstime.rb b/benchmark/lib/benchmark_driver/runner/cstime.rb
new file mode 100644
index 0000000000..3c3453e527
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/cstime.rb
@@ -0,0 +1,22 @@
+require 'benchmark_driver/runner/total'
+
+class BenchmarkDriver::Runner::Cstime < BenchmarkDriver::Runner::Total
+  METRIC = BenchmarkDriver::Metric.new(name: 'cstime', unit: 's', larger_better: false)
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  private
+
+  # Overriding BenchmarkDriver::Runner::Total#metric
+  def metric
+    METRIC
+  end
+
+  # Overriding BenchmarkDriver::Runner::Total#target
+  def target
+    :cstime
+  end
+end
diff --git a/benchmark/lib/benchmark_driver/runner/cutime.rb b/benchmark/lib/benchmark_driver/runner/cutime.rb
new file mode 100644
index 0000000000..e139962ef2
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/cutime.rb
@@ -0,0 +1,22 @@
+require 'benchmark_driver/runner/total'
+
+class BenchmarkDriver::Runner::Cutime < BenchmarkDriver::Runner::Total
+  METRIC = BenchmarkDriver::Metric.new(name: 'cutime', unit: 's', larger_better: false)
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  private
+
+  # Overriding BenchmarkDriver::Runner::Total#metric
+  def metric
+    METRIC
+  end
+
+  # Overriding BenchmarkDriver::Runner::Total#target
+  def target
+    :cutime
+  end
+end
diff --git a/benchmark/lib/benchmark_driver/runner/peak.rb b/benchmark/lib/benchmark_driver/runner/peak.rb
new file mode 100644
index 0000000000..d04f2e51ff
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/peak.rb
@@ -0,0 +1,151 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'benchmark_driver/default_job'
+require 'benchmark_driver/default_job_parser'
+require 'tempfile'
+
+class BenchmarkDriver::Runner::Peak
+  METRIC = BenchmarkDriver::Metric.new(
+    name: 'Peak memory usage', unit: 'bytes', larger_better: false, worse_word: 'larger',
+  )
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output] output
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
+    @config = config
+    @output = output
+    @contexts = contexts
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Runner::Peak::Job>] jobs
+  def run(jobs)
+    if jobs.any? { |job| job.loop_count.nil? }
+      jobs = jobs.map do |job|
+        job.loop_count ? job : Job.new(job.to_h.merge(loop_count: 1))
+      end
+    end
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          job.runnable_contexts(@contexts).each do |context|
+            value = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
+              run_benchmark(job, context: context)
+            end
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
+              @output.report(values: { metric => value }, loop_count: job.loop_count)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Context] context
+  # @return [BenchmarkDriver::Metrics]
+  def run_benchmark(job, context:)
+    benchmark = BenchmarkScript.new(
+      preludes:   [context.prelude, job.prelude],
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+    )
+
+    memory_status = File.expand_path('../../../../tool/lib/memory_status', __dir__)
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      with_script(benchmark.render) do |path|
+        output = IO.popen([*context.executable.command, path, f.path, target, memory_status], &:read)
+        if $?.success?
+          Integer(f.read)
+        else
+          $stdout.print(output)
+          BenchmarkDriver::Result::ERROR
+        end
+      end
+    end
+  end
+
+  # Overridden by BenchmarkDriver::Runner::Size
+  def target
+    'peak'
+  end
+
+  # Overridden by BenchmarkDriver::Runner::Size
+  def metric
+    METRIC
+  end
+
+  def with_script(script)
+    if @config.verbose >= 2
+      sep = '-' * 30
+      $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
+    end
+
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      f.puts script
+      f.close
+      return yield(f.path)
+    end
+  end
+
+  # @param [String] prelude
+  # @param [String] script
+  # @param [String] teardown
+  # @param [Integer] loop_count
+  BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do
+    def render
+      prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n")
+      <<-RUBY
+#{prelude}
+#{while_loop(script, loop_count)}
+#{teardown}
+
+result_file, target, memory_status = ARGV
+require_relative memory_status
+
+ms = Memory::Status.new
+case target.to_sym
+when :peak
+  key = ms.respond_to?(:hwm) ? :hwm : :peak
+when :size
+  key = ms.respond_to?(:rss) ? :rss : :size
+else
+  raise('unexpected target: ' + target)
+end
+
+File.write(result_file, ms[key])
+      RUBY
+    end
+
+    private
+
+    def while_loop(content, times)
+      if !times.is_a?(Integer) || times <= 0
+        raise ArgumentError.new("Unexpected times: #{times.inspect}")
+      end
+
+      if times > 1
+        <<-RUBY
+__bmdv_i = 0
+while __bmdv_i < #{times}
+  #{content}
+  __bmdv_i += 1
+end
+        RUBY
+      else
+        content
+      end
+    end
+  end
+  private_constant :BenchmarkScript
+end
diff --git a/benchmark/lib/benchmark_driver/runner/ractor.rb b/benchmark/lib/benchmark_driver/runner/ractor.rb
new file mode 100644
index 0000000000..fd9c2dd4db
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/ractor.rb
@@ -0,0 +1,122 @@
+require 'erb'
+
+# A runner to measure performance *inside* Ractor
+class BenchmarkDriver::Runner::Ractor < BenchmarkDriver::Runner::Ips
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob) do
+    attr_accessor :ractor
+  end
+
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]).extend(Module.new{
+    def parse(ractor: 1, **kwargs)
+      super(**kwargs).each do |job|
+        job.ractor = ractor
+      end
+    end
+  })
+
+  private
+
+  unless private_instance_methods.include?(:run_benchmark)
+    raise "#run_benchmark is no longer defined in BenchmarkDriver::Runner::Ips"
+  end
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Context] context
+  # @return [BenchmarkDriver::Metrics]
+  def run_benchmark(job, context:)
+    benchmark = BenchmarkScript.new(
+      preludes:   [context.prelude, job.prelude],
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+    )
+
+    results = job.ractor.times.map do
+      Tempfile.open('benchmark_driver_result')
+    end
+    duration = with_script(benchmark.render(results: results.map(&:path))) do |path|
+      success = execute(*context.executable.command, path, exception: false)
+      if success && ((value = results.map { |f| Float(f.read) }.max) > 0)
+        value
+      else
+        BenchmarkDriver::Result::ERROR
+      end
+    end
+    results.each(&:close)
+
+    value_duration(
+      loop_count: job.loop_count,
+      duration: duration,
+    )
+  end
+
+  # @param [String] prelude
+  # @param [String] script
+  # @param [String] teardown
+  # @param [Integer] loop_count
+  BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(results:)
+      prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n")
+      ERB.new(<<-RUBY).result_with_hash(results: results)
+Warning[:experimental] = false
+# shareable-constant-value: experimental_everything
+#{prelude}
+
+if #{loop_count} == 1
+  __bmdv_loop_before = 0
+  __bmdv_loop_after = 0
+else
+  __bmdv_loop_before = Time.new
+  #{while_loop('', loop_count, id: 0)}
+  __bmdv_loop_after = Time.new
+end
+
+__bmdv_ractors = []
+<% results.size.times do %>
+__bmdv_ractors << Ractor.new(__bmdv_loop_after - __bmdv_loop_before) { |__bmdv_loop_time|
+  __bmdv_time = Time
+  __bmdv_script_before = __bmdv_time.new
+  #{while_loop(script, loop_count, id: 1)}
+  __bmdv_script_after = __bmdv_time.new
+
+  (__bmdv_script_after - __bmdv_script_before) - __bmdv_loop_time
+}
+<% end %>
+
+# Wait for all Ractors before executing code to write results
+__bmdv_ractors.map!(&:value)
+
+<% results.each do |result| %>
+File.write(<%= result.dump %>, __bmdv_ractors.shift)
+<% end %>
+
+#{teardown}
+      RUBY
+    end
+
+    private
+
+    # id is to prevent:
+    # can not isolate a Proc because it accesses outer variables (__bmdv_i)
+    def while_loop(content, times, id:)
+      if !times.is_a?(Integer) || times <= 0
+        raise ArgumentError.new("Unexpected times: #{times.inspect}")
+      elsif times == 1
+        return content
+      end
+
+      # TODO: execute in batch
+      <<-RUBY
+__bmdv_i#{id} = 0
+while __bmdv_i#{id} < #{times}
+  #{content}
+  __bmdv_i#{id} += 1
+end
+      RUBY
+    end
+  end
+  private_constant :BenchmarkScript
+end
diff --git a/benchmark/lib/benchmark_driver/runner/size.rb b/benchmark/lib/benchmark_driver/runner/size.rb
new file mode 100644
index 0000000000..1b31f901c7
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/size.rb
@@ -0,0 +1,25 @@
+require 'benchmark_driver/runner/peak'
+
+# Actually the same as BenchmarkDriver::Runner::Memory
+class BenchmarkDriver::Runner::Size < BenchmarkDriver::Runner::Peak
+  METRIC = BenchmarkDriver::Metric.new(
+    name: 'Max resident set size', unit: 'bytes', larger_better: false, worse_word: 'larger',
+  )
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  private
+
+  # Overriding BenchmarkDriver::Runner::Peak#metric
+  def metric
+    METRIC
+  end
+
+  # Overriding BenchmarkDriver::Runner::Peak#target
+  def target
+    'size'
+  end
+end
diff --git a/benchmark/lib/benchmark_driver/runner/stime.rb b/benchmark/lib/benchmark_driver/runner/stime.rb
new file mode 100644
index 0000000000..4577fb0bf8
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/stime.rb
@@ -0,0 +1,22 @@
+require 'benchmark_driver/runner/total'
+
+class BenchmarkDriver::Runner::Stime < BenchmarkDriver::Runner::Total
+  METRIC = BenchmarkDriver::Metric.new(name: 'stime', unit: 's', larger_better: false)
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  private
+
+  # Overriding BenchmarkDriver::Runner::Total#metric
+  def metric
+    METRIC
+  end
+
+  # Overriding BenchmarkDriver::Runner::Total#target
+  def target
+    :stime
+  end
+end
diff --git a/benchmark/lib/benchmark_driver/runner/total.rb b/benchmark/lib/benchmark_driver/runner/total.rb
new file mode 100644
index 0000000000..64dc14f84e
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/total.rb
@@ -0,0 +1,137 @@
+require 'benchmark_driver/struct'
+require 'benchmark_driver/metric'
+require 'benchmark_driver/default_job'
+require 'benchmark_driver/default_job_parser'
+require 'tempfile'
+
+class BenchmarkDriver::Runner::Total
+  METRIC = BenchmarkDriver::Metric.new(name: 'Total time', unit: 's', larger_better: false)
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  # @param [BenchmarkDriver::Config::RunnerConfig] config
+  # @param [BenchmarkDriver::Output] output
+  # @param [BenchmarkDriver::Context] contexts
+  def initialize(config:, output:, contexts:)
+    @config = config
+    @output = output
+    @contexts = contexts
+  end
+
+  # This method is dynamically called by `BenchmarkDriver::JobRunner.run`
+  # @param [Array<BenchmarkDriver::Runner::Total::Job>] jobs
+  def run(jobs)
+    if jobs.any? { |job| job.loop_count.nil? }
+      raise 'missing loop_count is not supported in Ruby repository'
+    end
+
+    @output.with_benchmark do
+      jobs.each do |job|
+        @output.with_job(name: job.name) do
+          job.runnable_contexts(@contexts).each do |context|
+            duration = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: false) do
+              run_benchmark(job, context: context)
+            end
+            @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do
+              @output.report(values: { metric => duration }, duration: duration, loop_count: job.loop_count)
+            end
+          end
+        end
+      end
+    end
+  end
+
+  private
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Context] context
+  # @return [BenchmarkDriver::Metrics]
+  def run_benchmark(job, context:)
+    benchmark = BenchmarkScript.new(
+      preludes:   [context.prelude, job.prelude],
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+    )
+
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      with_script(benchmark.render(result: f.path, target: target)) do |path|
+        IO.popen([*context.executable.command, path], &:read) # TODO: print stdout if verbose=2
+        if $?.success?
+          Float(f.read)
+        else
+          BenchmarkDriver::Result::ERROR
+        end
+      end
+    end
+  end
+
+  # This method is overridden by some subclasses
+  def metric
+    METRIC
+  end
+
+  # This method is overridden by some subclasses
+  def target
+    :total
+  end
+
+  def with_script(script)
+    if @config.verbose >= 2
+      sep = '-' * 30
+      $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n"
+    end
+
+    Tempfile.open(['benchmark_driver-', '.rb']) do |f|
+      f.puts script
+      f.close
+      return yield(f.path)
+    end
+  end
+
+  # @param [String] prelude
+  # @param [String] script
+  # @param [String] teardown
+  # @param [Integer] loop_count
+  BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(result:, target:)
+      prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n")
+      <<-RUBY
+#{prelude}
+
+require 'benchmark'
+__bmdv_result = Benchmark.measure {
+  #{while_loop(script, loop_count)}
+}
+
+#{teardown}
+
+File.write(#{result.dump}, __bmdv_result.#{target})
+      RUBY
+    end
+
+    private
+
+    def while_loop(content, times)
+      if !times.is_a?(Integer) || times <= 0
+        raise ArgumentError.new("Unexpected times: #{times.inspect}")
+      elsif times == 1
+        return content
+      end
+
+      # TODO: execute in batch
+      <<-RUBY
+__bmdv_i = 0
+while __bmdv_i < #{times}
+  #{content}
+  __bmdv_i += 1
+end
+      RUBY
+    end
+  end
+  private_constant :BenchmarkScript
+end
diff --git a/benchmark/lib/benchmark_driver/runner/utime.rb b/benchmark/lib/benchmark_driver/runner/utime.rb
new file mode 100644
index 0000000000..b61d83a188
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/utime.rb
@@ -0,0 +1,22 @@
+require 'benchmark_driver/runner/total'
+
+class BenchmarkDriver::Runner::Utime < BenchmarkDriver::Runner::Total
+  METRIC = BenchmarkDriver::Metric.new(name: 'utime', unit: 's', larger_better: false)
+
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob)
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC])
+
+  private
+
+  # Overriding BenchmarkDriver::Runner::Total#metric
+  def metric
+    METRIC
+  end
+
+  # Overriding BenchmarkDriver::Runner::Total#target
+  def target
+    :utime
+  end
+end
diff --git a/benchmark/lib/load.rb b/benchmark/lib/load.rb
new file mode 100644
index 0000000000..31b770c484
--- /dev/null
+++ b/benchmark/lib/load.rb
@@ -0,0 +1,18 @@
+# How to use this file:
+#   1. write a `$(srcdir)/test.rb` like:
+=begin
+require_relative 'benchmark/lib/load'
+
+Benchmark.driver(repeat_count: 5){|x|
+  x.executable name: 'clean-miniruby', command: %w'../clean-trunk/miniruby'
+  x.executable name: 'modif-miniruby', command: %w'./miniruby'
+
+  x.report %q{
+    h = {a: 1, b: 2, c: 3, d: 4}
+  }
+}
+=end
+#
+#  2. `make run`
+$:.unshift(File.join(__dir__, '../benchmark-driver/lib'))
+require 'benchmark_driver'