From 27382eb9fc3f8de4884a5b14903fecb64ba76011 Mon Sep 17 00:00:00 2001
From: Takashi Kokubun <takashikkbn@gmail.com>
Date: Wed, 10 Feb 2021 21:24:25 -0800
Subject: Add a benchmark-driver runner for Ractor (#4172)

* Add a benchmark-driver runner for Ractor

* Process.clock_gettime(Process:CLOCK_MONOTONIC) could be slow

in Ruby 3.0 Ractor

* Fetching Time could also be slow

* Fix a comment

* Assert overriding a private method
---
 benchmark/lib/benchmark_driver/runner/ractor.rb | 119 ++++++++++++++++++++++++
 benchmark/ractor_const.yml                      |   4 +
 benchmark/ractor_float_to_s.yml                 |   8 ++
 common.mk                                       |   2 +-
 4 files changed, 132 insertions(+), 1 deletion(-)
 create mode 100644 benchmark/lib/benchmark_driver/runner/ractor.rb
 create mode 100644 benchmark/ractor_const.yml
 create mode 100644 benchmark/ractor_float_to_s.yml

diff --git a/benchmark/lib/benchmark_driver/runner/ractor.rb b/benchmark/lib/benchmark_driver/runner/ractor.rb
new file mode 100644
index 0000000000..15893b45cf
--- /dev/null
+++ b/benchmark/lib/benchmark_driver/runner/ractor.rb
@@ -0,0 +1,119 @@
+require 'erb'
+
+# A runner to measure performance *inside* Ractor
+class BenchmarkDriver::Runner::Ractor < BenchmarkDriver::Runner::Ips
+  # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job"
+  Job = Class.new(BenchmarkDriver::DefaultJob) do
+    attr_accessor :ractor
+  end
+
+  # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse`
+  JobParser = BenchmarkDriver::DefaultJobParser.for(klass: Job, metrics: [METRIC]).extend(Module.new{
+    def parse(ractor: 1, **kwargs)
+      super(**kwargs).each do |job|
+        job.ractor = ractor
+      end
+    end
+  })
+
+  private
+
+  unless private_instance_methods.include?(:run_benchmark)
+    raise "#run_benchmark is no longer defined in BenchmarkDriver::Runner::Ips"
+  end
+
+  # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil
+  # @param [BenchmarkDriver::Context] context
+  # @return [BenchmarkDriver::Metrics]
+  def run_benchmark(job, context:)
+    benchmark = BenchmarkScript.new(
+      preludes:   [context.prelude, job.prelude],
+      script:     job.script,
+      teardown:   job.teardown,
+      loop_count: job.loop_count,
+    )
+
+    results = job.ractor.times.map do
+      Tempfile.open('benchmark_driver_result')
+    end
+    duration = with_script(benchmark.render(results: results.map(&:path))) do |path|
+      success = execute(*context.executable.command, path, exception: false)
+      if success && ((value = results.map { |f| Float(f.read) }.max) > 0)
+        value
+      else
+        BenchmarkDriver::Result::ERROR
+      end
+    end
+    results.each(&:close)
+
+    value_duration(
+      loop_count: job.loop_count,
+      duration: duration,
+    )
+  end
+
+  # @param [String] prelude
+  # @param [String] script
+  # @param [String] teardown
+  # @param [Integer] loop_count
+  BenchmarkScript = ::BenchmarkDriver::Struct.new(:preludes, :script, :teardown, :loop_count) do
+    # @param [String] result - A file to write result
+    def render(results:)
+      prelude = preludes.reject(&:nil?).reject(&:empty?).join("\n")
+      ERB.new(<<-RUBY).result_with_hash(results: results)
+Warning[:experimental] = false
+# shareable-constant-value: experimental_everything
+#{prelude}
+
+if #{loop_count} == 1
+  __bmdv_empty_before = 0
+  __bmdv_empty_after = 0
+else
+  __bmdv_empty_before = Time.new
+  #{while_loop('', loop_count, id: 0)}
+  __bmdv_empty_after = Time.new
+end
+
+ractors = []
+<% results.each do |result| %>
+ractors << Ractor.new(__bmdv_empty_after - __bmdv_empty_before) { |loop_time|
+  __bmdv_time = Time
+  __bmdv_script_before = __bmdv_time.new
+  #{while_loop(script, loop_count, id: 1)}
+  __bmdv_script_after = __bmdv_time.new
+
+  File.write(
+    <%= result.dump %>,
+    ((__bmdv_script_after - __bmdv_script_before) - loop_time).inspect,
+  )
+}
+<% end %>
+ractors.each(&:take)
+
+#{teardown}
+      RUBY
+    end
+
+    private
+
+    # id is to prevent:
+    # can not isolate a Proc because it accesses outer variables (__bmdv_i)
+    def while_loop(content, times, id:)
+      if !times.is_a?(Integer) || times <= 0
+        raise ArgumentError.new("Unexpected times: #{times.inspect}")
+      elsif times == 1
+        return content
+      end
+
+      # TODO: execute in batch
+      <<-RUBY
+__bmdv_i#{id} = 0
+while __bmdv_i#{id} < #{times}
+  #{content}
+  __bmdv_i#{id} += 1
+end
+      RUBY
+    end
+  end
+  private_constant :BenchmarkScript
+end
diff --git a/benchmark/ractor_const.yml b/benchmark/ractor_const.yml
new file mode 100644
index 0000000000..d7ab74bdca
--- /dev/null
+++ b/benchmark/ractor_const.yml
@@ -0,0 +1,4 @@
+type: lib/benchmark_driver/runner/ractor
+benchmark:
+  ractor_const: Object
+ractor: 1
diff --git a/benchmark/ractor_float_to_s.yml b/benchmark/ractor_float_to_s.yml
new file mode 100644
index 0000000000..8f492be668
--- /dev/null
+++ b/benchmark/ractor_float_to_s.yml
@@ -0,0 +1,8 @@
+type: lib/benchmark_driver/runner/ractor
+prelude: |
+  FLOATS = [*0.0.step(1.0, 0.001)]
+benchmark:
+  ractor_float_to_s: |
+    FLOATS.each {|f| f.to_s}
+loop_count: 100
+ractor: 2
diff --git a/common.mk b/common.mk
index 1a3de30b46..c9507d2ad8 100644
--- a/common.mk
+++ b/common.mk
@@ -48,7 +48,7 @@ GEM_PATH =
 GEM_VENDOR =
 
 BENCHMARK_DRIVER_GIT_URL = https://github.com/benchmark-driver/benchmark-driver
-BENCHMARK_DRIVER_GIT_REF = v0.15.15
+BENCHMARK_DRIVER_GIT_REF = v0.15.17
 SIMPLECOV_GIT_URL = https://github.com/colszowka/simplecov.git
 SIMPLECOV_GIT_REF = v0.17.0
 SIMPLECOV_HTML_GIT_URL = https://github.com/colszowka/simplecov-html.git
-- 
cgit v1.2.3