From 4a4c5028258e53f3395af29655a66bcef796fd73 Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Thu, 26 Sep 2019 12:57:43 +0900 Subject: Add special runner to benchmark mjit_exec I wanted to dynamically generate benchmark cases to test various number of methods. Thus I added a dedicated runner of benchmark-driver. --- benchmark/lib/benchmark_driver/runner/mjit_exec.rb | 237 +++++++++++++++++++++ benchmark/mjit_exec.yml | 46 ---- benchmark/mjit_exec_jt2jt.yml | 11 + benchmark/mjit_exec_vm2jt.yml | 10 + benchmark/mjit_exec_vm2vm.yml | 10 + 5 files changed, 268 insertions(+), 46 deletions(-) create mode 100644 benchmark/lib/benchmark_driver/runner/mjit_exec.rb delete mode 100644 benchmark/mjit_exec.yml create mode 100644 benchmark/mjit_exec_jt2jt.yml create mode 100644 benchmark/mjit_exec_vm2jt.yml create mode 100644 benchmark/mjit_exec_vm2vm.yml (limited to 'benchmark') diff --git a/benchmark/lib/benchmark_driver/runner/mjit_exec.rb b/benchmark/lib/benchmark_driver/runner/mjit_exec.rb new file mode 100644 index 0000000000..7477fa1830 --- /dev/null +++ b/benchmark/lib/benchmark_driver/runner/mjit_exec.rb @@ -0,0 +1,237 @@ +require 'benchmark_driver/struct' +require 'benchmark_driver/metric' +require 'erb' + +# A special runner dedicated for measuring mjit_exec overhead. +class BenchmarkDriver::Runner::MjitExec + METRIC = BenchmarkDriver::Metric.new(name: 'Iteration per second', unit: 'i/s') + + # JobParser returns this, `BenchmarkDriver::Runner.runner_for` searches "*::Job" + Job = ::BenchmarkDriver::Struct.new( + :name, # @param [String] name - This is mandatory for all runner + :metrics, # @param [Array] + :num_methods, # @param [Integer] num_methods - The number of methods to be defined + :loop_count, # @param [Integer] loop_count + :from_jit, # @param [TrueClass,FalseClass] from_jit - Whether the mjit_exec() is from JIT or not + :to_jit, # @param [TrueClass,FalseClass] to_jit - Whether the mjit_exec() is to JIT or not + ) + # Dynamically fetched and used by `BenchmarkDriver::JobParser.parse` + class << JobParser = Module.new + # @param [Array,String] num_methods + # @param [Integer] loop_count + # @param [TrueClass,FalseClass] from_jit + # @param [TrueClass,FalseClass] to_jit + def parse(num_methods:, loop_count:, from_jit:, to_jit:) + if num_methods.is_a?(String) + num_methods = eval(num_methods) + end + + num_methods.map do |num| + if num_methods.size > 1 + suffix = "[#{'%4d' % num}]" + else + suffix = "_#{num}" + end + Job.new( + name: "mjit_exec_#{from_jit ? 'JT' : 'VM'}2#{to_jit ? 'JT' : 'VM'}#{suffix}", + metrics: [METRIC], + num_methods: num, + loop_count: loop_count, + from_jit: from_jit, + to_jit: to_jit, + ) + end + end + end + + # @param [BenchmarkDriver::Config::RunnerConfig] config + # @param [BenchmarkDriver::Output] output + # @param [BenchmarkDriver::Context] contexts + def initialize(config:, output:, contexts:) + @config = config + @output = output + @contexts = contexts + end + + # This method is dynamically called by `BenchmarkDriver::JobRunner.run` + # @param [Array] jobs + def run(jobs) + @output.with_benchmark do + jobs.each do |job| + @output.with_job(name: job.name) do + @contexts.each do |context| + result = BenchmarkDriver::Repeater.with_repeat(config: @config, larger_better: true, rest_on_average: :average) do + run_benchmark(job, context: context) + end + value, duration = result.value + @output.with_context(name: context.name, executable: context.executable, gems: context.gems, prelude: context.prelude) do + @output.report(values: { METRIC => value }, duration: duration, loop_count: job.loop_count) + end + end + end + end + end + end + + private + + # @param [BenchmarkDriver::Runner::Ips::Job] job - loop_count is not nil + # @param [BenchmarkDriver::Context] context + # @return [BenchmarkDriver::Metrics] + def run_benchmark(job, context:) + if job.from_jit + if job.to_jit + benchmark = BenchmarkJT2JT.new(num_methods: job.num_methods, loop_count: job.loop_count) + else + raise NotImplementedError, "JT2VM is not implemented yet" + end + else + if job.to_jit + benchmark = BenchmarkVM2JT.new(num_methods: job.num_methods, loop_count: job.loop_count) + else + benchmark = BenchmarkVM2VM.new(num_methods: job.num_methods, loop_count: job.loop_count) + end + end + + duration = Tempfile.open(['benchmark_driver-result', '.txt']) do |f| + with_script(benchmark.render(result: f.path)) do |path| + opt = [] + if context.executable.command.any? { |c| c.start_with?('--jit') } + opt << '--jit-min-calls=2' + end + IO.popen([*context.executable.command, '--disable-gems', *opt, path], &:read) + if $?.success? + Float(f.read) + else + BenchmarkDriver::Result::ERROR + end + end + end + + [job.loop_count.to_f / duration, duration] + end + + def with_script(script) + if @config.verbose >= 2 + sep = '-' * 30 + $stdout.puts "\n\n#{sep}[Script begin]#{sep}\n#{script}#{sep}[Script end]#{sep}\n\n" + end + + Tempfile.open(['benchmark_driver-', '.rb']) do |f| + f.puts script + f.close + return yield(f.path) + end + end + + # @param [Integer] num_methods + # @param [Integer] loop_count + BenchmarkVM2VM = ::BenchmarkDriver::Struct.new(:num_methods, :loop_count) do + # @param [String] result - A file to write result + def render(result:) + ERB.new(<<~EOS, trim_mode: '%').result(binding) + % num_methods.times do |i| + def a<%= i %> + nil + end + % end + RubyVM::MJIT.pause if RubyVM::MJIT.enabled? + + def vm + t = Process.clock_gettime(Process::CLOCK_MONOTONIC) + i = 0 + while i < 10000 + % (loop_count / 10000).times do |i| + a<%= i % num_methods %> + % end + i += 1 + end + % (loop_count % 10000).times do |i| + a<%= i % num_methods %> + % end + Process.clock_gettime(Process::CLOCK_MONOTONIC) - t + end + + vm # warmup call cache + File.write(<%= result.dump %>, vm) + EOS + end + end + private_constant :BenchmarkVM2VM + + # @param [Integer] num_methods + # @param [Integer] loop_count + BenchmarkVM2JT = ::BenchmarkDriver::Struct.new(:num_methods, :loop_count) do + # @param [String] result - A file to write result + def render(result:) + ERB.new(<<~EOS, trim_mode: '%').result(binding) + % num_methods.times do |i| + def a<%= i %> + nil + end + a<%= i %> + a<%= i %> # --jit-min-calls=2 + % end + RubyVM::MJIT.pause if RubyVM::MJIT.enabled? + + def vm + t = Process.clock_gettime(Process::CLOCK_MONOTONIC) + i = 0 + while i < 10000 + % (loop_count / 10000).times do |i| + a<%= i % num_methods %> + % end + i += 1 + end + % (loop_count % 10000).times do |i| + a<%= i % num_methods %> + % end + Process.clock_gettime(Process::CLOCK_MONOTONIC) - t + end + + vm # warmup call cache + File.write(<%= result.dump %>, vm) + EOS + end + end + private_constant :BenchmarkVM2JT + + # @param [Integer] num_methods + # @param [Integer] loop_count + BenchmarkJT2JT = ::BenchmarkDriver::Struct.new(:num_methods, :loop_count) do + # @param [String] result - A file to write result + def render(result:) + ERB.new(<<~EOS, trim_mode: '%').result(binding) + % num_methods.times do |i| + def a<%= i %> + nil + end + % end + + # You may need to: + # * Increase `JIT_ISEQ_SIZE_THRESHOLD` to 10000000 in mjit.h + # * Always return false in `inlinable_iseq_p()` of mjit_compile.c + def jit + t = Process.clock_gettime(Process::CLOCK_MONOTONIC) + i = 0 + while i < 10000 + % (loop_count / 10000).times do |i| + a<%= i % num_methods %> + % end + i += 1 + end + % (loop_count % 10000).times do |i| + a<%= i % num_methods %> + % end + Process.clock_gettime(Process::CLOCK_MONOTONIC) - t + end + + jit + jit + RubyVM::MJIT.pause if RubyVM::MJIT.enabled? + File.write(<%= result.dump %>, jit) + EOS + end + end + private_constant :BenchmarkJT2JT +end diff --git a/benchmark/mjit_exec.yml b/benchmark/mjit_exec.yml deleted file mode 100644 index d2621888f2..0000000000 --- a/benchmark/mjit_exec.yml +++ /dev/null @@ -1,46 +0,0 @@ -prelude: | - # to be used with: --disable-gems --jit-min-calls=2 - def compile(call) - eval(<<~EOS) - #{call}; #{call} - if RubyVM::MJIT.enabled? - RubyVM::MJIT.pause(wait: true) - end - EOS - end -benchmark: - - name: "mjit_exec_iseq_vme_jit " - prelude: | - def jit() end - compile('jit') - script: jit - - name: mjit_exec_iseq_vme_jit_jit - prelude: | - def jit2() end - def jit() jit2() end - compile('jit') - script: jit - - name: mjit_exec_iseq_vme_jit_vme - prelude: | - def jit2() rescue; end - def jit() jit2() end - compile('jit') - script: jit - - name: "mjit_exec_send_vme_jit " - prelude: | - def jit() end - compile('send(:jit)') - script: send(:jit) - - name: mjit_exec_send_vme_jit_jit - prelude: | - def jit2() end - def jit() send(:jit2) end - compile('send(:jit)') - script: send(:jit) - - name: mjit_exec_send_vme_jit_vme - prelude: | - def jit2() rescue; end - def jit() send(:jit2) end - compile('send(:jit)') - script: send(:jit) -loop_count: 30000000 diff --git a/benchmark/mjit_exec_jt2jt.yml b/benchmark/mjit_exec_jt2jt.yml new file mode 100644 index 0000000000..78c31b3ca8 --- /dev/null +++ b/benchmark/mjit_exec_jt2jt.yml @@ -0,0 +1,11 @@ +# Usage: +# RUBYOPT=-Ibenchmark/lib benchmark-driver -e 'ruby --jit' benchmark/mjit_exec_vm2jt.yml +type: mjit_exec # benchmark/lib/benchmark_driver/runner/mjit_exec.rb +num_methods: [1] +#num_methods: [1, 10, 20, 30, 40, 50, 100, 200, 500, 1000] +#num_methods: [1, 1000] +#num_methods: (1..100) +#num_methods: (1..10).map { |i| i * 10 } +loop_count: 10000000 +from_jit: true +to_jit: true diff --git a/benchmark/mjit_exec_vm2jt.yml b/benchmark/mjit_exec_vm2jt.yml new file mode 100644 index 0000000000..165677e249 --- /dev/null +++ b/benchmark/mjit_exec_vm2jt.yml @@ -0,0 +1,10 @@ +# Usage: +# RUBYOPT=-Ibenchmark/lib benchmark-driver -e 'ruby --jit' benchmark/mjit_exec_vm2jt.yml +type: mjit_exec # benchmark/lib/benchmark_driver/runner/mjit_exec.rb +num_methods: [1] +#num_methods: [1, 10, 20, 30, 40, 50, 100, 200, 500, 1000] +#num_methods: (1..100) +#num_methods: (1..10).map { |i| i * 10 } +loop_count: 10000000 +from_jit: false +to_jit: true diff --git a/benchmark/mjit_exec_vm2vm.yml b/benchmark/mjit_exec_vm2vm.yml new file mode 100644 index 0000000000..3f9274d8bd --- /dev/null +++ b/benchmark/mjit_exec_vm2vm.yml @@ -0,0 +1,10 @@ +# Usage: +# RUBYOPT=-Ibenchmark/lib benchmark-driver -e 'ruby --jit' benchmark/mjit_exec_vm2vm.yml +type: mjit_exec # benchmark/lib/benchmark_driver/runner/mjit_exec.rb +num_methods: [1] +#num_methods: [1, 10, 20, 30, 40, 50, 100, 200, 500, 1000] +#num_methods: (1..100) +#num_methods: (1..10).map { |i| i * 10 } +loop_count: 10000000 +from_jit: false +to_jit: false -- cgit v1.2.3