#!/usr/bin/env ruby # frozen_string_literal: true require 'fileutils' require 'optparse' require 'tmpdir' require 'logger' require 'digest' require 'shellwords' GitRef = Struct.new(:ref, :commit_hash) RUBIES_DIR = File.join(Dir.home, '.zjit-diff') BEFORE_NAME = 'ruby-zjit-before' AFTER_NAME = 'ruby-zjit-after' LOG = Logger.new($stderr) def macos? Gem::Platform.local == 'darwin' end class CommandRunner def initialize(quiet: false) @quiet = quiet end def cmd(*args, **options) options[:out] ||= @quiet ? File::NULL : $stderr options = options.merge(exception: true) system(*args, **options) end end class ZJITDiff DATA_FILENAME = File.join('data', 'zjit_diff') RUBY_BENCH_REPO_URL = 'https://github.com/ruby/ruby-bench.git' def initialize(before_hash:, after_hash:, runner:, options:) @before_hash = before_hash @after_hash = after_hash @runner = runner @options = options end def bench! LOG.info('Running benchmarks') ruby_bench_path = @options[:bench_path] || setup_ruby_bench run_benchmarks(ruby_bench_path) end private def run_benchmarks(ruby_bench_path) Dir.chdir(ruby_bench_path) do @runner.cmd({ 'RUBIES_DIR' => RUBIES_DIR }, './run_benchmarks.rb', '--chruby', "before::#{@before_hash} --zjit-stats;after::#{@after_hash} --zjit-stats", '--out-name', DATA_FILENAME, *@options[:bench_args], *@options[:name_filters]) @runner.cmd('./misc/zjit_diff.rb', "#{DATA_FILENAME}.json", out: $stdout) end end def setup_ruby_bench path = File.join(Dir.tmpdir, 'ruby-bench') if Dir.exist?(path) LOG.info('ruby-bench already cloned, pulling from upstream') Dir.chdir(path) do @runner.cmd('git', 'pull') end else LOG.info("ruby-bench not cloned yet, cloning repository to #{path}") @runner.cmd('git', 'clone', RUBY_BENCH_REPO_URL, path) end path end end class RubyWorktree attr_reader :hash BREW_REQUIRED_PACKAGES = %w[openssl readline libyaml].freeze def initialize(name:, ref:, runner:, force_rebuild: false) @path = File.join(Dir.tmpdir, name) @ref = ref @force_rebuild = force_rebuild @runner = runner @hash = nil setup_worktree end def build! Dir.chdir(@path) do configure_cmd_args = ['--enable-zjit=dev', '--disable-install-doc'] if macos? brew_prefixes = BREW_REQUIRED_PACKAGES.map do |pkg| `brew --prefix #{pkg}`.strip end configure_cmd_args << "--with-opt-dir=#{brew_prefixes.join(':')}" end configure_cmd_hash = Digest::MD5.hexdigest(configure_cmd_args.join('')) build_cmd_args = ['-j', 'miniruby'] build_cmd_hash = Digest::MD5.hexdigest(build_cmd_args.join('')) @hash = "#{configure_cmd_hash}-#{build_cmd_hash}-#{@ref.commit_hash}" prefix = File.join(RUBIES_DIR, @hash) if Dir.exist?(prefix) && !@force_rebuild LOG.info("Found existing build for #{@ref.ref}, skipping build") return end @runner.cmd('./autogen.sh') cmd = [ './configure', *configure_cmd_args, "--prefix=#{prefix}" ] @runner.cmd(*cmd) @runner.cmd('make', *build_cmd_args) @runner.cmd('make', 'install') end end private def setup_worktree if Dir.exist?(@path) LOG.info("Existing worktree found at #{@path}") Dir.chdir(@path) do @runner.cmd('git', 'checkout', @ref.commit_hash) end else LOG.info("Creating worktree for ref '#{@ref.ref}' at #{@path}") @runner.cmd('git', 'worktree', 'add', '--detach', @path, @ref.commit_hash) end end end def clean! [BEFORE_NAME, AFTER_NAME].each do |name| path = File.join(Dir.tmpdir, name) if Dir.exist?(path) LOG.info("Removing worktree at #{path}") system('git', 'worktree', 'remove', '--force', path) end end if Dir.exist?(RUBIES_DIR) LOG.info("Removing ruby installations from #{RUBIES_DIR}") FileUtils.rm_rf(RUBIES_DIR) end bench_path = File.join(Dir.tmpdir, 'ruby-bench') return unless Dir.exist?(bench_path) LOG.info("Removing ruby-bench clone at #{bench_path}") FileUtils.rm_rf(bench_path) end def parse_ref(ref) out = `git rev-parse --verify #{ref}` return nil unless $?.success? GitRef.new(ref: ref, commit_hash: out.strip) end DEFAULT_BENCHMARKS = %w[lobsters railsbench].freeze options = {} subtext = <<~HELP Subcommands: bench : Run benchmarks clean : Clean temporary files created by benchmarks See '#{$PROGRAM_NAME} COMMAND --help' for more information on a specific command. HELP top_level = OptionParser.new do |opts| opts.banner = "Usage: #{$PROGRAM_NAME} [options]" opts.separator('') opts.separator(subtext) end subcommands = { 'bench' => OptionParser.new do |opts| opts.banner = "Usage: #{$PROGRAM_NAME} [options] " opts.on('--before REF', 'Git ref for ruby (before)') do |ref| git_ref = parse_ref ref if git_ref.nil? warn "Error: '#{ref}' is not a valid git ref" exit 1 end options[:before] = git_ref end opts.on('--after REF', 'Git ref for ruby (after)') do |ref| git_ref = parse_ref ref if git_ref.nil? warn "Error: '#{ref}' is not a valid git ref" exit 1 end options[:after] = git_ref end opts.on('--bench-path PATH', 'Path to an existing ruby-bench repository clone ' \ '(if not specified, ruby-bench will be cloned automatically to a temporary directory)') do |path| options[:bench_path] = path end opts.on('--bench-args ARGS', 'Args to pass to ruby-bench') do |bench_args| options[:bench_args] = bench_args.shellsplit end opts.on('--force-rebuild', 'Force building ruby again instead of using even if existing builds exist in the cache at ~/.diffs') do options[:force_rebuild] = true end opts.on('--quiet', 'Silence output of commands except for benchmark result') do options[:quiet] = true end opts.separator('') opts.separator('If no benchmarks are specified, the benchmarks that will be run are:') opts.separator(DEFAULT_BENCHMARKS.join(', ')) end, 'clean' => OptionParser.new do |opts| end } top_level.order! command = ARGV.shift subcommands[command].order! case command when 'bench' options[:name_filters] = ARGV.empty? ? DEFAULT_BENCHMARKS : ARGV options[:after] ||= parse_ref('HEAD') runner = CommandRunner.new(quiet: options[:quiet]) before = RubyWorktree.new(name: BEFORE_NAME, ref: options[:before], runner: runner, force_rebuild: options[:force_rebuild]) before.build! after = RubyWorktree.new(name: AFTER_NAME, ref: options[:after], runner: runner, force_rebuild: options[:force_rebuild]) after.build! zjit_diff = ZJITDiff.new(runner: runner, before_hash: before.hash, after_hash: after.hash, options: options) zjit_diff.bench! when 'clean' clean! end