diff options
Diffstat (limited to 'tool/lib')
| -rw-r--r-- | tool/lib/_tmpdir.rb | 6 | ||||
| -rw-r--r-- | tool/lib/bundle_env.rb | 4 | ||||
| -rw-r--r-- | tool/lib/bundled_gem.rb | 8 | ||||
| -rw-r--r-- | tool/lib/core_assertions.rb | 151 | ||||
| -rw-r--r-- | tool/lib/dump.gdb | 17 | ||||
| -rw-r--r-- | tool/lib/dump.lldb | 13 | ||||
| -rw-r--r-- | tool/lib/envutil.rb | 110 | ||||
| -rw-r--r-- | tool/lib/gem_env.rb | 1 | ||||
| -rw-r--r-- | tool/lib/leakchecker.rb | 2 | ||||
| -rw-r--r-- | tool/lib/memory_status.rb | 100 | ||||
| -rw-r--r-- | tool/lib/test/jobserver.rb | 47 | ||||
| -rw-r--r-- | tool/lib/test/unit.rb | 47 | ||||
| -rw-r--r-- | tool/lib/test/unit/assertions.rb | 32 | ||||
| -rw-r--r-- | tool/lib/vcs.rb | 319 |
14 files changed, 457 insertions, 400 deletions
diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb index fd429dab37..daa1a1f235 100644 --- a/tool/lib/_tmpdir.rb +++ b/tool/lib/_tmpdir.rb @@ -4,11 +4,11 @@ template = "rubytest." # Assume the directory by these environment variables are safe. base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp| next unless tmp and tmp.size <= 50 and File.directory?(tmp) - # On macOS, the default TMPDIR is very long, inspite of UNIX socket - # path length is limited. + # On macOS, the default TMPDIR is very long, in spite of UNIX socket + # path length being limited. # # Also Rubygems creates its own temporary directory per tests, and - # some tests copy the full path of gemhome there. In that caes, the + # some tests copy the full path of gemhome there. In that case, the # path contains both temporary names twice, and can exceed path name # limit very easily. tmp diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb new file mode 100644 index 0000000000..9ad5ea220b --- /dev/null +++ b/tool/lib/bundle_env.rb @@ -0,0 +1,4 @@ +ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__) +ENV["BUNDLE_PATH__SYSTEM"] = "true" +ENV["BUNDLE_WITHOUT"] = "lint doc" diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb index d870925d05..d2ed61a508 100644 --- a/tool/lib/bundled_gem.rb +++ b/tool/lib/bundled_gem.rb @@ -12,14 +12,18 @@ module BundledGem "singleton", # prime "ipaddr", # rinda "forwardable", # prime, rinda - "strscan" # rexml + "strscan", # rexml + "psych" # rdoc ] module_function def unpack(file, *rest) pkg = Gem::Package.new(file) - prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)} + prepare_test(pkg.spec, *rest) do |dir| + pkg.extract_files(dir) + FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)}) + end puts "Unpacked #{file}" rescue Gem::Package::FormatError, Errno::ENOENT puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby." diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb index 62ada2bdca..e29a0e3c25 100644 --- a/tool/lib/core_assertions.rb +++ b/tool/lib/core_assertions.rb @@ -75,9 +75,18 @@ module Test require_relative 'envutil' require 'pp' begin - require '-test-/asan' + require '-test-/sanitizers' rescue LoadError + # in test-unit-ruby-core gem + def sanitizers + nil + end + else + def sanitizers + Test::Sanitizers + end end + module_function :sanitizers nil.pretty_inspect @@ -97,11 +106,14 @@ module Test end def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil, - success: nil, **opt) + success: nil, failed: nil, gems: false, **opt) args = Array(args).dup - args.insert((Hash === args[0] ? 1 : 0), '--disable=gems') + unless gems.nil? + args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems") + end stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt) - desc = FailDesc[status, message, stderr] + desc = failed[status, message, stderr] if failed + desc ||= FailDesc[status, message, stderr] if block_given? raise "test_stdout ignored, use block only or without block" if test_stdout != [] raise "test_stderr ignored, use block only or without block" if test_stderr != [] @@ -159,7 +171,7 @@ module Test pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled? # ASAN has the same problem - its shadow memory greatly increases memory usage # (plus asan has better ways to detect memory leaks than this assertion) - pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if defined?(Test::ASAN) && Test::ASAN.enabled? + pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled? require_relative 'memory_status' raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status) @@ -291,9 +303,34 @@ module Test def separated_runner(token, out = nil) include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) }) + out = out ? IO.new(out, 'w') : STDOUT + + # avoid method redefinitions + out_write = out.method(:write) + integer_to_s = Integer.instance_method(:to_s) + array_pack = Array.instance_method(:pack) + marshal_dump = Marshal.method(:dump) + assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set) + assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get) + Test::Unit::Assertions.module_eval do + @_assertions = 0 + + undef _assertions= + define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)}) + + undef _assertions + define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)}) + end + # assume Method#call and UnboundMethod#bind_call need to work as the original + at_exit { - out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}" + assertions = assertions_ivar_get.call(:@_assertions) + out_write.call <<~OUT + <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}> + #{array_pack.bind_call([marshal_dump.call($!)], 'm0')} + </error id="#{token}"> + OUT } if defined?(Test::Unit::Runner) Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true) @@ -327,7 +364,16 @@ eom args = args.dup args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"}) args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag + # power_assert 3 requires ruby 3.1 or later + args << "-W:no-experimental" if RUBY_VERSION < "3.1." stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt) + + if sanitizers&.lsan_enabled? + # LSAN may output messages like the following line into stderr. We should ignore it. + # ==276855==Running thread 276851 was not suspended. False leaks are possible. + # See https://github.com/google/sanitizers/issues/1479 + stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "") + end ensure if res_c res_c.close @@ -338,15 +384,16 @@ eom end raise if $! abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig)) + marshal_error = nil assert(!abort, FailDesc[status, nil, stderr]) - self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i - begin - res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m")) + res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do + self._assertions += $1.to_i + res = Marshal.load($2.unpack1("m")) or next rescue => marshal_error ignore_stderr = nil res = nil - end - if res and !(SystemExit === res) + else + next if SystemExit === res if bt = res.backtrace bt.each do |l| l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"} @@ -358,7 +405,7 @@ eom raise res end - # really is it succeed? + # really did it succeed? unless ignore_stderr # the body of assert_separately must not output anything to detect error assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr]) @@ -369,9 +416,17 @@ eom # Run Ractor-related test without influencing the main test suite def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt) - return unless defined?(Ractor) + omit unless defined?(Ractor) + + # https://bugs.ruby-lang.org/issues/21262 + shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value) + shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join) + + if require + require = [require] unless require.is_a?(Array) + require = require.map {|r| "require #{r.inspect}"}.join("\n") + end - require = "require #{require.inspect}" if require if require_relative dir = File.dirname(caller_locations[0,1][0].absolute_path) full_path = File.expand_path(require_relative, dir) @@ -379,6 +434,8 @@ eom end assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt) + #{shim_value} + #{shim_join} #{require} previous_verbose = $VERBOSE $VERBOSE = nil @@ -489,19 +546,15 @@ eom case expected when String assert = :assert_equal - when Regexp - assert = :assert_match else - raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}" + assert_respond_to(expected, :===) + assert = :assert_match end - ex = m = nil - EnvUtil.with_default_internal(expected.encoding) do - ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do - yield - end - m = ex.message + ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do + yield end + m = ex.message msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"} if assert == :assert_equal @@ -514,6 +567,43 @@ eom ex end + # :call-seq: + # assert_raise_kind_of(*args, &block) + # + #Tests if the given block raises one of the given exceptions or + #sub exceptions of the given exceptions. If the last argument + #is a String, it will be used as the error message. + # + # assert_raise do #Fails, no Exceptions are raised + # end + # + # assert_raise SystemCallErr do + # Dir.chdir(__FILE__) #Raises Errno::ENOTDIR, so assertion succeeds + # end + def assert_raise_kind_of(*exp, &b) + case exp.last + when String, Proc + msg = exp.pop + end + + begin + yield + rescue Test::Unit::PendedError => e + raise e unless exp.include? Test::Unit::PendedError + rescue *exp => e + pass + rescue Exception => e + flunk(message(msg) {"#{mu_pp(exp)} family exception expected, not #{mu_pp(e)}"}) + ensure + unless e + exp = exp.first if exp.size == 1 + + flunk(message(msg) {"#{mu_pp(exp)} family expected but nothing was raised"}) + end + end + e + end + TEST_DIR = File.join(__dir__, "test/unit") #:nodoc: # :call-seq: @@ -633,7 +723,7 @@ eom def assert_warning(pat, msg = nil) result = nil - stderr = EnvUtil.with_default_internal(pat.encoding) { + stderr = EnvUtil.with_default_internal(of: pat) { EnvUtil.verbose_warning { result = yield } @@ -647,17 +737,15 @@ eom assert_warning(*args) {$VERBOSE = false; yield} end - def assert_deprecated_warning(mesg = /deprecated/) + def assert_deprecated_warning(mesg = /deprecated/, &block) assert_warning(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end - def assert_deprecated_warn(mesg = /deprecated/) + def assert_deprecated_warn(mesg = /deprecated/, &block) assert_warn(mesg) do - Warning[:deprecated] = true if Warning.respond_to?(:[]=) - yield + EnvUtil.deprecation_warning(&block) end end @@ -794,6 +882,9 @@ eom rescue # Constants may be defined but not implemented, e.g., mingw. else + unless Process.clock_getres(clk) < 1.0e-03 + next # needs msec precision + end PERFORMANCE_CLOCK = clk end end diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb new file mode 100644 index 0000000000..56b420a546 --- /dev/null +++ b/tool/lib/dump.gdb @@ -0,0 +1,17 @@ +set height 0 +set width 0 +set confirm off + +echo \n>>> Threads\n\n +info threads + +echo \n>>> Machine level backtrace\n\n +thread apply all info stack full + +echo \n>>> Dump Ruby level backtrace (if possible)\n\n +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +echo ">>> Finish\n" +detach +quit diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb new file mode 100644 index 0000000000..ed9cb89010 --- /dev/null +++ b/tool/lib/dump.lldb @@ -0,0 +1,13 @@ +script print("\n>>> Threads\n\n") +thread list + +script print("\n>>> Machine level backtrace\n\n") +thread backtrace all + +script print("\n>>> Dump Ruby level backtrace (if possible)\n\n") +call rb_vmdebug_stack_dump_all_threads() +call fflush(stderr) + +script print(">>> Finish\n") +detach +quit diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb index 7b8aa99a39..6089605056 100644 --- a/tool/lib/envutil.rb +++ b/tool/lib/envutil.rb @@ -79,6 +79,72 @@ module EnvUtil end module_function :timeout + class Debugger + @list = [] + + attr_accessor :name + + def self.register(name, &block) + @list << new(name, &block) + end + + def initialize(name, &block) + @name = name + instance_eval(&block) + end + + def usable?; false; end + + def start(pid, *args) end + + def dump(pid, timeout: 60, reprieve: timeout&.div(4)) + dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err) + rescue Errno::ENOENT + return + else + return unless dpid + [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig| + begin + return EnvUtil.timeout(t) {Process.wait(dpid)} + rescue Timeout::Error + Process.kill(sig, dpid) + end + end + true + end + + # sudo -n: --non-interactive + PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM) + + def spawn(*args, **opts) + super(*PRECOMMAND, *args, **opts) + end + + register("gdb") do + class << self + def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end + def start(pid, *args, **opts) + spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts) + end + def command_file(file) "--command=#{file}"; end + end + end + + register("lldb") do + class << self + def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end + def start(pid, *args, **opts) + spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts) + end + def command_file(file) ["--source", file]; end + end + end + + def self.search + @debugger ||= @list.find(&:usable?) + end + end + def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1) reprieve = apply_timeout_scale(reprieve) if reprieve @@ -94,17 +160,12 @@ module EnvUtil pgroup = pid end - lldb = true if /darwin/ =~ RUBY_PLATFORM - + dumped = false while signal = signals.shift - if lldb and [:ABRT, :KILL].include?(signal) - lldb = false - # sudo -n: --non-interactive - # lldb -p: attach - # -o: run command - system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit]) - true + if !dumped and [:ABRT, :KILL].include?(signal) + Debugger.search&.dump(pid) + dumped = true end begin @@ -166,8 +227,8 @@ module EnvUtil args = [args] if args.kind_of?(String) # use the same parser as current ruby - if args.none? { |arg| arg.start_with?("--parser=") } - current_parser = RUBY_DESCRIPTION =~ /prism/i ? "prism" : "parse.y" + if (args.none? { |arg| arg.start_with?("--parser=") } and + /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read)) args = ["--parser=#{current_parser}"] + args end pid = spawn(child_env, *precommand, rubybin, *args, opt) @@ -217,6 +278,12 @@ module EnvUtil end module_function :invoke_ruby + def current_parser + features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}] + features&.split&.include?("+PRISM") ? "prism" : "parse.y" + end + module_function :current_parser + def verbose_warning class << (stderr = "".dup) alias write concat @@ -233,6 +300,21 @@ module EnvUtil end module_function :verbose_warning + if defined?(Warning.[]=) + def deprecation_warning + previous_deprecated = Warning[:deprecated] + Warning[:deprecated] = true + yield + ensure + Warning[:deprecated] = previous_deprecated + end + else + def deprecation_warning + yield + end + end + module_function :deprecation_warning + def default_warning $VERBOSE = false yield @@ -279,7 +361,8 @@ module EnvUtil end module_function :without_gc - def with_default_external(enc) + def with_default_external(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_external = enc } yield ensure @@ -287,7 +370,8 @@ module EnvUtil end module_function :with_default_external - def with_default_internal(enc) + def with_default_internal(enc = nil, of: nil) + enc = of.encoding if defined?(of.encoding) suppress_warning { Encoding.default_internal = enc } yield ensure diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb new file mode 100644 index 0000000000..1893e07657 --- /dev/null +++ b/tool/lib/gem_env.rb @@ -0,0 +1 @@ +ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__) diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb index 69aeb2c254..69df9a64b8 100644 --- a/tool/lib/leakchecker.rb +++ b/tool/lib/leakchecker.rb @@ -112,7 +112,7 @@ class LeakChecker } unless fd_leaked.empty? unless @@try_lsof == false - @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) + @@try_lsof |= system(*%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output) end end h.each {|fd, list| diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb index 60632523a8..429e5f6a1d 100644 --- a/tool/lib/memory_status.rb +++ b/tool/lib/memory_status.rb @@ -20,48 +20,68 @@ module Memory data.scan(pat) {|k, v| keys << k.downcase.intern} when /mswin|mingw/ =~ RUBY_PLATFORM - require 'fiddle/import' - require 'fiddle/types' - - module Win32 - extend Fiddle::Importer - dlload "kernel32.dll", "psapi.dll" - include Fiddle::Win32Types - typealias "SIZE_T", "size_t" - - PROCESS_MEMORY_COUNTERS = struct [ - "DWORD cb", - "DWORD PageFaultCount", - "SIZE_T PeakWorkingSetSize", - "SIZE_T WorkingSetSize", - "SIZE_T QuotaPeakPagedPoolUsage", - "SIZE_T QuotaPagedPoolUsage", - "SIZE_T QuotaPeakNonPagedPoolUsage", - "SIZE_T QuotaNonPagedPoolUsage", - "SIZE_T PagefileUsage", - "SIZE_T PeakPagefileUsage", - ] - - typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" - - extern "HANDLE GetCurrentProcess()", :stdcall - extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall - - module_function - def memory_info - size = PROCESS_MEMORY_COUNTERS.size - data = PROCESS_MEMORY_COUNTERS.malloc - data.cb = size - data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + keys.push(:size, :rss, :peak) + + begin + require 'fiddle/import' + require 'fiddle/types' + rescue LoadError + # Fallback to PowerShell command to get memory information for current process + def self.read_status + cmd = [ + "powershell.exe", "-NoProfile", "-Command", + "Get-Process -Id #{$$} | " \ + "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }" + ] + + IO.popen(cmd, "r", err: [:child, :out]) do |out| + if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read + yield :size, $1.to_i + yield :rss, $2.to_i + yield :peak, $3.to_i + end + end + end + else + module Win32 + extend Fiddle::Importer + dlload "kernel32.dll", "psapi.dll" + include Fiddle::Win32Types + typealias "SIZE_T", "size_t" + + PROCESS_MEMORY_COUNTERS = struct [ + "DWORD cb", + "DWORD PageFaultCount", + "SIZE_T PeakWorkingSetSize", + "SIZE_T WorkingSetSize", + "SIZE_T QuotaPeakPagedPoolUsage", + "SIZE_T QuotaPagedPoolUsage", + "SIZE_T QuotaPeakNonPagedPoolUsage", + "SIZE_T QuotaNonPagedPoolUsage", + "SIZE_T PagefileUsage", + "SIZE_T PeakPagefileUsage", + ] + + typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*" + + extern "HANDLE GetCurrentProcess()", :stdcall + extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall + + module_function + def memory_info + size = PROCESS_MEMORY_COUNTERS.size + data = PROCESS_MEMORY_COUNTERS.malloc + data.cb = size + data if GetProcessMemoryInfo(GetCurrentProcess(), data, size) + end end - end - keys.push(:size, :rss, :peak) - def self.read_status - if info = Win32.memory_info - yield :size, info.PagefileUsage - yield :rss, info.WorkingSetSize - yield :peak, info.PeakWorkingSetSize + def self.read_status + if info = Win32.memory_info + yield :size, info.PagefileUsage + yield :rss, info.WorkingSetSize + yield :peak, info.PeakWorkingSetSize + end end end when (require_relative 'find_executable' diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb new file mode 100644 index 0000000000..7b889163b0 --- /dev/null +++ b/tool/lib/test/jobserver.rb @@ -0,0 +1,47 @@ +module Test + module JobServer + end +end + +class << Test::JobServer + def connect(makeflags = ENV["MAKEFLAGS"]) + return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags + begin + if fifo = $3 + fifo.gsub!(/\\(?=.)/, '') + r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) + w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) + else + r = IO.for_fd($1.to_i(10), "rb", autoclose: false) + w = IO.for_fd($2.to_i(10), "wb", autoclose: false) + end + rescue + r&.close + nil + else + return r, w + end + end + + def acquire_possible(r, w, max) + return unless tokens = r.read_nonblock(max - 1, exception: false) + if (jobs = tokens.size) > 0 + jobserver, w = w, nil + at_exit do + jobserver.print(tokens) + jobserver.close + end + end + return jobs + 1 + rescue Errno::EBADF + ensure + r&.close + w&.close + end + + def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"]) + if max > 1 and (r, w = connect(makeflags)) + acquire_possible(r, w, max) + end + end +end diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb index 30f30df62e..2663b7b76a 100644 --- a/tool/lib/test/unit.rb +++ b/tool/lib/test/unit.rb @@ -19,6 +19,7 @@ require_relative '../envutil' require_relative '../colorize' require_relative '../leakchecker' require_relative '../test/unit/testcase' +require_relative '../test/jobserver' require 'optparse' # See Test::Unit @@ -249,6 +250,8 @@ module Test end module Parallel # :nodoc: all + attr_accessor :prefix + def process_args(args = []) return @options if @options options = super @@ -260,27 +263,8 @@ module Test def non_options(files, options) @jobserver = nil - makeflags = ENV.delete("MAKEFLAGS") - if !options[:parallel] and - /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags - begin - if fifo = $3 - fifo.gsub!(/\\(?=.)/, '') - r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY) - w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY) - else - r = IO.for_fd($1.to_i(10), "rb", autoclose: false) - w = IO.for_fd($2.to_i(10), "wb", autoclose: false) - end - rescue - r.close if r - nil - else - r.close_on_exec = true - w.close_on_exec = true - @jobserver = [r, w] - options[:parallel] ||= 256 # number of tokens to acquire first - end + if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS")) + options[:parallel] ||= 256 # number of tokens to acquire first end @worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200) super @@ -298,7 +282,7 @@ module Test opts.separator "parallel test options:" - options[:retry] = true + options[:retry] = false opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a| options[:testing] = true & t # For testing @@ -370,8 +354,12 @@ module Test @io.puts(*args) end - def run(task,type) - @file = File.basename(task, ".rb") + def run(task, type, base = nil) + if base + @file = task.delete_prefix(base).chomp(".rb") + else + @file = File.basename(task, ".rb") + end @real_file = task begin puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}" @@ -415,6 +403,7 @@ module Test end def kill + EnvUtil::Debugger.search&.dump(@pid) signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV Process.kill(signal, @pid) warn "worker #{to_s} does not respond; #{signal} is sent" @@ -597,7 +586,7 @@ module Test worker.quit worker = launch_worker end - worker.run(task, type) + worker.run(task, type, (@prefix unless @options[:job_status] == :replace)) @test_count += 1 jobs_status(worker) @@ -1292,10 +1281,15 @@ module Test parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n| options[:repeat_count] = n end + options[:keep_repeating] = false + parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n| + options[:keep_repeating] = true + end end def _run_anything(type) @repeat_count = @options[:repeat_count] + @keep_repeating = @options[:keep_repeating] super end end @@ -1617,7 +1611,7 @@ module Test [(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type, t, @test_count.fdiv(t), @assertion_count.fdiv(t)] end while @repeat_count && @@current_repeat_count < @repeat_count && - report.empty? && failures.zero? && errors.zero? + (@keep_repeating || report.empty? && failures.zero? && errors.zero?) output.sync = old_sync if sync @@ -1856,6 +1850,7 @@ module Test @force_standalone = force_standalone @runner = Runner.new do |files, options| base = options[:base_directory] ||= default_dir + @runner.prefix = base ? (base + "/") : nil files << default_dir if files.empty? and default_dir @to_run = files yield self if block_given? diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb index fe3351107e..19581fc3ab 100644 --- a/tool/lib/test/unit/assertions.rb +++ b/tool/lib/test/unit/assertions.rb @@ -798,33 +798,37 @@ EOT MIN_MEASURABLE = 1.0 / MIN_HZ def assert_cpu_usage_low(msg = nil, pct: 0.05, wait: 1.0, stop: nil) - require 'benchmark' - wait = EnvUtil.apply_timeout_scale(wait) if wait < 0.1 # TIME_QUANTUM_USEC in thread_pthread.c warn "test #{msg || 'assert_cpu_usage_low'} too short to be accurate" end - tms = Benchmark.measure(msg || '') do - if stop - th = Thread.start {sleep wait; stop.call} - yield - th.join - else - begin - Timeout.timeout(wait) {yield} - rescue Timeout::Error - end + + t0, r0 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + if stop + th = Thread.start {sleep wait; stop.call} + yield + th.join + else + begin + Timeout.timeout(wait) {yield} + rescue Timeout::Error end end - max = pct * tms.real + t1, r1 = Process.times, Process.clock_gettime(Process::CLOCK_MONOTONIC) + + total = t1.utime - t0.utime + t1.stime - t0.stime + t1.cutime - t0.cutime + t1.cstime - t0.cstime + real = r1 - r0 + + max = pct * real min_measurable = MIN_MEASURABLE min_measurable *= 1.30 # add a little (30%) to account for misc. overheads if max < min_measurable max = min_measurable end - assert_operator tms.total, :<=, max, msg + assert_operator total, :<=, max, msg end def assert_is_minus_zero(f) diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb index 3894f9c8e8..26c9763c13 100644 --- a/tool/lib/vcs.rb +++ b/tool/lib/vcs.rb @@ -51,23 +51,9 @@ module DebugPOpen end using DebugPOpen module DebugSystem - def system(*args) + def system(*args, exception: true, **opts) VCS.dump(args, "args: ") if $DEBUG - exception = false - opts = Hash.try_convert(args[-1]) - if RUBY_VERSION >= "2.6" - unless opts - opts = {} - args << opts - end - exception = opts.fetch(:exception) {opts[:exception] = true} - elsif opts - exception = opts.delete(:exception) {true} - args.pop if opts.empty? - end - ret = super(*args) - raise "Command failed with status (#$?): #{args[0]}" if exception and !ret - ret + super(*args, exception: exception, **opts) end end @@ -271,155 +257,6 @@ class VCS code end - class SVN < self - register(".svn") - COMMAND = ENV['SVN'] || 'svn' - - def self.revision_name(rev) - "r#{rev}" - end - - def _get_revisions(path, srcdir = nil) - if srcdir and self.class.local_path?(path) - path = File.join(srcdir, path) - end - if srcdir - info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}") - info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s - end - info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}") - _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/) - modified = info_xml[/<date>([^<>]*)/, 1] - branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1] - [Integer(last), Integer(changed), modified, branch] - end - - def self.search_root(path) - return unless local_path?(path) - parent = File.realpath(path) - begin - parent = File.dirname(wkdir = parent) - return wkdir if File.directory?(wkdir + "/.svn") - end until parent == wkdir - end - - def get_info - @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}") - end - - def url - @url ||= begin - url = get_info[/<root>(.*)<\/root>/, 1] - @url = URI.parse(url+"/") if url - end - end - - def wcroot - @wcroot ||= begin - info = get_info - @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1] - @wcroot ||= self.class.search_root(@srcdir) - end - end - - def branch(name) - return trunk if name == "trunk" - url + "branches/#{name}" - end - - def tag(name) - url + "tags/#{name}" - end - - def trunk - url + "trunk" - end - alias master trunk - - def branch_list(pat) - IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f| - f.each do |line| - line.chomp! - line.chomp!('/') - yield(line) if File.fnmatch?(pat, line) - end - end - end - - def grep(pat, tag, *files, &block) - cmd = %W"#{COMMAND} cat" - files.map! {|n| File.join(tag, n)} if tag - set = block.binding.eval("proc {|match| $~ = match}") - IO.popen([cmd, *files]) do |f| - f.grep(pat) do |s| - set[$~] - yield s - end - end - end - - def export(revision, url, dir, keep_temp = false) - if @srcdir and (rootdir = wcroot) - srcdir = File.realpath(@srcdir) - rootdir << "/" - if srcdir.start_with?(rootdir) - subdir = srcdir[rootdir.size..-1] - subdir = nil if subdir.empty? - FileUtils.mkdir_p(svndir = dir+"/.svn") - FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir) - system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false - FileUtils.rm_rf(svndir) unless keep_temp - if subdir - tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}") - File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}") - FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir) - begin - Dir.rmdir("#{dir}/#{subdir}") - end until (subdir = File.dirname(subdir)) == '.' - FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir) - Dir.rmdir(tmpdir) - end - return self - end - end - IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe| - pipe.each {|line| /^A/ =~ line or yield line} - end - self if $?.success? - end - - def after_export(dir) - super - FileUtils.rm_rf(dir+"/.svn") - end - - def branch_beginning(url) - # `--limit` of svn-log is useless in this case, because it is - # applied before `--search`. - rev = IO.pread(%W[ #{COMMAND} log --xml - --search=matz --search-and=has\ started - -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1] - rev.to_i if rev - end - - def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path) - range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':') - IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -r#{range} #{url}") do |r| - IO.copy_stream(r, path) - end - end - - def commit - args = %W"#{COMMAND} commit" - if dryrun? - VCS.dump(args, "commit: ") - return true - end - system(*args) - end - end - class GIT < self register(".git") do |path, dir| SAFE_DIRECTORIES ||= @@ -479,7 +316,7 @@ class VCS last = cmd_read_at(srcdir, [[*gitcmd, 'rev-parse', ref, err: w]]).rstrip w.close unless r.eof? - raise "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" + raise VCS::NotFoundError, "#{COMMAND} rev-parse failed\n#{r.read.gsub(/^(?=\s*\S)/, ' ')}" end end log = cmd_read_at(srcdir, [[*gitcmd, 'log', '-n1', '--date=iso', '--pretty=fuller', *path]]) @@ -533,15 +370,6 @@ class VCS rev[0, 10] end - def revision_handler(rev) - case rev - when Integer - SVN - else - super - end - end - def without_gitconfig envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h| h[v] = ENV.delete(v) @@ -616,32 +444,26 @@ class VCS def export(revision, url, dir, keep_temp = false) system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return - (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir)) + GIT.new(File.expand_path(dir)) end def branch_beginning(url) - cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H + year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i + cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25 --author=matz --committer=matz --grep=started\\.$ - #{url.to_str} -- version.h include/ruby/version.h]) + #{url} -- version.h include/ruby/version.h])[/.*/] end - def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil) - svn = nil + def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true) from, to = [from, to].map do |rev| rev or next - if Integer === rev - svn = true - rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'}, - %W"#{COMMAND} log -n1 --format=format:%H" << - "--grep=^ *git-svn-id: .*@#{rev} ") - end rev unless rev.empty? end - unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from)) + unless from&.match?(/./) or (from = branch_beginning(url))&.match?(/./) warn "no starting commit found", uplevel: 1 from = nil end - if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", + if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits", chdir: @srcdir, exception: false) system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix", chdir: @srcdir, exception: false) @@ -654,22 +476,18 @@ class VCS else arg = ["--since=25 Dec 00:00:00", to] end - writer = - if svn - format_changelog_as_svn(path, arg) - else - if base_url == true - remote, = upstream - if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) - remote.chomp! - # hack to redirect git.r-l.o to github - remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') - remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') - end - base_url = remote - end - format_changelog(path, arg, base_url) + if base_url == true + env = CHANGELOG_ENV + remote, = upstream + if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}]) + remote.chomp! + # hack to redirect git.r-l.o to github + remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/') + remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/') end + base_url = remote + end + writer = changelog_formatter(path, arg, base_url) if !path or path == '-' writer[$stdout] else @@ -678,9 +496,10 @@ class VCS end LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&' + CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} - def format_changelog(path, arg, base_url = nil) - env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'} + def changelog_formatter(path, arg, base_url = nil) + env = CHANGELOG_ENV cmd = %W[#{COMMAND} log --format=fuller --notes=commits --notes=log-fix --topo-order --no-merges --fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci] @@ -692,17 +511,32 @@ class VCS cmd << date cmd.concat(arg) proc do |w| - w.print "-*- coding: utf-8 -*-\n\n" - w.print "base-url = #{base_url}\n\n" if base_url + w.print "-*- coding: utf-8 -*-\n" + w.print "\n""base-url = #{base_url}\n" if base_url + + begin + ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true) + .grep_v(/^ *(?:#|$)/) + .to_h {|v| [v, true]} + ignore_revs = nil if ignore_revs.empty? + rescue Errno::ENOENT + end + cmd_pipe(env, cmd, chdir: @srcdir) do |r| - while s = r.gets("\ncommit ") + r.gets(sep = "commit ") + sep = "\n" + sep + while s = r.gets(sep, chomp: true) h, s = s.split(/^$/, 2) + if ignore_revs&.key?(h[/\A\h{40}/]) + next + end next if /^Author: *dependabot\[bot\]/ =~ h h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&') if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '') fix = $1 + next if /\A *skip\Z/ =~ fix s = s.lines fix.each_line do |x| next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '') @@ -739,7 +573,7 @@ class VCS next end end - message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] + message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"] from = [1, n-2].max to = [s.size-1, n+2].min s.each_with_index do |e, i| @@ -761,6 +595,10 @@ class VCS s = s.join('') end + s.gsub!(%r[(?!<\w)([-\w]+/[-\w]+)(?:@(\h{8,40})|#(\d{5,}))\b]) do + path = defined?($2) ? "commit/#{$2}" : "pull/#{$3}" + "[#$&](https://github.com/#{$1}/#{path})" + end if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s issue = "#{$1}pull/" s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue} @@ -768,28 +606,7 @@ class VCS s.gsub!(/ +\n/, "\n") s.sub!(/^Notes:/, ' \&') - w.print h, s - end - end - end - end - - def format_changelog_as_svn(path, arg) - cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B" - cmd.concat(arg) - proc do |w| - sep = "-"*72 + "\n" - w.print sep - cmd_pipe(cmd) do |r| - while s = r.gets("\0") - s.chomp!("\0") - author, time, s = s.split("\n", 3) - s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '') - rev = $1 - time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)") - lines = s.count("\n") + 1 - lines = "#{lines} line#{lines == 1 ? '' : 's'}" - w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep + w.print sep, h, s end end end @@ -826,46 +643,6 @@ class VCS end end - class GITSVN < GIT - def self.revision_name(rev) - SVN.revision_name(rev) - end - - def last_changed_revision - rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1] - com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp - return rev, com - end - - def commit(opts = {}) - rev, com = last_changed_revision - head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp - - commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n") - commits.each_with_index do |l, i| - r, a, c = l.split(' ') - dcommit = [COMMAND, "svn", "dcommit"] - dcommit.insert(-2, "-n") if dryrun? - dcommit << "--add-author-from" unless a == c - dcommit << r - system(*dcommit) or return false - system(COMMAND, "checkout", head) or return false - system(COMMAND, "rebase") or return false - end - - if rev - old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp] - old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1] - 3.times do - sleep 2 - system(*%W"#{COMMAND} pull --no-edit --rebase") - break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp) - end - end - true - end - end - class Null < self def get_revisions(path, srcdir = nil) @modified ||= Time.now - 10 |
