summaryrefslogtreecommitdiff
path: root/tool/lib
diff options
context:
space:
mode:
Diffstat (limited to 'tool/lib')
-rw-r--r--tool/lib/_tmpdir.rb121
-rw-r--r--tool/lib/bundle_env.rb4
-rw-r--r--tool/lib/bundled_gem.rb54
-rw-r--r--tool/lib/colorize.rb72
-rw-r--r--tool/lib/core_assertions.rb119
-rw-r--r--tool/lib/dump.gdb17
-rw-r--r--tool/lib/dump.lldb13
-rw-r--r--tool/lib/envutil.rb118
-rw-r--r--tool/lib/gem_env.rb3
-rw-r--r--tool/lib/leakchecker.rb36
-rw-r--r--tool/lib/memory_status.rb100
-rw-r--r--tool/lib/output.rb13
-rw-r--r--tool/lib/test/jobserver.rb47
-rw-r--r--tool/lib/test/unit.rb47
-rw-r--r--tool/lib/test/unit/assertions.rb10
-rw-r--r--tool/lib/vcs.rb357
16 files changed, 638 insertions, 493 deletions
diff --git a/tool/lib/_tmpdir.rb b/tool/lib/_tmpdir.rb
index fd429dab37..ac5b9be792 100644
--- a/tool/lib/_tmpdir.rb
+++ b/tool/lib/_tmpdir.rb
@@ -4,11 +4,11 @@ template = "rubytest."
# Assume the directory by these environment variables are safe.
base = [ENV["TMPDIR"], ENV["TMP"], "/tmp"].find do |tmp|
next unless tmp and tmp.size <= 50 and File.directory?(tmp)
- # On macOS, the default TMPDIR is very long, inspite of UNIX socket
- # path length is limited.
+ # On macOS, the default TMPDIR is very long, in spite of UNIX socket
+ # path length being limited.
#
# Also Rubygems creates its own temporary directory per tests, and
- # some tests copy the full path of gemhome there. In that caes, the
+ # some tests copy the full path of gemhome there. In that case, the
# path contains both temporary names twice, and can exceed path name
# limit very easily.
tmp
@@ -28,66 +28,71 @@ END {
Dir.rmdir(tmpdir)
rescue Errno::ENOENT
rescue Errno::ENOTEMPTY
- require_relative "colorize"
- colorize = Colorize.new
- ls = Struct.new(:colorize) do
- def mode_inspect(m, s)
- [
- (m & 0o4 == 0 ? ?- : ?r),
- (m & 0o2 == 0 ? ?- : ?w),
- (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)),
- ]
- end
- def decorate_path(path, st)
- case
- when st.directory?
- color = "bold;blue"
- type = "/"
- when st.symlink?
- color = "bold;cyan"
- # type = "@"
- when st.executable?
- color = "bold;green"
- type = "*"
- when path.end_with?(".gem")
- color = "green"
+ unless $no_report_tmpdir ||= nil
+ require_relative "colorize"
+ colorize = Colorize.new
+ ls = Struct.new(:colorize) do
+ def mode_inspect(m, s)
+ [
+ (m & 0o4 == 0 ? ?- : ?r),
+ (m & 0o2 == 0 ? ?- : ?w),
+ (m & 0o1 == 0 ? (s ? s.upcase : ?-) : (s || ?x)),
+ ]
end
- colorize.decorate(path, color) + (type || "")
- end
- def list_tree(parent, indent = "", &block)
- children = Dir.children(parent).map do |child|
- [child, path = File.join(parent, child), File.lstat(path)]
+ def decorate_path(path, st)
+ case
+ when st.directory?
+ color = "bold;blue"
+ type = "/"
+ when st.symlink?
+ color = "bold;cyan"
+ # type = "@"
+ when st.executable?
+ color = "bold;green"
+ type = "*"
+ when path.end_with?(".gem")
+ color = "green"
+ end
+ colorize.decorate(path, color) + (type || "")
end
- nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size
- size_width = children.map {|child, path, st| st.size}.max.to_s.size
+ def list_tree(parent, indent = "", &block)
+ children = Dir.children(parent).map do |child|
+ [child, path = File.join(parent, child), File.lstat(path)]
+ end
+ nlink_width = children.map {|child, path, st| st.nlink}.max.to_s.size
+ size_width = children.map {|child, path, st| st.size}.max.to_s.size
- children.each do |child, path, st|
- m = st.mode
- m = [
- (st.file? ? ?- : st.ftype[0]),
- mode_inspect(m >> 6, (?s unless m & 04000 == 0)),
- mode_inspect(m >> 3, (?s unless m & 02000 == 0)),
- mode_inspect(m, (?t unless m & 01000 == 0)),
- ].join("")
- warn sprintf("%s* %s %*d %*d %s % s%s",
- indent, m, nlink_width, st.nlink, size_width, st.size,
- st.mtime.to_s, decorate_path(child, st),
- (" -> " + decorate_path(File.readlink(path), File.stat(path)) if
- st.symlink?))
- if st.directory?
- list_tree(File.join(parent, child), indent + " ", &block)
+ children.each do |child, path, st|
+ m = st.mode
+ m = [
+ (st.file? ? ?- : st.ftype[0]),
+ mode_inspect(m >> 6, (?s unless m & 04000 == 0)),
+ mode_inspect(m >> 3, (?s unless m & 02000 == 0)),
+ mode_inspect(m, (?t unless m & 01000 == 0)),
+ ].join("")
+ warn sprintf("%s* %s %*d %*d %s % s%s",
+ indent, m, nlink_width, st.nlink, size_width, st.size,
+ st.mtime.to_s, decorate_path(child, st),
+ (" -> " + decorate_path(File.readlink(path), File.stat(path)) if
+ st.symlink?))
+ if st.directory?
+ list_tree(File.join(parent, child), indent + " ", &block)
+ end
+ yield path, st if block
end
- yield path, st if block
end
- end
- end.new(colorize)
- warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":"
- Dir.chdir(tmpdir) do
- ls.list_tree(".") do |path, st|
- if st.directory?
- Dir.rmdir(path)
- else
- File.unlink(path)
+ end.new(colorize)
+ warn colorize.notice("Children under ")+colorize.fail(tmpdir)+":"
+ Dir.chdir(tmpdir) do
+ ls.list_tree(".") do |path, st|
+ if st.directory?
+ Dir.rmdir(path)
+ else
+ File.unlink(path)
+ end
+ rescue Errno::EACCES
+ # On Windows, a killed process may still hold file locks briefly.
+ # Ignore and let FileUtils.rm_rf handle it below.
end
end
end
diff --git a/tool/lib/bundle_env.rb b/tool/lib/bundle_env.rb
new file mode 100644
index 0000000000..9ad5ea220b
--- /dev/null
+++ b/tool/lib/bundle_env.rb
@@ -0,0 +1,4 @@
+ENV["GEM_HOME"] = File.expand_path("../../.bundle", __dir__)
+ENV["BUNDLE_APP_CONFIG"] = File.expand_path("../../.bundle", __dir__)
+ENV["BUNDLE_PATH__SYSTEM"] = "true"
+ENV["BUNDLE_WITHOUT"] = "lint doc"
diff --git a/tool/lib/bundled_gem.rb b/tool/lib/bundled_gem.rb
index 45e41ac648..ad103825bc 100644
--- a/tool/lib/bundled_gem.rb
+++ b/tool/lib/bundled_gem.rb
@@ -16,11 +16,21 @@ module BundledGem
"psych" # rdoc
]
+ def self.command(gem, cmd)
+ if stub = Gem::Specification.latest_spec_for(gem)
+ spec = stub.spec
+ File.join(spec.gem_dir, spec.bindir, cmd)
+ end
+ end
+
module_function
def unpack(file, *rest)
pkg = Gem::Package.new(file)
- prepare_test(pkg.spec, *rest) {|dir| pkg.extract_files(dir)}
+ prepare_test(pkg.spec, *rest) do |dir|
+ pkg.extract_files(dir)
+ FileUtils.rm_rf(Dir.glob(".git*", base: dir).map {|n| File.join(dir, n)})
+ end
puts "Unpacked #{file}"
rescue Gem::Package::FormatError, Errno::ENOENT
puts "Try with hash version of bundled gems instead of #{file}. We don't use this gem with release version of Ruby."
@@ -120,4 +130,46 @@ module BundledGem
command = "#{git} checkout --detach #{rev}"
system(command, chdir: gemdir) or raise "failed: #{command}"
end
+
+ class GemspecLoader
+ module NoPipe
+ refine IO.singleton_class do
+ def popen(...) ""; end
+ end
+ end
+ using NoPipe
+
+ def `(command) ""; end
+
+ def load_gemspec(file)
+ code = File.read(file, encoding: "utf-8:-")
+ eval(code, binding, file)
+ rescue
+ nil
+ end
+ end
+
+ def load_gemspec(g)
+ spec = GemspecLoader.new.load_gemspec(g)
+ spec.files.clear
+ spec.extensions.clear
+ src = spec.to_ruby
+ src.sub!(/^$$/) {
+ %[# default: #{g} #{File.mtime(g).strftime(%[%s.%N])}\n]
+ }
+ return spec.full_name+'.gemspec', src
+ end
+
+ def update_default_gemspecs(basedirs, out, quiet: true)
+ basedirs.each do |basedir|
+ Dir.glob(basedir+'/**/*.gemspec') do |g|
+ name, src = BundledGem.load_gemspec(g)
+ unless src
+ puts "Ignoring #{g}" unless quiet
+ next
+ end
+ out.write(src, name: name, newer: File.mtime(g), quiet: quiet)
+ end
+ end
+ end
end
diff --git a/tool/lib/colorize.rb b/tool/lib/colorize.rb
index 0904312119..89da90e075 100644
--- a/tool/lib/colorize.rb
+++ b/tool/lib/colorize.rb
@@ -1,57 +1,78 @@
# frozen-string-literal: true
+# Decorate TTY output using ANSI Select Graphic Rendition control
+# sequences.
class Colorize
# call-seq:
# Colorize.new(colorize = nil)
# Colorize.new(color: color, colors_file: colors_file)
- def initialize(color = nil, opts = ((_, color = color, nil)[0] if Hash === color))
- @colors = @reset = nil
- @color = opts && opts[:color] || color
+ #
+ # Creates and load color settings.
+ def initialize(_color = nil, color: _color, colors_file: nil)
+ @colors = nil
+ @color = color
if color or (color == nil && coloring?)
- if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", :err => IO::NULL, &:read)} rescue nil)
+ if (%w[smso so].any? {|attr| /\A\e\[.*m\z/ =~ IO.popen("tput #{attr}", "r", err: IO::NULL, &:read)} rescue nil)
@beg = "\e["
- colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(/(\w+)=([^:\n]*)/)] : {}
- if opts and colors_file = opts[:colors_file]
+ colors = (colors = ENV['TEST_COLORS']) ? Hash[colors.scan(COLORS_PATTERN)] : {}
+ if colors_file
begin
- File.read(colors_file).scan(/(\w+)=([^:\n]*)/) do |n, c|
+ File.read(colors_file).scan(COLORS_PATTERN) do |n, c|
colors[n] ||= c
end
rescue Errno::ENOENT
end
end
@colors = colors
- @reset = "#{@beg}m"
end
end
self
end
+ COLORS_PATTERN = /(\w+)=([^:\n]*)/
+ private_constant :COLORS_PATTERN
+
DEFAULTS = {
# color names
"black"=>"30", "red"=>"31", "green"=>"32", "yellow"=>"33",
"blue"=>"34", "magenta"=>"35", "cyan"=>"36", "white"=>"37",
- "bold"=>"1", "underline"=>"4", "reverse"=>"7",
+ "bold"=>"1", "faint"=>"2", "underline"=>"4", "reverse"=>"7",
"bright_black"=>"90", "bright_red"=>"91", "bright_green"=>"92", "bright_yellow"=>"93",
"bright_blue"=>"94", "bright_magenta"=>"95", "bright_cyan"=>"96", "bright_white"=>"97",
+ "bg_black"=>"40", "bg_red"=>"41", "bg_green"=>"42", "bg_yellow"=>"43",
+ "bg_blue"=>"44", "bg_magenta"=>"45", "bg_cyan"=>"46", "bg_white"=>"47",
+ "bg_bright_black"=>"100", "bg_bright_red"=>"101",
+ "bg_bright_green"=>"102", "bg_bright_yellow"=>"103",
+ "bg_bright_blue"=>"104", "bg_bright_magenta"=>"105",
+ "bg_bright_cyan"=>"106", "bg_bright_white"=>"107",
# abstract decorations
"pass"=>"green", "fail"=>"red;bold", "skip"=>"yellow;bold",
"note"=>"bright_yellow", "notice"=>"bright_yellow", "info"=>"bright_magenta",
- }
-
- def coloring?
- STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?)
- end
+ }.freeze
+ private_constant :DEFAULTS
# colorize.decorate(str, name = color_name)
def decorate(str, name = @color)
if coloring? and color = resolve_color(name)
- "#{@beg}#{color}m#{str}#{@reset}"
+ "#{@beg}#{color}m#{str}#{reset_color(color)}"
else
str
end
end
+ DEFAULTS.each_key do |name|
+ define_method(name) {|str|
+ decorate(str, name)
+ }
+ end
+
+ private
+
+ def coloring?
+ STDOUT.tty? && (!(nc = ENV['NO_COLOR']) || nc.empty?)
+ end
+
def resolve_color(color = @color, seen = {}, colors = nil)
return unless @colors
color.to_s.gsub(/\b[a-z][\w ]+/) do |n|
@@ -69,10 +90,23 @@ class Colorize
end
end
- DEFAULTS.each_key do |name|
- define_method(name) {|str|
- decorate(str, name)
- }
+ def reset_color(colors)
+ resets = []
+ colors.scan(/\G;*\K(?:[34]8;(?:5;\d+|2(?:;\d+){3})|\d+)/) do |c|
+ case c
+ when '1', '2'
+ resets << '22'
+ when '4'
+ resets << '24'
+ when '7'
+ resets << '27'
+ when /\A[39]\d(?:;|\z)/
+ resets << '39'
+ when /\A(?:4|10)\d(?:;|\z)/
+ resets << '49'
+ end
+ end
+ "#{@beg}#{resets.reverse.join(';')}m"
end
end
diff --git a/tool/lib/core_assertions.rb b/tool/lib/core_assertions.rb
index 66822b4e3b..5ca318a598 100644
--- a/tool/lib/core_assertions.rb
+++ b/tool/lib/core_assertions.rb
@@ -75,9 +75,18 @@ module Test
require_relative 'envutil'
require 'pp'
begin
- require '-test-/asan'
+ require '-test-/sanitizers'
rescue LoadError
+ # in test-unit-ruby-core gem
+ def sanitizers
+ nil
+ end
+ else
+ def sanitizers
+ Test::Sanitizers
+ end
end
+ module_function :sanitizers
nil.pretty_inspect
@@ -97,11 +106,14 @@ module Test
end
def assert_in_out_err(args, test_stdin = "", test_stdout = [], test_stderr = [], message = nil,
- success: nil, **opt)
+ success: nil, failed: nil, gems: false, **opt)
args = Array(args).dup
- args.insert((Hash === args[0] ? 1 : 0), '--disable=gems')
+ unless gems.nil?
+ args.insert((Hash === args[0] ? 1 : 0), "--#{gems ? 'enable' : 'disable'}=gems")
+ end
stdout, stderr, status = EnvUtil.invoke_ruby(args, test_stdin, true, true, **opt)
- desc = FailDesc[status, message, stderr]
+ desc = failed[status, message, stderr] if failed
+ desc ||= FailDesc[status, message, stderr]
if block_given?
raise "test_stdout ignored, use block only or without block" if test_stdout != []
raise "test_stderr ignored, use block only or without block" if test_stderr != []
@@ -159,7 +171,7 @@ module Test
pend 'assert_no_memory_leak may consider MJIT memory usage as leak' if defined?(RubyVM::MJIT) && RubyVM::MJIT.enabled?
# ASAN has the same problem - its shadow memory greatly increases memory usage
# (plus asan has better ways to detect memory leaks than this assertion)
- pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if defined?(Test::ASAN) && Test::ASAN.enabled?
+ pend 'assert_no_memory_leak may consider ASAN memory usage as leak' if sanitizers&.asan_enabled?
require_relative 'memory_status'
raise Test::Unit::PendedError, "unsupported platform" unless defined?(Memory::Status)
@@ -291,9 +303,34 @@ module Test
def separated_runner(token, out = nil)
include(*Test::Unit::TestCase.ancestors.select {|c| !c.is_a?(Class) })
+
out = out ? IO.new(out, 'w') : STDOUT
+
+ # avoid method redefinitions
+ out_write = out.method(:write)
+ integer_to_s = Integer.instance_method(:to_s)
+ array_pack = Array.instance_method(:pack)
+ marshal_dump = Marshal.method(:dump)
+ assertions_ivar_set = Test::Unit::Assertions.method(:instance_variable_set)
+ assertions_ivar_get = Test::Unit::Assertions.method(:instance_variable_get)
+ Test::Unit::Assertions.module_eval do
+ @_assertions = 0
+
+ undef _assertions=
+ define_method(:_assertions=, ->(n) {assertions_ivar_set.call(:@_assertions, n)})
+
+ undef _assertions
+ define_method(:_assertions, -> {assertions_ivar_get.call(:@_assertions)})
+ end
+ # assume Method#call and UnboundMethod#bind_call need to work as the original
+
at_exit {
- out.puts "#{token}<error>", [Marshal.dump($!)].pack('m'), "#{token}</error>", "#{token}assertions=#{self._assertions}"
+ assertions = assertions_ivar_get.call(:@_assertions)
+ out_write.call <<~OUT
+ <error id="#{token}" assertions=#{integer_to_s.bind_call(assertions)}>
+ #{array_pack.bind_call([marshal_dump.call($!)], 'm0')}
+ </error id="#{token}">
+ OUT
}
if defined?(Test::Unit::Runner)
Test::Unit::Runner.class_variable_set(:@@stop_auto_run, true)
@@ -327,7 +364,16 @@ eom
args = args.dup
args.insert((Hash === args.first ? 1 : 0), "-w", "--disable=gems", *$:.map {|l| "-I#{l}"})
args << "--debug" if RUBY_ENGINE == 'jruby' # warning: tracing (e.g. set_trace_func) will not capture all events without --debug flag
+ # power_assert 3 requires ruby 3.1 or later
+ args << "-W:no-experimental" if RUBY_VERSION < "3.1."
stdout, stderr, status = EnvUtil.invoke_ruby(args, src, capture_stdout, true, **opt)
+
+ if sanitizers&.lsan_enabled?
+ # LSAN may output messages like the following line into stderr. We should ignore it.
+ # ==276855==Running thread 276851 was not suspended. False leaks are possible.
+ # See https://github.com/google/sanitizers/issues/1479
+ stderr.gsub!(/==\d+==Running thread \d+ was not suspended\. False leaks are possible\.\n/, "")
+ end
ensure
if res_c
res_c.close
@@ -338,15 +384,16 @@ eom
end
raise if $!
abort = status.coredump? || (status.signaled? && ABORT_SIGNALS.include?(status.termsig))
+ marshal_error = nil
assert(!abort, FailDesc[status, nil, stderr])
- self._assertions += res[/^#{token_re}assertions=(\d+)/, 1].to_i
- begin
- res = Marshal.load(res[/^#{token_re}<error>\n\K.*\n(?=#{token_re}<\/error>$)/m].unpack1("m"))
+ res.scan(/^<error id="#{token_re}" assertions=(\d+)>\n(.*?)\n(?=<\/error id="#{token_re}">$)/m) do
+ self._assertions += $1.to_i
+ res = Marshal.load($2.unpack1("m")) or next
rescue => marshal_error
ignore_stderr = nil
res = nil
- end
- if res and !(SystemExit === res)
+ else
+ next if SystemExit === res
if bt = res.backtrace
bt.each do |l|
l.sub!(/\A-:(\d+)/){"#{file}:#{line + $1.to_i}"}
@@ -358,7 +405,7 @@ eom
raise res
end
- # really is it succeed?
+ # really did it succeed?
unless ignore_stderr
# the body of assert_separately must not output anything to detect error
assert(stderr.empty?, FailDesc[status, "assert_separately failed with error message", stderr])
@@ -369,9 +416,17 @@ eom
# Run Ractor-related test without influencing the main test suite
def assert_ractor(src, args: [], require: nil, require_relative: nil, file: nil, line: nil, ignore_stderr: nil, **opt)
- return unless defined?(Ractor)
+ omit unless defined?(Ractor)
+
+ # https://bugs.ruby-lang.org/issues/21262
+ shim_value = "class Ractor; alias value take; end" unless Ractor.method_defined?(:value)
+ shim_join = "class Ractor; alias join take; end" unless Ractor.method_defined?(:join)
+
+ if require
+ require = [require] unless require.is_a?(Array)
+ require = require.map {|r| "require #{r.inspect}"}.join("\n")
+ end
- require = "require #{require.inspect}" if require
if require_relative
dir = File.dirname(caller_locations[0,1][0].absolute_path)
full_path = File.expand_path(require_relative, dir)
@@ -379,6 +434,8 @@ eom
end
assert_separately(args, file, line, <<~RUBY, ignore_stderr: ignore_stderr, **opt)
+ #{shim_value}
+ #{shim_join}
#{require}
previous_verbose = $VERBOSE
$VERBOSE = nil
@@ -489,19 +546,15 @@ eom
case expected
when String
assert = :assert_equal
- when Regexp
- assert = :assert_match
else
- raise TypeError, "Expected #{expected.inspect} to be a kind of String or Regexp, not #{expected.class}"
+ assert_respond_to(expected, :===)
+ assert = :assert_match
end
- ex = m = nil
- EnvUtil.with_default_internal(expected.encoding) do
- ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do
- yield
- end
- m = ex.message
+ ex = assert_raise(exception, msg || proc {"Exception(#{exception}) with message matches to #{expected.inspect}"}) do
+ yield
end
+ m = ex.message
msg = message(msg, "") {"Expected Exception(#{exception}) was raised, but the message doesn't match"}
if assert == :assert_equal
@@ -670,7 +723,7 @@ eom
def assert_warning(pat, msg = nil)
result = nil
- stderr = EnvUtil.with_default_internal(pat.encoding) {
+ stderr = EnvUtil.with_default_internal(of: pat) {
EnvUtil.verbose_warning {
result = yield
}
@@ -684,17 +737,15 @@ eom
assert_warning(*args) {$VERBOSE = false; yield}
end
- def assert_deprecated_warning(mesg = /deprecated/)
+ def assert_deprecated_warning(mesg = /deprecated/, &block)
assert_warning(mesg) do
- Warning[:deprecated] = true if Warning.respond_to?(:[]=)
- yield
+ EnvUtil.deprecation_warning(&block)
end
end
- def assert_deprecated_warn(mesg = /deprecated/)
+ def assert_deprecated_warn(mesg = /deprecated/, &block)
assert_warn(mesg) do
- Warning[:deprecated] = true if Warning.respond_to?(:[]=)
- yield
+ EnvUtil.deprecation_warning(&block)
end
end
@@ -831,6 +882,9 @@ eom
rescue
# Constants may be defined but not implemented, e.g., mingw.
else
+ unless Process.clock_getres(clk) < 1.0e-03
+ next # needs msec precision
+ end
PERFORMANCE_CLOCK = clk
end
end
@@ -857,10 +911,11 @@ eom
first = seq.first
*arg = pre.call(first)
- times = (0..(rehearsal || (2 * first))).map do
+ raw_times = (0..(rehearsal || (2 * first))).map do
measure[arg, "rehearsal"].nonzero?
end
- times.compact!
+ times = raw_times.compact
+ raise "all measurements are zero: #{raw_times.inspect}" if times.empty?
tmin, tmax = times.minmax
# safe_factor * tmax * rehearsal_time_variance_factor(equals to 1 when variance is small)
diff --git a/tool/lib/dump.gdb b/tool/lib/dump.gdb
new file mode 100644
index 0000000000..56b420a546
--- /dev/null
+++ b/tool/lib/dump.gdb
@@ -0,0 +1,17 @@
+set height 0
+set width 0
+set confirm off
+
+echo \n>>> Threads\n\n
+info threads
+
+echo \n>>> Machine level backtrace\n\n
+thread apply all info stack full
+
+echo \n>>> Dump Ruby level backtrace (if possible)\n\n
+call rb_vmdebug_stack_dump_all_threads()
+call fflush(stderr)
+
+echo ">>> Finish\n"
+detach
+quit
diff --git a/tool/lib/dump.lldb b/tool/lib/dump.lldb
new file mode 100644
index 0000000000..ed9cb89010
--- /dev/null
+++ b/tool/lib/dump.lldb
@@ -0,0 +1,13 @@
+script print("\n>>> Threads\n\n")
+thread list
+
+script print("\n>>> Machine level backtrace\n\n")
+thread backtrace all
+
+script print("\n>>> Dump Ruby level backtrace (if possible)\n\n")
+call rb_vmdebug_stack_dump_all_threads()
+call fflush(stderr)
+
+script print(">>> Finish\n")
+detach
+quit
diff --git a/tool/lib/envutil.rb b/tool/lib/envutil.rb
index 7b8aa99a39..b4c7d1d035 100644
--- a/tool/lib/envutil.rb
+++ b/tool/lib/envutil.rb
@@ -63,6 +63,14 @@ module EnvUtil
end
end
+ if RUBY_ENGINE == "truffleruby"
+ # Tests relying on timeout have high variance on TruffleRuby due to the highly-optimizing JIT, deoptimization, profiling interpreter, different GC, etc.
+ # Setting a default timeout scale helps avoid transient failures for tests relying on timeouts.
+ # We choose 10 because it is the same number used in CRuby CI on macOS:
+ # https://github.com/ruby/ruby/blob/9d46b0c735877f152a0b4b16b8153c6f395dee28/.github/workflows/macos.yml#L133
+ self.timeout_scale = 10
+ end
+
def apply_timeout_scale(t)
if scale = EnvUtil.timeout_scale
t * scale
@@ -79,6 +87,72 @@ module EnvUtil
end
module_function :timeout
+ class Debugger
+ @list = []
+
+ attr_accessor :name
+
+ def self.register(name, &block)
+ @list << new(name, &block)
+ end
+
+ def initialize(name, &block)
+ @name = name
+ instance_eval(&block)
+ end
+
+ def usable?; false; end
+
+ def start(pid, *args) end
+
+ def dump(pid, timeout: 60, reprieve: timeout&.div(4))
+ dpid = start(pid, *command_file(File.join(__dir__, "dump.#{name}")), out: :err)
+ rescue Errno::ENOENT
+ return
+ else
+ return unless dpid
+ [[timeout, :TERM], [reprieve, :KILL]].find do |t, sig|
+ begin
+ return EnvUtil.timeout(t) {Process.wait(dpid)}
+ rescue Timeout::Error
+ Process.kill(sig, dpid)
+ end
+ end
+ true
+ end
+
+ # sudo -n: --non-interactive
+ PRECOMMAND = (%[sudo -n] if /darwin/ =~ RUBY_PLATFORM)
+
+ def spawn(*args, **opts)
+ super(*PRECOMMAND, *args, **opts)
+ end
+
+ register("gdb") do
+ class << self
+ def usable?; system(*%w[gdb --batch --quiet --nx -ex exit]); end
+ def start(pid, *args, **opts)
+ spawn(*%W[gdb --batch --quiet --pid #{pid}], *args, **opts)
+ end
+ def command_file(file) "--command=#{file}"; end
+ end
+ end
+
+ register("lldb") do
+ class << self
+ def usable?; system(*%w[lldb -Q --no-lldbinit -o exit]); end
+ def start(pid, *args, **opts)
+ spawn(*%W[lldb --batch -Q --attach-pid #{pid}], *args, **opts)
+ end
+ def command_file(file) ["--source", file]; end
+ end
+ end
+
+ def self.search
+ @debugger ||= @list.find(&:usable?)
+ end
+ end
+
def terminate(pid, signal = :TERM, pgroup = nil, reprieve = 1)
reprieve = apply_timeout_scale(reprieve) if reprieve
@@ -94,17 +168,12 @@ module EnvUtil
pgroup = pid
end
- lldb = true if /darwin/ =~ RUBY_PLATFORM
-
+ dumped = false
while signal = signals.shift
- if lldb and [:ABRT, :KILL].include?(signal)
- lldb = false
- # sudo -n: --non-interactive
- # lldb -p: attach
- # -o: run command
- system(*%W[sudo -n lldb -p #{pid} --batch -o bt\ all -o call\ rb_vmdebug_stack_dump_all_threads() -o quit])
- true
+ if !dumped and [:ABRT, :KILL].include?(signal)
+ Debugger.search&.dump(pid)
+ dumped = true
end
begin
@@ -166,8 +235,8 @@ module EnvUtil
args = [args] if args.kind_of?(String)
# use the same parser as current ruby
- if args.none? { |arg| arg.start_with?("--parser=") }
- current_parser = RUBY_DESCRIPTION =~ /prism/i ? "prism" : "parse.y"
+ if (args.none? { |arg| arg.start_with?("--parser=") } and
+ /^ +--parser=/ =~ IO.popen([rubybin, "--help"], &:read))
args = ["--parser=#{current_parser}"] + args
end
pid = spawn(child_env, *precommand, rubybin, *args, opt)
@@ -217,6 +286,12 @@ module EnvUtil
end
module_function :invoke_ruby
+ def current_parser
+ features = RUBY_DESCRIPTION[%r{\)\K [-+*/%._0-9a-zA-Z\[\] ]*(?=\[[-+*/%._0-9a-zA-Z]+\]\z)}]
+ features&.split&.include?("+PRISM") ? "prism" : "parse.y"
+ end
+ module_function :current_parser
+
def verbose_warning
class << (stderr = "".dup)
alias write concat
@@ -233,6 +308,21 @@ module EnvUtil
end
module_function :verbose_warning
+ if defined?(Warning.[]=)
+ def deprecation_warning
+ previous_deprecated = Warning[:deprecated]
+ Warning[:deprecated] = true
+ yield
+ ensure
+ Warning[:deprecated] = previous_deprecated
+ end
+ else
+ def deprecation_warning
+ yield
+ end
+ end
+ module_function :deprecation_warning
+
def default_warning
$VERBOSE = false
yield
@@ -279,7 +369,8 @@ module EnvUtil
end
module_function :without_gc
- def with_default_external(enc)
+ def with_default_external(enc = nil, of: nil)
+ enc = of.encoding if defined?(of.encoding)
suppress_warning { Encoding.default_external = enc }
yield
ensure
@@ -287,7 +378,8 @@ module EnvUtil
end
module_function :with_default_external
- def with_default_internal(enc)
+ def with_default_internal(enc = nil, of: nil)
+ enc = of.encoding if defined?(of.encoding)
suppress_warning { Encoding.default_internal = enc }
yield
ensure
diff --git a/tool/lib/gem_env.rb b/tool/lib/gem_env.rb
index 70a2469db2..1893e07657 100644
--- a/tool/lib/gem_env.rb
+++ b/tool/lib/gem_env.rb
@@ -1,2 +1 @@
-ENV['GEM_HOME'] = gem_home = File.expand_path('.bundle')
-ENV['GEM_PATH'] = [gem_home, File.expand_path('../../../.bundle', __FILE__)].uniq.join(File::PATH_SEPARATOR)
+ENV['GEM_HOME'] = File.expand_path('../../.bundle', __dir__)
diff --git a/tool/lib/leakchecker.rb b/tool/lib/leakchecker.rb
index 69aeb2c254..33a546699f 100644
--- a/tool/lib/leakchecker.rb
+++ b/tool/lib/leakchecker.rb
@@ -77,6 +77,7 @@ class LeakChecker
end
(h[fd] ||= []) << [io, autoclose, inspect]
}
+ inspect = {}
fd_leaked.select! {|fd|
str = ''.dup
pos = nil
@@ -98,6 +99,7 @@ class LeakChecker
s = io.stat
rescue Errno::EBADF
# something un-stat-able
+ live2.delete(fd)
next
else
next if /darwin/ =~ RUBY_PLATFORM and [0, -1].include?(s.dev)
@@ -106,15 +108,41 @@ class LeakChecker
io&.close
end
end
- puts "Leaked file descriptor: #{test_name}: #{fd}#{str}"
- puts " The IO was created at #{pos}" if pos
+ inspect[fd] = [str, pos]
true
}
unless fd_leaked.empty?
unless @@try_lsof == false
- @@try_lsof |= system(*%W[lsof -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], out: Test::Unit::Runner.output)
+ begin
+ open_list = IO.popen(%W[lsof -w -a -d #{fd_leaked.minmax.uniq.join("-")} -p #$$], &:readlines)
+ rescue
+ @@try_lsof = false
+ else
+ @@try_lsof |= $?.success?
+ end
+ if header = open_list&.shift
+ columns = header.split
+ fd_index, node_index = columns.index('FD'), columns.index('NODE')
+ open_list.reject! do |of|
+ of = of.chomp.split(' ', node_index + 2)
+ if of[node_index] == 'TCP' and of.last.end_with?('(CLOSE_WAIT)')
+ fd = of[fd_index].to_i
+ inspect.delete(fd)
+ h.delete(fd)
+ live2.delete(fd)
+ true
+ else
+ false
+ end
+ end
+ puts(header, open_list) unless open_list.empty?
+ end
end
end
+ inspect.each {|fd, (str, pos)|
+ puts "Leaked file descriptor: #{test_name}: #{fd}#{str}"
+ puts " The IO was created at #{pos}" if pos
+ }
h.each {|fd, list|
next if list.length <= 1
if 1 < list.count {|io, autoclose, inspect| autoclose }
@@ -156,7 +184,7 @@ class LeakChecker
[prev_count, []]
else
tempfiles = ObjectSpace.each_object(Tempfile).reject {|t|
- t.instance_variables.empty? || t.closed?
+ t.instance_variables.empty? || (t.closed? rescue true)
}
[count, tempfiles]
end
diff --git a/tool/lib/memory_status.rb b/tool/lib/memory_status.rb
index 60632523a8..429e5f6a1d 100644
--- a/tool/lib/memory_status.rb
+++ b/tool/lib/memory_status.rb
@@ -20,48 +20,68 @@ module Memory
data.scan(pat) {|k, v| keys << k.downcase.intern}
when /mswin|mingw/ =~ RUBY_PLATFORM
- require 'fiddle/import'
- require 'fiddle/types'
-
- module Win32
- extend Fiddle::Importer
- dlload "kernel32.dll", "psapi.dll"
- include Fiddle::Win32Types
- typealias "SIZE_T", "size_t"
-
- PROCESS_MEMORY_COUNTERS = struct [
- "DWORD cb",
- "DWORD PageFaultCount",
- "SIZE_T PeakWorkingSetSize",
- "SIZE_T WorkingSetSize",
- "SIZE_T QuotaPeakPagedPoolUsage",
- "SIZE_T QuotaPagedPoolUsage",
- "SIZE_T QuotaPeakNonPagedPoolUsage",
- "SIZE_T QuotaNonPagedPoolUsage",
- "SIZE_T PagefileUsage",
- "SIZE_T PeakPagefileUsage",
- ]
-
- typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*"
-
- extern "HANDLE GetCurrentProcess()", :stdcall
- extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall
-
- module_function
- def memory_info
- size = PROCESS_MEMORY_COUNTERS.size
- data = PROCESS_MEMORY_COUNTERS.malloc
- data.cb = size
- data if GetProcessMemoryInfo(GetCurrentProcess(), data, size)
+ keys.push(:size, :rss, :peak)
+
+ begin
+ require 'fiddle/import'
+ require 'fiddle/types'
+ rescue LoadError
+ # Fallback to PowerShell command to get memory information for current process
+ def self.read_status
+ cmd = [
+ "powershell.exe", "-NoProfile", "-Command",
+ "Get-Process -Id #{$$} | " \
+ "% { Write-Output $_.PagedMemorySize64 $_.WorkingSet64 $_.PeakWorkingSet64 }"
+ ]
+
+ IO.popen(cmd, "r", err: [:child, :out]) do |out|
+ if /^(\d+)\n(\d+)\n(\d+)$/ =~ out.read
+ yield :size, $1.to_i
+ yield :rss, $2.to_i
+ yield :peak, $3.to_i
+ end
+ end
+ end
+ else
+ module Win32
+ extend Fiddle::Importer
+ dlload "kernel32.dll", "psapi.dll"
+ include Fiddle::Win32Types
+ typealias "SIZE_T", "size_t"
+
+ PROCESS_MEMORY_COUNTERS = struct [
+ "DWORD cb",
+ "DWORD PageFaultCount",
+ "SIZE_T PeakWorkingSetSize",
+ "SIZE_T WorkingSetSize",
+ "SIZE_T QuotaPeakPagedPoolUsage",
+ "SIZE_T QuotaPagedPoolUsage",
+ "SIZE_T QuotaPeakNonPagedPoolUsage",
+ "SIZE_T QuotaNonPagedPoolUsage",
+ "SIZE_T PagefileUsage",
+ "SIZE_T PeakPagefileUsage",
+ ]
+
+ typealias "PPROCESS_MEMORY_COUNTERS", "PROCESS_MEMORY_COUNTERS*"
+
+ extern "HANDLE GetCurrentProcess()", :stdcall
+ extern "BOOL GetProcessMemoryInfo(HANDLE, PPROCESS_MEMORY_COUNTERS, DWORD)", :stdcall
+
+ module_function
+ def memory_info
+ size = PROCESS_MEMORY_COUNTERS.size
+ data = PROCESS_MEMORY_COUNTERS.malloc
+ data.cb = size
+ data if GetProcessMemoryInfo(GetCurrentProcess(), data, size)
+ end
end
- end
- keys.push(:size, :rss, :peak)
- def self.read_status
- if info = Win32.memory_info
- yield :size, info.PagefileUsage
- yield :rss, info.WorkingSetSize
- yield :peak, info.PeakWorkingSetSize
+ def self.read_status
+ if info = Win32.memory_info
+ yield :size, info.PagefileUsage
+ yield :rss, info.WorkingSetSize
+ yield :peak, info.PeakWorkingSetSize
+ end
end
end
when (require_relative 'find_executable'
diff --git a/tool/lib/output.rb b/tool/lib/output.rb
index 8cb426ae4a..8590e0ffe2 100644
--- a/tool/lib/output.rb
+++ b/tool/lib/output.rb
@@ -31,8 +31,8 @@ class Output
@vpath.def_options(opt)
end
- def write(data, overwrite: @overwrite, create_only: @create_only)
- unless @path
+ def write(data, overwrite: @overwrite, create_only: @create_only, name: nil, newer: nil, quiet: false)
+ unless (name = name ? (@path ? File.join(@path, name) : name) : @path)
$stdout.print data
return true
end
@@ -41,20 +41,21 @@ class Output
updated = color.fail("updated")
outpath = nil
- if (@ifchange or overwrite or create_only) and (@vpath.open(@path, "rb") {|f|
+ if (@ifchange or overwrite or create_only or newer) and (@vpath.open(name, "rb") {|f|
outpath = f.path
+ next true if newer and f.mtime > newer
if @ifchange or create_only
original = f.read
(@ifchange and original == data) or (create_only and !original.empty?)
end
} rescue false)
- puts "#{outpath} #{unchanged}"
+ puts "#{outpath} #{unchanged}" unless quiet
written = false
else
unless overwrite and outpath and (File.binwrite(outpath, data) rescue nil)
- File.binwrite(outpath = @path, data)
+ File.binwrite(outpath = name, data)
end
- puts "#{outpath} #{updated}"
+ puts "#{outpath} #{updated}" unless quiet
written = true
end
if timestamp = @timestamp
diff --git a/tool/lib/test/jobserver.rb b/tool/lib/test/jobserver.rb
new file mode 100644
index 0000000000..7b889163b0
--- /dev/null
+++ b/tool/lib/test/jobserver.rb
@@ -0,0 +1,47 @@
+module Test
+ module JobServer
+ end
+end
+
+class << Test::JobServer
+ def connect(makeflags = ENV["MAKEFLAGS"])
+ return unless /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags
+ begin
+ if fifo = $3
+ fifo.gsub!(/\\(?=.)/, '')
+ r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY)
+ w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY)
+ else
+ r = IO.for_fd($1.to_i(10), "rb", autoclose: false)
+ w = IO.for_fd($2.to_i(10), "wb", autoclose: false)
+ end
+ rescue
+ r&.close
+ nil
+ else
+ return r, w
+ end
+ end
+
+ def acquire_possible(r, w, max)
+ return unless tokens = r.read_nonblock(max - 1, exception: false)
+ if (jobs = tokens.size) > 0
+ jobserver, w = w, nil
+ at_exit do
+ jobserver.print(tokens)
+ jobserver.close
+ end
+ end
+ return jobs + 1
+ rescue Errno::EBADF
+ ensure
+ r&.close
+ w&.close
+ end
+
+ def max_jobs(max = 2, makeflags = ENV["MAKEFLAGS"])
+ if max > 1 and (r, w = connect(makeflags))
+ acquire_possible(r, w, max)
+ end
+ end
+end
diff --git a/tool/lib/test/unit.rb b/tool/lib/test/unit.rb
index 30f30df62e..2663b7b76a 100644
--- a/tool/lib/test/unit.rb
+++ b/tool/lib/test/unit.rb
@@ -19,6 +19,7 @@ require_relative '../envutil'
require_relative '../colorize'
require_relative '../leakchecker'
require_relative '../test/unit/testcase'
+require_relative '../test/jobserver'
require 'optparse'
# See Test::Unit
@@ -249,6 +250,8 @@ module Test
end
module Parallel # :nodoc: all
+ attr_accessor :prefix
+
def process_args(args = [])
return @options if @options
options = super
@@ -260,27 +263,8 @@ module Test
def non_options(files, options)
@jobserver = nil
- makeflags = ENV.delete("MAKEFLAGS")
- if !options[:parallel] and
- /(?:\A|\s)--jobserver-(?:auth|fds)=(?:(\d+),(\d+)|fifo:((?:\\.|\S)+))/ =~ makeflags
- begin
- if fifo = $3
- fifo.gsub!(/\\(?=.)/, '')
- r = File.open(fifo, IO::RDONLY|IO::NONBLOCK|IO::BINARY)
- w = File.open(fifo, IO::WRONLY|IO::NONBLOCK|IO::BINARY)
- else
- r = IO.for_fd($1.to_i(10), "rb", autoclose: false)
- w = IO.for_fd($2.to_i(10), "wb", autoclose: false)
- end
- rescue
- r.close if r
- nil
- else
- r.close_on_exec = true
- w.close_on_exec = true
- @jobserver = [r, w]
- options[:parallel] ||= 256 # number of tokens to acquire first
- end
+ if !options[:parallel] and @jobserver = Test::JobServer.connect(ENV.delete("MAKEFLAGS"))
+ options[:parallel] ||= 256 # number of tokens to acquire first
end
@worker_timeout = EnvUtil.apply_timeout_scale(options[:worker_timeout] || 1200)
super
@@ -298,7 +282,7 @@ module Test
opts.separator "parallel test options:"
- options[:retry] = true
+ options[:retry] = false
opts.on '-j N', '--jobs N', /\A(t)?(\d+)\z/, "Allow run tests with N jobs at once" do |_, t, a|
options[:testing] = true & t # For testing
@@ -370,8 +354,12 @@ module Test
@io.puts(*args)
end
- def run(task,type)
- @file = File.basename(task, ".rb")
+ def run(task, type, base = nil)
+ if base
+ @file = task.delete_prefix(base).chomp(".rb")
+ else
+ @file = File.basename(task, ".rb")
+ end
@real_file = task
begin
puts "loadpath #{[Marshal.dump($:-@loadpath)].pack("m0")}"
@@ -415,6 +403,7 @@ module Test
end
def kill
+ EnvUtil::Debugger.search&.dump(@pid)
signal = RUBY_PLATFORM =~ /mswin|mingw/ ? :KILL : :SEGV
Process.kill(signal, @pid)
warn "worker #{to_s} does not respond; #{signal} is sent"
@@ -597,7 +586,7 @@ module Test
worker.quit
worker = launch_worker
end
- worker.run(task, type)
+ worker.run(task, type, (@prefix unless @options[:job_status] == :replace))
@test_count += 1
jobs_status(worker)
@@ -1292,10 +1281,15 @@ module Test
parser.on '--repeat-count=NUM', "Number of times to repeat", Integer do |n|
options[:repeat_count] = n
end
+ options[:keep_repeating] = false
+ parser.on '--[no-]keep-repeating', "Keep repeating even failed" do |n|
+ options[:keep_repeating] = true
+ end
end
def _run_anything(type)
@repeat_count = @options[:repeat_count]
+ @keep_repeating = @options[:keep_repeating]
super
end
end
@@ -1617,7 +1611,7 @@ module Test
[(@repeat_count ? "(#{@@current_repeat_count}/#{@repeat_count}) " : ""), type,
t, @test_count.fdiv(t), @assertion_count.fdiv(t)]
end while @repeat_count && @@current_repeat_count < @repeat_count &&
- report.empty? && failures.zero? && errors.zero?
+ (@keep_repeating || report.empty? && failures.zero? && errors.zero?)
output.sync = old_sync if sync
@@ -1856,6 +1850,7 @@ module Test
@force_standalone = force_standalone
@runner = Runner.new do |files, options|
base = options[:base_directory] ||= default_dir
+ @runner.prefix = base ? (base + "/") : nil
files << default_dir if files.empty? and default_dir
@to_run = files
yield self if block_given?
diff --git a/tool/lib/test/unit/assertions.rb b/tool/lib/test/unit/assertions.rb
index 19581fc3ab..0908666166 100644
--- a/tool/lib/test/unit/assertions.rb
+++ b/tool/lib/test/unit/assertions.rb
@@ -128,8 +128,16 @@ module Test
def assert_in_delta exp, act, delta = 0.001, msg = nil
n = (exp - act).abs
+ loadavg = begin
+ if File.readable?("/proc/loadavg")
+ " (/proc/loadavg=#{File.read("/proc/loadavg").strip})"
+ end
+ rescue StandardError
+ nil
+ end
+ loadavg ||= ""
msg = message(msg) {
- "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}"
+ "Expected |#{exp} - #{act}| (#{n}) to be <= #{delta}#{loadavg}"
}
assert delta >= n, msg
end
diff --git a/tool/lib/vcs.rb b/tool/lib/vcs.rb
index 51cdb0fdc3..d6374f9de0 100644
--- a/tool/lib/vcs.rb
+++ b/tool/lib/vcs.rb
@@ -51,23 +51,9 @@ module DebugPOpen
end
using DebugPOpen
module DebugSystem
- def system(*args)
+ def system(*args, exception: true, **opts)
VCS.dump(args, "args: ") if $DEBUG
- exception = false
- opts = Hash.try_convert(args[-1])
- if RUBY_VERSION >= "2.6"
- unless opts
- opts = {}
- args << opts
- end
- exception = opts.fetch(:exception) {opts[:exception] = true}
- elsif opts
- exception = opts.delete(:exception) {true}
- args.pop if opts.empty?
- end
- ret = super(*args)
- raise "Command failed with status (#$?): #{args[0]}" if exception and !ret
- ret
+ super(*args, exception: exception, **opts)
end
end
@@ -183,19 +169,7 @@ class VCS
)
last or raise VCS::NotFoundError, "last revision not found"
changed or raise VCS::NotFoundError, "changed revision not found"
- if modified
- /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ modified or
- raise "unknown time format - #{modified}"
- match = $~[1..6].map { |x| x.to_i }
- off = $7 ? "#{$7}:#{$8}" : "+00:00"
- match << off
- begin
- modified = Time.new(*match)
- rescue ArgumentError
- modified = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60
- end
- modified = modified.getlocal(@zone)
- end
+ modified &&= parse_iso_date(modified)
return last, changed, modified, *rest
end
@@ -204,9 +178,9 @@ class VCS
modified
end
- def relative_to(path)
+ def relative_to(path, srcdir = @srcdir)
if path
- srcdir = File.realpath(@srcdir)
+ srcdir = File.realpath(srcdir || @srcdir)
path = File.realdirpath(path)
list1 = srcdir.split(%r{/})
list2 = path.split(%r{/})
@@ -224,6 +198,20 @@ class VCS
end
end
+ def parse_iso_date(date)
+ /\A(\d+)-(\d+)-(\d+)\D(\d+):(\d+):(\d+(?:\.\d+)?)\s*(?:Z|([-+]\d\d)(\d\d))\z/ =~ date or
+ raise "unknown time format - #{date}"
+ match = $~[1..6].map { |x| x.to_i }
+ off = $7 ? "#{$7}:#{$8}" : "+00:00"
+ match << off
+ begin
+ date = Time.new(*match)
+ rescue ArgumentError
+ date = Time.utc(*$~[1..6]) + $7.to_i * 3600 + $8.to_i * 60
+ end
+ date.getlocal(@zone)
+ end
+
def after_export(dir)
FileUtils.rm_rf(Dir.glob("#{dir}/.git*"))
FileUtils.rm_rf(Dir.glob("#{dir}/.mailmap"))
@@ -271,155 +259,6 @@ class VCS
code
end
- class SVN < self
- register(".svn")
- COMMAND = ENV['SVN'] || 'svn'
-
- def self.revision_name(rev)
- "r#{rev}"
- end
-
- def _get_revisions(path, srcdir = nil)
- if srcdir and self.class.local_path?(path)
- path = File.join(srcdir, path)
- end
- if srcdir
- info_xml = IO.pread(%W"#{COMMAND} info --xml #{srcdir}")
- info_xml = nil unless info_xml[/<url>(.*)<\/url>/, 1] == path.to_s
- end
- info_xml ||= IO.pread(%W"#{COMMAND} info --xml #{path}")
- _, last, _, changed, _ = info_xml.split(/revision="(\d+)"/)
- modified = info_xml[/<date>([^<>]*)/, 1]
- branch = info_xml[%r'<relative-url>\^/(?:branches/|tags/)?([^<>]+)', 1]
- [Integer(last), Integer(changed), modified, branch]
- end
-
- def self.search_root(path)
- return unless local_path?(path)
- parent = File.realpath(path)
- begin
- parent = File.dirname(wkdir = parent)
- return wkdir if File.directory?(wkdir + "/.svn")
- end until parent == wkdir
- end
-
- def get_info
- @info ||= IO.pread(%W"#{COMMAND} info --xml #{@srcdir}")
- end
-
- def url
- @url ||= begin
- url = get_info[/<root>(.*)<\/root>/, 1]
- @url = URI.parse(url+"/") if url
- end
- end
-
- def wcroot
- @wcroot ||= begin
- info = get_info
- @wcroot = info[/<wcroot-abspath>(.*)<\/wcroot-abspath>/, 1]
- @wcroot ||= self.class.search_root(@srcdir)
- end
- end
-
- def branch(name)
- return trunk if name == "trunk"
- url + "branches/#{name}"
- end
-
- def tag(name)
- url + "tags/#{name}"
- end
-
- def trunk
- url + "trunk"
- end
- alias master trunk
-
- def branch_list(pat)
- IO.popen(%W"#{COMMAND} ls #{branch('')}") do |f|
- f.each do |line|
- line.chomp!
- line.chomp!('/')
- yield(line) if File.fnmatch?(pat, line)
- end
- end
- end
-
- def grep(pat, tag, *files, &block)
- cmd = %W"#{COMMAND} cat"
- files.map! {|n| File.join(tag, n)} if tag
- set = block.binding.eval("proc {|match| $~ = match}")
- IO.popen([cmd, *files]) do |f|
- f.grep(pat) do |s|
- set[$~]
- yield s
- end
- end
- end
-
- def export(revision, url, dir, keep_temp = false)
- if @srcdir and (rootdir = wcroot)
- srcdir = File.realpath(@srcdir)
- rootdir << "/"
- if srcdir.start_with?(rootdir)
- subdir = srcdir[rootdir.size..-1]
- subdir = nil if subdir.empty?
- FileUtils.mkdir_p(svndir = dir+"/.svn")
- FileUtils.ln_s(Dir.glob(rootdir+"/.svn/*"), svndir)
- system(COMMAND, "-q", "revert", "-R", subdir || ".", :chdir => dir) or return false
- FileUtils.rm_rf(svndir) unless keep_temp
- if subdir
- tmpdir = Dir.mktmpdir("tmp-co.", "#{dir}/#{subdir}")
- File.rename(tmpdir, tmpdir = "#{dir}/#{File.basename(tmpdir)}")
- FileUtils.mv(Dir.glob("#{dir}/#{subdir}/{.[^.]*,..?*,*}"), tmpdir)
- begin
- Dir.rmdir("#{dir}/#{subdir}")
- end until (subdir = File.dirname(subdir)) == '.'
- FileUtils.mv(Dir.glob("#{tmpdir}/#{subdir}/{.[^.]*,..?*,*}"), dir)
- Dir.rmdir(tmpdir)
- end
- return self
- end
- end
- IO.popen(%W"#{COMMAND} export -r #{revision} #{url} #{dir}") do |pipe|
- pipe.each {|line| /^A/ =~ line or yield line}
- end
- self if $?.success?
- end
-
- def after_export(dir)
- super
- FileUtils.rm_rf(dir+"/.svn")
- end
-
- def branch_beginning(url)
- # `--limit` of svn-log is useless in this case, because it is
- # applied before `--search`.
- rev = IO.pread(%W[ #{COMMAND} log --xml
- --search=matz --search-and=has\ started
- -- #{url}/version.h])[/<logentry\s+revision="(\d+)"/m, 1]
- rev.to_i if rev
- end
-
- def export_changelog(url = '.', from = nil, to = nil, _path = nil, path: _path)
- range = [to || 'HEAD', (from ? from+1 : branch_beginning(url))].compact.join(':')
- IO.popen({'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'},
- %W"#{COMMAND} log -r#{range} #{url}") do |r|
- IO.copy_stream(r, path)
- end
- end
-
- def commit
- args = %W"#{COMMAND} commit"
- if dryrun?
- VCS.dump(args, "commit: ")
- return true
- end
- system(*args)
- end
- end
-
class GIT < self
register(".git") do |path, dir|
SAFE_DIRECTORIES ||=
@@ -525,6 +364,11 @@ class VCS
[last, changed, modified, branch, title]
end
+ def author_date(path, srcdir = @srcdir)
+ log = cmd_read_at(srcdir, [[COMMAND, 'log', '-n1', '--pretty=%at', path]])
+ Time.at(log.to_i, in: @zone)
+ end
+
def self.revision_name(rev)
short_revision(rev)
end
@@ -533,15 +377,6 @@ class VCS
rev[0, 10]
end
- def revision_handler(rev)
- case rev
- when Integer
- SVN
- else
- super
- end
- end
-
def without_gitconfig
envs = (%w'HOME XDG_CONFIG_HOME' + ENV.keys.grep(/\AGIT_/)).each_with_object({}) do |v, h|
h[v] = ENV.delete(v)
@@ -616,60 +451,50 @@ class VCS
def export(revision, url, dir, keep_temp = false)
system(COMMAND, "clone", "-c", "advice.detachedHead=false", "-s", (@srcdir || '.').to_s, "-b", url, dir) or return
- (Integer === revision ? GITSVN : GIT).new(File.expand_path(dir))
+ GIT.new(File.expand_path(dir))
end
def branch_beginning(url)
- cmd_read(%W[ #{COMMAND} log -n1 --format=format:%H
+ year = cmd_read(%W[ #{COMMAND} log -n1 --format=%cd --date=format:%Y #{url} --]).to_i
+ cmd_read(%W[ #{COMMAND} log --format=format:%H --reverse --since=#{year-1}-12-25
--author=matz --committer=matz --grep=started\\.$
- #{url.to_str} -- version.h include/ruby/version.h])
+ #{url} -- version.h include/ruby/version.h])[/.*/]
end
- def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: nil)
- svn = nil
+ def export_changelog(url = '@', from = nil, to = nil, _path = nil, path: _path, base_url: true)
from, to = [from, to].map do |rev|
rev or next
- if Integer === rev
- svn = true
- rev = cmd_read({'LANG' => 'C', 'LC_ALL' => 'C'},
- %W"#{COMMAND} log -n1 --format=format:%H" <<
- "--grep=^ *git-svn-id: .*@#{rev} ")
- end
rev unless rev.empty?
end
- unless (from && /./.match(from)) or ((from = branch_beginning(url)) && /./.match(from))
+ to ||= url.to_str
+ unless from&.match?(/./) or (from = branch_beginning(to))&.match?(/./)
warn "no starting commit found", uplevel: 1
from = nil
end
- if svn or system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits",
+ if system(*%W"#{COMMAND} fetch origin refs/notes/commits:refs/notes/commits",
chdir: @srcdir, exception: false)
system(*%W"#{COMMAND} fetch origin refs/notes/log-fix:refs/notes/log-fix",
chdir: @srcdir, exception: false)
else
warn "Could not fetch notes/commits tree", uplevel: 1
end
- to ||= url.to_str
if from
arg = ["#{from}^..#{to}"]
else
arg = ["--since=25 Dec 00:00:00", to]
end
- writer =
- if svn
- format_changelog_as_svn(path, arg)
- else
- if base_url == true
- remote, = upstream
- if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}])
- remote.chomp!
- # hack to redirect git.r-l.o to github
- remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/')
- remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/')
- end
- base_url = remote
- end
- format_changelog(path, arg, base_url)
+ if base_url == true
+ env = CHANGELOG_ENV
+ remote, = upstream
+ if remote &&= cmd_read(env, %W[#{COMMAND} remote get-url --no-push #{remote}])
+ remote.chomp!
+ # hack to redirect git.r-l.o to github
+ remote.sub!(/\Agit@git\.ruby-lang\.org:/, 'git@github.com:ruby/')
+ remote.sub!(/\Agit@(.*?):(.*?)(?:\.git)?\z/, 'https://\1/\2/commit/')
end
+ base_url = remote
+ end
+ writer = changelog_formatter(path, arg, base_url)
if !path or path == '-'
writer[$stdout]
else
@@ -678,9 +503,10 @@ class VCS
end
LOG_FIX_REGEXP_SEPARATORS = '/!:;|,#%&'
+ CHANGELOG_ENV = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}
- def format_changelog(path, arg, base_url = nil)
- env = {'TZ' => 'JST-9', 'LANG' => 'C', 'LC_ALL' => 'C'}
+ def changelog_formatter(path, arg, base_url = nil)
+ env = CHANGELOG_ENV
cmd = %W[#{COMMAND} log
--format=fuller --notes=commits --notes=log-fix --topo-order --no-merges
--fixed-strings --invert-grep --grep=[ci\ skip] --grep=[skip\ ci]
@@ -692,17 +518,32 @@ class VCS
cmd << date
cmd.concat(arg)
proc do |w|
- w.print "-*- coding: utf-8 -*-\n\n"
- w.print "base-url = #{base_url}\n\n" if base_url
+ w.print "-*- coding: utf-8 -*-\n"
+ w.print "\n""base-url = #{base_url}\n" if base_url
+
+ begin
+ ignore_revs = File.readlines(File.join(@srcdir, ".git-blame-ignore-revs"), chomp: true)
+ .grep_v(/^ *(?:#|$)/)
+ .to_h {|v| [v, true]}
+ ignore_revs = nil if ignore_revs.empty?
+ rescue Errno::ENOENT
+ end
+
cmd_pipe(env, cmd, chdir: @srcdir) do |r|
- while s = r.gets("\ncommit ")
+ r.gets(sep = "commit ")
+ sep = "\n" + sep
+ while s = r.gets(sep, chomp: true)
h, s = s.split(/^$/, 2)
+ if ignore_revs&.key?(h[/\A\h{40}/])
+ next
+ end
next if /^Author: *dependabot\[bot\]/ =~ h
h.gsub!(/^(?:(?:Author|Commit)(?:Date)?|Date): /, ' \&')
if s.sub!(/\nNotes \(log-fix\):\n((?: +.*\n)+)/, '')
fix = $1
+ next if /\A *skip\Z/ =~ fix
s = s.lines
fix.each_line do |x|
next unless x.sub!(/^(\s+)(?:(\d+)|\$(?:-\d+)?)/, '')
@@ -739,7 +580,7 @@ class VCS
next
end
end
- message = ["format_changelog failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"]
+ message = ["changelog_formatter failed to replace #{wrong.dump} with #{correct.dump} at #{n}\n"]
from = [1, n-2].max
to = [s.size-1, n+2].min
s.each_with_index do |e, i|
@@ -761,35 +602,9 @@ class VCS
s = s.join('')
end
- if %r[^ +(https://github\.com/[^/]+/[^/]+/)commit/\h+\n(?=(?: +\n(?i: +Co-authored-by: .*\n)+)?(?:\n|\Z))] =~ s
- issue = "#{$1}pull/"
- s.gsub!(/\b(?:(?i:fix(?:e[sd])?) +|GH-)\K#(?=\d+\b)|\(\K#(?=\d+\))/) {issue}
- end
-
s.gsub!(/ +\n/, "\n")
s.sub!(/^Notes:/, ' \&')
- w.print h, s
- end
- end
- end
- end
-
- def format_changelog_as_svn(path, arg)
- cmd = %W"#{COMMAND} log --topo-order --no-notes -z --format=%an%n%at%n%B"
- cmd.concat(arg)
- proc do |w|
- sep = "-"*72 + "\n"
- w.print sep
- cmd_pipe(cmd) do |r|
- while s = r.gets("\0")
- s.chomp!("\0")
- author, time, s = s.split("\n", 3)
- s.sub!(/\n\ngit-svn-id: .*@(\d+) .*\n\Z/, '')
- rev = $1
- time = Time.at(time.to_i).getlocal("+09:00").strftime("%F %T %z (%a, %d %b %Y)")
- lines = s.count("\n") + 1
- lines = "#{lines} line#{lines == 1 ? '' : 's'}"
- w.print "r#{rev} | #{author} | #{time} | #{lines}\n\n", s, "\n", sep
+ w.print sep, h, s
end
end
end
@@ -826,46 +641,6 @@ class VCS
end
end
- class GITSVN < GIT
- def self.revision_name(rev)
- SVN.revision_name(rev)
- end
-
- def last_changed_revision
- rev = cmd_read(%W"#{COMMAND} svn info"+[STDERR=>[:child, :out]])[/^Last Changed Rev: (\d+)/, 1]
- com = cmd_read(%W"#{COMMAND} svn find-rev r#{rev}").chomp
- return rev, com
- end
-
- def commit(opts = {})
- rev, com = last_changed_revision
- head = cmd_read(%W"#{COMMAND} symbolic-ref --short HEAD").chomp
-
- commits = cmd_read([COMMAND, "log", "--reverse", "--format=%H %ae %ce", "#{com}..@"], "rb").split("\n")
- commits.each_with_index do |l, i|
- r, a, c = l.split(' ')
- dcommit = [COMMAND, "svn", "dcommit"]
- dcommit.insert(-2, "-n") if dryrun?
- dcommit << "--add-author-from" unless a == c
- dcommit << r
- system(*dcommit) or return false
- system(COMMAND, "checkout", head) or return false
- system(COMMAND, "rebase") or return false
- end
-
- if rev
- old = [cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp]
- old << cmd_read(%W"#{COMMAND} svn reset -r#{rev}")[/^r#{rev} = (\h+)/, 1]
- 3.times do
- sleep 2
- system(*%W"#{COMMAND} pull --no-edit --rebase")
- break unless old.include?(cmd_read(%W"#{COMMAND} log -1 --format=%H").chomp)
- end
- end
- true
- end
- end
-
class Null < self
def get_revisions(path, srcdir = nil)
@modified ||= Time.now - 10
generated by cgit v1.2.3 (git 2.25.1) at 2026-05-29 21:54:00 +0000