6 files changed, 73 insertions, 56 deletions

diff --git a/tool/test/testunit/test_assertion.rb b/tool/test/testunit/test_assertion.rb
index b0c2267b31..d9bdc8f3c5 100644
--- a/tool/test/testunit/test_assertion.rb
+++ b/tool/test/testunit/test_assertion.rb
@@ -8,6 +8,8 @@ class TestAssertion < Test::Unit::TestCase
   end
 
   def test_timeout_separately
+    pend "hang-up" if /mswin|mingw/ =~ RUBY_PLATFORM
+
     assert_raise(Timeout::Error) do
       assert_separately([], <<~"end;", timeout: 0.1)
         sleep
@@ -15,6 +17,29 @@ class TestAssertion < Test::Unit::TestCase
     end
   end
 
+  def test_assertion_count_separately
+    beginning = self._assertions
+
+    assert_separately([], "")
+    assertions_at_nothing = self._assertions - beginning
+
+    prev_assertions = self._assertions + assertions_at_nothing
+    assert_separately([], "assert true")
+    assert_equal(1, self._assertions - prev_assertions)
+
+    omit unless Process.respond_to?(:fork)
+    prev_assertions = self._assertions + assertions_at_nothing
+    assert_separately([], "Process.fork {assert true}; assert true")
+    assert_equal(2, self._assertions - prev_assertions)
+
+    prev_assertions = self._assertions + assertions_at_nothing
+    # TODO: assertions before `fork` are counted twice; it is possible
+    # to reset `_assertions` at `Process._fork`, but the hook can
+    # interfere in other tests.
+    assert_separately([], "assert true; Process.fork {assert true}")
+    assert_equal(3, self._assertions - prev_assertions)
+  end
+
   def return_in_assert_raise
     assert_raise(RuntimeError) do
       return
diff --git a/tool/test/testunit/test_minitest_unit.rb b/tool/test/testunit/test_minitest_unit.rb
index 84b6cf688c..7f53e4b7dd 100644
--- a/tool/test/testunit/test_minitest_unit.rb
+++ b/tool/test/testunit/test_minitest_unit.rb
@@ -646,7 +646,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase
 
   def test_assert_in_delta_triggered
     x = "1.0e-06"
-    util_assert_triggered "Expected |0.0 - 0.001| (0.001) to be <= #{x}." do
+    util_assert_triggered "Expected |0.0 - 0.001| (0.001) to be <= #{x}.", strip: /\s+\(\/proc\/loadavg=.*\)/ do
       @tc.assert_in_delta 0.0, 1.0 / 1000, 0.000001
     end
   end
@@ -678,7 +678,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase
   end
 
   def test_assert_in_epsilon_triggered
-    util_assert_triggered 'Expected |10000 - 9990| (10) to be <= 9.99.' do
+    util_assert_triggered 'Expected |10000 - 9990| (10) to be <= 9.99.', strip: /\s+\(\/proc\/loadavg=.*\)/ do
       @tc.assert_in_epsilon 10000, 9990
     end
   end
@@ -686,7 +686,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase
   def test_assert_in_epsilon_triggered_negative_case
     x = "0.100000xxx"
     y = "0.1"
-    util_assert_triggered "Expected |-1.1 - -1| (#{x}) to be <= #{y}." do
+    util_assert_triggered "Expected |-1.1 - -1| (#{x}) to be <= #{y}.", strip: /\s+\(\/proc\/loadavg=.*\)/ do
       @tc.assert_in_epsilon(-1.1, -1, 0.1)
     end
   end
@@ -1352,7 +1352,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase
     assert_equal expected, sample_test_case.test_methods.sort
   end
 
-  def assert_triggered expected, klass = Test::Unit::AssertionFailedError
+  def assert_triggered expected, klass = Test::Unit::AssertionFailedError, strip: nil
     e = assert_raise klass do
       yield
     end
@@ -1360,6 +1360,7 @@ class TestMiniTestUnitTestCase < Test::Unit::TestCase
     msg = e.message.sub(/(---Backtrace---).*/m, '\1')
     msg.gsub!(/\(oid=[-0-9]+\)/, '(oid=N)')
     msg.gsub!(/(\d\.\d{6})\d+/, '\1xxx') # normalize: ruby version, impl, platform
+    msg.gsub!(strip, '') if strip
 
     assert_equal expected, msg
   end
diff --git a/tool/test/testunit/test_parallel.rb b/tool/test/testunit/test_parallel.rb
index 2766e9f573..adf7d62ecd 100644
--- a/tool/test/testunit/test_parallel.rb
+++ b/tool/test/testunit/test_parallel.rb
@@ -8,6 +8,14 @@ module TestParallel
   # use large timeout for --jit-wait
   TIMEOUT = EnvUtil.apply_timeout_scale(100)
 
+  def self.timeout(n, &blk)
+    start_time = Time.now
+    Timeout.timeout(n, &blk)
+  rescue Timeout::Error
+    end_time = Time.now
+    raise Timeout::Error, "execution expired (start: #{ start_time }, end: #{ end_time })"
+  end
+
   class TestParallelWorker < Test::Unit::TestCase
     def setup
       i, @worker_in = IO.pipe
@@ -25,7 +33,7 @@ module TestParallel
             @worker_in.puts "quit normal"
           rescue IOError, Errno::EPIPE
           end
-          Timeout.timeout(2) do
+          ::TestParallel.timeout(2) do
             Process.waitpid(@worker_pid)
           end
         rescue Timeout::Error
@@ -45,7 +53,7 @@ module TestParallel
     end
 
     def test_run
-      Timeout.timeout(TIMEOUT) do
+      ::TestParallel.timeout(TIMEOUT) do
         assert_match(/^ready/,@worker_out.gets)
         @worker_in.puts "run #{TESTS}/ptest_first.rb test"
         assert_match(/^okay/,@worker_out.gets)
@@ -58,7 +66,7 @@ module TestParallel
     end
 
     def test_run_multiple_testcase_in_one_file
-      Timeout.timeout(TIMEOUT) do
+      ::TestParallel.timeout(TIMEOUT) do
         assert_match(/^ready/,@worker_out.gets)
         @worker_in.puts "run #{TESTS}/ptest_second.rb test"
         assert_match(/^okay/,@worker_out.gets)
@@ -75,7 +83,7 @@ module TestParallel
     end
 
     def test_accept_run_command_multiple_times
-      Timeout.timeout(TIMEOUT) do
+      ::TestParallel.timeout(TIMEOUT) do
         assert_match(/^ready/,@worker_out.gets)
         @worker_in.puts "run #{TESTS}/ptest_first.rb test"
         assert_match(/^okay/,@worker_out.gets)
@@ -99,7 +107,7 @@ module TestParallel
     end
 
     def test_p
-      Timeout.timeout(TIMEOUT) do
+      ::TestParallel.timeout(TIMEOUT) do
         @worker_in.puts "run #{TESTS}/ptest_first.rb test"
         while buf = @worker_out.gets
           break if /^p (.+?)$/ =~ buf
@@ -110,7 +118,7 @@ module TestParallel
     end
 
     def test_done
-      Timeout.timeout(TIMEOUT) do
+      ::TestParallel.timeout(TIMEOUT) do
         @worker_in.puts "run #{TESTS}/ptest_forth.rb test"
         while buf = @worker_out.gets
           break if /^done (.+?)$/ =~ buf
@@ -118,24 +126,24 @@ module TestParallel
         assert_not_nil($1, "'done' was not found")
 
         result = Marshal.load($1.chomp.unpack1("m"))
-        assert_equal(5, result[0])
-        pend "TODO: result[1] returns 17. We should investigate it" do # TODO: misusage of pend (pend doens't use given block)
-          assert_equal(12, result[1])
-        end
-        assert_kind_of(Array,result[2])
-        assert_kind_of(Array,result[3])
-        assert_kind_of(Array,result[4])
-        assert_kind_of(Array,result[2][1])
-        assert_kind_of(Test::Unit::AssertionFailedError,result[2][0][2])
-        assert_kind_of(Test::Unit::PendedError,result[2][1][2])
-        assert_kind_of(Test::Unit::PendedError,result[2][2][2])
-        assert_kind_of(Exception, result[2][3][2])
-        assert_equal(result[5], "TestE")
+        tests, asserts, reports, failures, loadpaths, suite = result
+        assert_equal(5, tests)
+        assert_equal(12, asserts)
+        assert_kind_of(Array, reports)
+        assert_kind_of(Array, failures)
+        assert_kind_of(Array, loadpaths)
+        reports.sort_by! {|_, t| t}
+        assert_kind_of(Array, reports[1])
+        assert_kind_of(Test::Unit::AssertionFailedError, reports[0][2])
+        assert_kind_of(Test::Unit::PendedError, reports[1][2])
+        assert_kind_of(Test::Unit::PendedError, reports[2][2])
+        assert_kind_of(Exception, reports[3][2])
+        assert_equal("TestE", suite)
       end
     end
 
     def test_quit
-      Timeout.timeout(TIMEOUT) do
+      ::TestParallel.timeout(TIMEOUT) do
         @worker_in.puts "quit normal"
         assert_match(/^bye$/m,@worker_out.read)
       end
@@ -143,9 +151,9 @@ module TestParallel
   end
 
   class TestParallel < Test::Unit::TestCase
-    def spawn_runner(*opt_args, jobs: "t1")
+    def spawn_runner(*opt_args, jobs: "t1", env: {})
       @test_out, o = IO.pipe
-      @test_pid = spawn(*@__runner_options__[:ruby], TESTS+"/runner.rb",
+      @test_pid = spawn(env, *@__runner_options__[:ruby], TESTS+"/runner.rb",
                         "--ruby", @__runner_options__[:ruby].join(" "),
                         "-j", jobs, *opt_args, out: o, err: o)
       o.close
@@ -154,7 +162,7 @@ module TestParallel
     def teardown
       begin
         if @test_pid
-          Timeout.timeout(2) do
+          ::TestParallel.timeout(2) do
             Process.waitpid(@test_pid)
           end
         end
@@ -167,54 +175,47 @@ module TestParallel
 
     def test_ignore_jzero
       spawn_runner(jobs: "0")
-      Timeout.timeout(TIMEOUT) {
+      ::TestParallel.timeout(TIMEOUT) {
         assert_match(/Error: parameter of -j option should be greater than 0/,@test_out.read)
       }
     end
 
     def test_should_run_all_without_any_leaks
       spawn_runner
-      buf = Timeout.timeout(TIMEOUT) {@test_out.read}
+      buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read}
       assert_match(/^9 tests/,buf)
     end
 
     def test_should_retry_failed_on_workers
       spawn_runner "--retry"
-      buf = Timeout.timeout(TIMEOUT) {@test_out.read}
+      buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read}
       assert_match(/^Retrying\.+$/,buf)
     end
 
     def test_no_retry_option
       spawn_runner "--no-retry"
-      buf = Timeout.timeout(TIMEOUT) {@test_out.read}
+      buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read}
       refute_match(/^Retrying\.+$/,buf)
       assert_match(/^ +\d+\) Failure:\nTestD#test_fail_at_worker/,buf)
     end
 
     def test_jobs_status
       spawn_runner "--jobs-status"
-      buf = Timeout.timeout(TIMEOUT) {@test_out.read}
+      buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read}
       assert_match(/\d+=ptest_(first|second|third|forth) */,buf)
     end
 
     def test_separate
       # this test depends to --jobs-status
       spawn_runner "--jobs-status", "--separate"
-      buf = Timeout.timeout(TIMEOUT) {@test_out.read}
+      buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read}
       assert(buf.scan(/^\[\s*\d+\/\d+\]\s*(\d+?)=/).flatten.uniq.size > 1,
              message("retried tests should run in different processes") {buf})
     end
 
     def test_hungup
-      spawn_runner "--worker-timeout=1", "--retry", "test4test_hungup.rb"
-      buf = Timeout.timeout(TIMEOUT) {@test_out.read}
-      assert_match(/^Retrying hung up testcases\.+$/, buf)
-      assert_match(/^2 tests,.* 0 failures,/, buf)
-    end
-
-    def test_retry_workers
-      spawn_runner "--worker-timeout=1", "--retry", "test4test_slow_0.rb", "test4test_slow_1.rb", jobs: "2"
-      buf = Timeout.timeout(TIMEOUT) {@test_out.read}
+      spawn_runner("--worker-timeout=1", "--retry", "test4test_hungup.rb", env: {"RUBY_CRASH_REPORT"=>nil})
+      buf = ::TestParallel.timeout(TIMEOUT) {@test_out.read}
       assert_match(/^Retrying hung up testcases\.+$/, buf)
       assert_match(/^2 tests,.* 0 failures,/, buf)
     end
diff --git a/tool/test/testunit/tests_for_parallel/ptest_forth.rb b/tool/test/testunit/tests_for_parallel/ptest_forth.rb
index 8831676e19..54474c828d 100644
--- a/tool/test/testunit/tests_for_parallel/ptest_forth.rb
+++ b/tool/test/testunit/tests_for_parallel/ptest_forth.rb
@@ -8,19 +8,19 @@ class TestE < Test::Unit::TestCase
     assert_equal(1,1)
   end
 
-  def test_always_skip
-    skip "always"
+  def test_always_omit
+    omit "always"
   end
 
   def test_always_fail
     assert_equal(0,1)
   end
 
-  def test_skip_after_unknown_error
+  def test_pend_after_unknown_error
     begin
       raise UnknownError, "unknown error"
     rescue
-      skip "after raise"
+      pend "after raise"
     end
   end
 
diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb
deleted file mode 100644
index a749b0e1d3..0000000000
--- a/tool/test/testunit/tests_for_parallel/test4test_slow_0.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-require_relative 'slow_helper'
-
-class TestSlowV0 < Test::Unit::TestCase
-  include TestSlowTimeout
-end
diff --git a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb b/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb
deleted file mode 100644
index 924a3b11fa..0000000000
--- a/tool/test/testunit/tests_for_parallel/test4test_slow_1.rb
+++ /dev/null
@@ -1,5 +0,0 @@
-require_relative 'slow_helper'
-
-class TestSlowV1 < Test::Unit::TestCase
-  include TestSlowTimeout
-end