bpo-37531: sync regrtest with master branch (GH-16285)

Lib/test/libregrtest/main.py

@@ -22,6 +22,12 @@ from test.libregrtest.utils import removepy, count, format_duration, printlist
 from test import support
 
 
+# bpo-38203: Maximum delay in seconds to exit Python (call Py_Finalize()).
+# Used to protect against threading._shutdown() hang.
+# Must be smaller than buildbot "1200 seconds without output" limit.
+EXIT_TIMEOUT = 120.0
+
+
 class Regrtest:
     """Execute a test suite.
 
@@ -158,11 +164,6 @@ class Regrtest:
     def parse_args(self, kwargs):
         ns = _parse_args(sys.argv[1:], **kwargs)
 
-        if ns.timeout and not hasattr(faulthandler, 'dump_traceback_later'):
-            print("Warning: The timeout option requires "
-                  "faulthandler.dump_traceback_later", file=sys.stderr)
-            ns.timeout = None
-
         if ns.xmlpath:
             support.junit_xml_list = self.testsuite_xml = []
 
@@ -616,6 +617,7 @@ class Regrtest:
 
         test_cwd = self.create_temp_dir()
 
+        try:
             # Run the tests in a context manager that temporarily changes the CWD
             # to a temporary and writable directory. If it's not possible to
             # create or change the CWD, the original CWD will be used.
@@ -625,7 +627,14 @@ class Regrtest:
                 # as their parent temporary directory. So when the main process
                 # exit, it removes also subdirectories of worker processes.
                 self.ns.tempdir = test_cwd
+
                 self._main(tests, kwargs)
+        except SystemExit as exc:
+            # bpo-38203: Python can hang at exit in Py_Finalize(), especially
+            # on threading._shutdown() call: put a timeout
+            faulthandler.dump_traceback_later(EXIT_TIMEOUT, exit=True)
+
+            sys.exit(exc.code)
 
     def getloadavg(self):
         if self.win_load_tracker is not None:

Lib/test/libregrtest/runtest_mp.py

@@ -185,14 +185,14 @@ class MultiprocessThread(threading.Thread):
     def _timedout(self, test_name):
         self._kill()
 
-        stdout = sterr = ''
+        stdout = stderr = ''
         popen = self._popen
         try:
             stdout, stderr = popen.communicate(timeout=JOIN_TIMEOUT)
         except (subprocess.TimeoutExpired, OSError) as exc:
             print("WARNING: Failed to read worker process %s output "
                   "(timeout=%.1f sec): %r"
-                  % (popen.pid, exc, timeout),
+                  % (popen.pid, JOIN_TIMEOUT, exc),
                   file=sys.stderr, flush=True)
 
         self._close_wait()