bpo-31234: Add test.support.wait_threads_exit() (#3578)

Use _thread.count() to wait until threads exit. The new context
manager prevents the "dangling thread" warning.

Lib/test/lock_tests.py

@@ -31,6 +31,9 @@ class Bunch(object):
         self.started = []
         self.finished = []
         self._can_exit = not wait_before_exit
+        self.wait_thread = support.wait_threads_exit()
+        self.wait_thread.__enter__()
+
         def task():
             tid = threading.get_ident()
             self.started.append(tid)
@@ -40,6 +43,7 @@ class Bunch(object):
                 self.finished.append(tid)
                 while not self._can_exit:
                     _wait()
+
         try:
             for i in range(n):
                 start_new_thread(task, ())
@@ -54,13 +58,8 @@ class Bunch(object):
     def wait_for_finished(self):
         while len(self.finished) < self.n:
             _wait()
-        # Wait a little bit longer to prevent the "threading_cleanup()
-        # failed to cleanup X threads" warning. The loop above is a weak
-        # synchronization. At the C level, t_bootstrap() can still be
-        # running and so _thread.count() still accounts the "almost dead"
-        # thead.
-        for _ in range(self.n):
-            _wait()
+        # Wait for threads exit
+        self.wait_thread.__exit__(None, None, None)
 
     def do_finish(self):
         self._can_exit = True
@@ -227,11 +226,14 @@ class LockTests(BaseLockTests):
         # Lock needs to be released before re-acquiring.
         lock = self.locktype()
         phase = []
+
         def f():
             lock.acquire()
             phase.append(None)
             lock.acquire()
             phase.append(None)
+
+        with support.wait_threads_exit():
             start_new_thread(f, ())
             while len(phase) == 0:
                 _wait()

Lib/test/support/__init__.py

@@ -2072,6 +2072,41 @@ def reap_threads(func):
     return decorator
 
 
+@contextlib.contextmanager
+def wait_threads_exit(timeout=60.0):
+    """
+    bpo-31234: Context manager to wait until all threads created in the with
+    statement exit.
+
+    Use _thread.count() to check if threads exited. Indirectly, wait until
+    threads exit the internal t_bootstrap() C function of the _thread module.
+
+    threading_setup() and threading_cleanup() are designed to emit a warning
+    if a test leaves running threads in the background. This context manager
+    is designed to cleanup threads started by the _thread.start_new_thread()
+    which doesn't allow to wait for thread exit, whereas thread.Thread has a
+    join() method.
+    """
+    old_count = _thread._count()
+    try:
+        yield
+    finally:
+        start_time = time.monotonic()
+        deadline = start_time + timeout
+        while True:
+            count = _thread._count()
+            if count <= old_count:
+                break
+            if time.monotonic() > deadline:
+                dt = time.monotonic() - start_time
+                msg = (f"wait_threads() failed to cleanup {count - old_count} "
+                       f"threads after {dt:.1f} seconds "
+                       f"(count: {count}, old count: {old_count})")
+                raise AssertionError(msg)
+            time.sleep(0.010)
+            gc_collect()
+
+
 def reap_children():
     """Use this function at the end of test_main() whenever sub-processes
     are started.  This will help ensure that no extra children (zombies)

Lib/test/test_socket.py

@@ -271,6 +271,9 @@ class ThreadableTest:
         self.server_ready.set()
 
     def _setUp(self):
+        self.wait_threads = support.wait_threads_exit()
+        self.wait_threads.__enter__()
+
         self.server_ready = threading.Event()
         self.client_ready = threading.Event()
         self.done = threading.Event()
@@ -297,6 +300,7 @@ class ThreadableTest:
     def _tearDown(self):
         self.__tearDown()
         self.done.wait()
+        self.wait_threads.__exit__(None, None, None)
 
         if self.queue.qsize():
             exc = self.queue.get()

Lib/test/test_thread.py

@@ -59,6 +59,7 @@ class ThreadRunningTests(BasicThreadTest):
                 self.done_mutex.release()
 
     def test_starting_threads(self):
+        with support.wait_threads_exit():
             # Basic test for thread creation.
             for i in range(NUMTASKS):
                 self.newtask()
@@ -94,6 +95,7 @@ class ThreadRunningTests(BasicThreadTest):
             verbose_print("trying stack_size = (%d)" % tss)
             self.next_ident = 0
             self.created = 0
+            with support.wait_threads_exit():
                 for i in range(NUMTASKS):
                     self.newtask()
 
@@ -109,10 +111,13 @@ class ThreadRunningTests(BasicThreadTest):
         mut = thread.allocate_lock()
         mut.acquire()
         started = []
+
         def task():
             started.append(None)
             mut.acquire()
             mut.release()
+
+        with support.wait_threads_exit():
             thread.start_new_thread(task, ())
             while not started:
                 time.sleep(POLL_SLEEP)
@@ -140,16 +145,14 @@ class ThreadRunningTests(BasicThreadTest):
             except ValueError:
                 pass
             real_write(self, *args)
-        c = thread._count()
         started = thread.allocate_lock()
         with support.captured_output("stderr") as stderr:
             real_write = stderr.write
             stderr.write = mywrite
             started.acquire()
+            with support.wait_threads_exit():
                 thread.start_new_thread(task, ())
                 started.acquire()
-            while thread._count() > c:
-                time.sleep(POLL_SLEEP)
         self.assertIn("Traceback", stderr.getvalue())
 
 
@@ -181,6 +184,7 @@ class Barrier:
 class BarrierTest(BasicThreadTest):
 
     def test_barrier(self):
+        with support.wait_threads_exit():
             self.bar = Barrier(NUMTASKS)
             self.running = NUMTASKS
             for i in range(NUMTASKS):
@@ -225,11 +229,10 @@ class TestForkInThread(unittest.TestCase):
     @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork')
     @support.reap_threads
     def test_forkinthread(self):
-        running = True
         status = "not set"
 
         def thread1():
-            nonlocal running, status
+            nonlocal status
 
             # fork in a thread
             pid = os.fork()
@@ -244,13 +247,11 @@ class TestForkInThread(unittest.TestCase):
                 # parent
                 os.close(self.write_fd)
                 pid, status = os.waitpid(pid, 0)
-                running = False
 
+        with support.wait_threads_exit():
             thread.start_new_thread(thread1, ())
             self.assertEqual(os.read(self.read_fd, 2), b"OK",
                              "Unable to fork() in thread")
-        while running:
-            time.sleep(POLL_SLEEP)
         self.assertEqual(status, 0)
 
     def tearDown(self):

Lib/test/test_threading.py

@@ -125,9 +125,10 @@ class ThreadTests(BaseTestCase):
             done.set()
         done = threading.Event()
         ident = []
-        _thread.start_new_thread(f, ())
+        with support.wait_threads_exit():
+            tid = _thread.start_new_thread(f, ())
             done.wait()
-        self.assertIsNotNone(ident[0])
+            self.assertEqual(ident[0], tid)
         # Kill the "immortal" _DummyThread
         del threading._active[ident[0]]
 
@@ -165,6 +166,7 @@ class ThreadTests(BaseTestCase):
 
         mutex = threading.Lock()
         mutex.acquire()
+        with support.wait_threads_exit():
             tid = _thread.start_new_thread(f, (mutex,))
             # Wait for the thread to finish.
             mutex.acquire()

Lib/test/test_threadsignals.py

@@ -4,8 +4,8 @@ import unittest
 import signal
 import os
 import sys
-from test.support import run_unittest, import_module
-thread = import_module('_thread')
+from test import support
+thread = support.import_module('_thread')
 import time
 
 if (sys.platform[:3] == 'win'):
@@ -39,6 +39,7 @@ def send_signals():
 class ThreadSignals(unittest.TestCase):
 
     def test_signals(self):
+        with support.wait_threads_exit():
             # Test signal handling semantics of threads.
             # We spawn a thread, have the thread send two signals, and
             # wait for it to finish. Check that we got both signals
@@ -46,6 +47,7 @@ class ThreadSignals(unittest.TestCase):
             signalled_all.acquire()
             self.spawnSignallingThread()
             signalled_all.acquire()
+
         # the signals that we asked the kernel to send
         # will come back, but we don't know when.
         # (it might even be after the thread exits
@@ -115,6 +117,8 @@ class ThreadSignals(unittest.TestCase):
             # thread.
             def other_thread():
                 rlock.acquire()
+
+            with support.wait_threads_exit():
                 thread.start_new_thread(other_thread, ())
                 # Wait until we can't acquire it without blocking...
                 while rlock.acquire(blocking=False):
@@ -133,6 +137,7 @@ class ThreadSignals(unittest.TestCase):
         self.sig_recvd = False
         def my_handler(signal, frame):
             self.sig_recvd = True
+
         old_handler = signal.signal(signal.SIGUSR1, my_handler)
         try:
             def other_thread():
@@ -147,6 +152,8 @@ class ThreadSignals(unittest.TestCase):
                 # the lock acquisition.  Then we'll let it run.
                 time.sleep(0.5)
                 lock.release()
+
+            with support.wait_threads_exit():
                 thread.start_new_thread(other_thread, ())
                 # Wait until we can't acquire it without blocking...
                 while lock.acquire(blocking=False):
@@ -193,6 +200,7 @@ class ThreadSignals(unittest.TestCase):
                     os.kill(process_pid, signal.SIGUSR1)
                 done.release()
 
+            with support.wait_threads_exit():
                 # Send the signals from the non-main thread, since the main thread
                 # is the only one that can process signals.
                 thread.start_new_thread(send_signals, ())
@@ -219,7 +227,7 @@ def test_main():
 
     oldsigs = registerSignals(handle_signals, handle_signals, handle_signals)
     try:
-        run_unittest(ThreadSignals)
+        support.run_unittest(ThreadSignals)
     finally:
         registerSignals(*oldsigs)