[3.13] gh-128770: raise warnings as errors in test suite - except for test_socket which still logs warnings, and internal test warnings that are now logged (#131802)

Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com>
(cherry picked from commit 8a00c9a4d2)

Lib/test/libregrtest/main.py

@@ -630,9 +630,9 @@ class Regrtest:
         if not sys.stdout.write_through:
             python_opts.append('-u')
 
-        # Add warnings filter 'default'
+        # Add warnings filter 'error'
         if 'default' not in sys.warnoptions:
-            python_opts.extend(('-W', 'default'))
+            python_opts.extend(('-W', 'error'))
 
         # Error on bytes/str comparison
         if sys.flags.bytes_warning < 2:

Lib/test/support/__init__.py

@@ -6,6 +6,7 @@ if __name__ != 'test.support':
 import contextlib
 import dataclasses
 import functools
+import logging
 import _opcode
 import os
 import re
@@ -386,7 +387,7 @@ def skip_if_buildbot(reason=None):
     try:
         isbuildbot = getpass.getuser().lower() == 'buildbot'
     except (KeyError, OSError) as err:
-        warnings.warn(f'getpass.getuser() failed {err}.', RuntimeWarning)
+        logging.getLogger(__name__).warning('getpass.getuser() failed %s.', err, exc_info=err)
         isbuildbot = False
     return unittest.skipIf(isbuildbot, reason)
 
@@ -1079,8 +1080,7 @@ class _MemoryWatchdog:
         try:
             f = open(self.procfile, 'r')
         except OSError as e:
-            warnings.warn('/proc not available for stats: {}'.format(e),
+            logging.getLogger(__name__).warning('/proc not available for stats: %s', e, exc_info=e)
-                          RuntimeWarning)
             sys.stderr.flush()
             return
 

Lib/test/support/numbers.py

@@ -0,0 +1,80 @@
+# These are shared with test_tokenize and other test modules.
+#
+# Note: since several test cases filter out floats by looking for "e" and ".",
+# don't add hexadecimal literals that contain "e" or "E".
+VALID_UNDERSCORE_LITERALS = [
+    '0_0_0',
+    '4_2',
+    '1_0000_0000',
+    '0b1001_0100',
+    '0xffff_ffff',
+    '0o5_7_7',
+    '1_00_00.5',
+    '1_00_00.5e5',
+    '1_00_00e5_1',
+    '1e1_0',
+    '.1_4',
+    '.1_4e1',
+    '0b_0',
+    '0x_f',
+    '0o_5',
+    '1_00_00j',
+    '1_00_00.5j',
+    '1_00_00e5_1j',
+    '.1_4j',
+    '(1_2.5+3_3j)',
+    '(.5_6j)',
+]
+INVALID_UNDERSCORE_LITERALS = [
+    # Trailing underscores:
+    '0_',
+    '42_',
+    '1.4j_',
+    '0x_',
+    '0b1_',
+    '0xf_',
+    '0o5_',
+    '0 if 1_Else 1',
+    # Underscores in the base selector:
+    '0_b0',
+    '0_xf',
+    '0_o5',
+    # Old-style octal, still disallowed:
+    '0_7',
+    '09_99',
+    # Multiple consecutive underscores:
+    '4_______2',
+    '0.1__4',
+    '0.1__4j',
+    '0b1001__0100',
+    '0xffff__ffff',
+    '0x___',
+    '0o5__77',
+    '1e1__0',
+    '1e1__0j',
+    # Underscore right before a dot:
+    '1_.4',
+    '1_.4j',
+    # Underscore right after a dot:
+    '1._4',
+    '1._4j',
+    '._5',
+    '._5j',
+    # Underscore right after a sign:
+    '1.0e+_1',
+    '1.0e+_1j',
+    # Underscore right before j:
+    '1.4_j',
+    '1.4e5_j',
+    # Underscore right before e:
+    '1_e1',
+    '1.4_e1',
+    '1.4_e1j',
+    # Underscore right after e:
+    '1e_1',
+    '1.4e_1',
+    '1.4e_1j',
+    # Complex cases with parens:
+    '(1+1.5_j_)',
+    '(1+1.5_j)',
+]

Lib/test/support/os_helper.py

@@ -1,6 +1,7 @@
 import collections.abc
 import contextlib
 import errno
+import logging
 import os
 import re
 import stat
@@ -378,8 +379,12 @@ if sys.platform.startswith("win"):
             # Increase the timeout and try again
             time.sleep(timeout)
             timeout *= 2
-        warnings.warn('tests may fail, delete still pending for ' + pathname,
+        logging.getLogger(__name__).warning(
-                      RuntimeWarning, stacklevel=4)
+            'tests may fail, delete still pending for %s',
+            pathname,
+            stack_info=True,
+            stacklevel=4,
+        )
 
     def _unlink(filename):
         _waitfor(os.unlink, filename)
@@ -494,9 +499,14 @@ def temp_dir(path=None, quiet=False):
         except OSError as exc:
             if not quiet:
                 raise
-            warnings.warn(f'tests may fail, unable to create '
+            logging.getLogger(__name__).warning(
-                          f'temporary directory {path!r}: {exc}',
+                "tests may fail, unable to create temporary directory %r: %s",
-                          RuntimeWarning, stacklevel=3)
+                path,
+                exc,
+                exc_info=exc,
+                stack_info=True,
+                stacklevel=3,
+            )
     if dir_created:
         pid = os.getpid()
     try:
@@ -527,9 +537,15 @@ def change_cwd(path, quiet=False):
     except OSError as exc:
         if not quiet:
             raise
-        warnings.warn(f'tests may fail, unable to change the current working '
+        logging.getLogger(__name__).warning(
-                      f'directory to {path!r}: {exc}',
+            'tests may fail, unable to change the current working directory '
-                      RuntimeWarning, stacklevel=3)
+            'to %r: %s',
+            path,
+            exc,
+            exc_info=exc,
+            stack_info=True,
+            stacklevel=3,
+        )
     try:
         yield os.getcwd()
     finally:

Lib/test/test_complex.py

@@ -2,8 +2,10 @@ import unittest
 import sys
 from test import support
 from test.support.testcase import ComplexesAreIdenticalMixin
-from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
+from test.support.numbers import (
-                               INVALID_UNDERSCORE_LITERALS)
+    VALID_UNDERSCORE_LITERALS,
+    INVALID_UNDERSCORE_LITERALS,
+)
 
 from random import random
 from math import isnan, copysign

Lib/test/test_decimal.py

@@ -24,6 +24,7 @@ you're working through IDLE, you can import this test module and call test()
 with the corresponding argument.
 """
 
+import logging
 import math
 import os, sys
 import operator
@@ -5932,8 +5933,9 @@ def tearDownModule():
     if C: C.setcontext(ORIGINAL_CONTEXT[C].copy())
     P.setcontext(ORIGINAL_CONTEXT[P].copy())
     if not C:
-        warnings.warn('C tests skipped: no module named _decimal.',
+        logging.getLogger(__name__).warning(
-                      UserWarning)
+            'C tests skipped: no module named _decimal.'
+        )
     if not orig_sys_decimal is sys.modules['decimal']:
         raise TestFailed("Internal error: unbalanced number of changes to "
                          "sys.modules['decimal'].")

Lib/test/test_float.py

@@ -9,8 +9,10 @@ import unittest
 
 from test import support
 from test.support.testcase import FloatsAreIdenticalMixin
-from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
+from test.support.numbers import (
-                               INVALID_UNDERSCORE_LITERALS)
+    VALID_UNDERSCORE_LITERALS,
+    INVALID_UNDERSCORE_LITERALS,
+)
 from math import isinf, isnan, copysign, ldexp
 import math
 

Lib/test/test_grammar.py

@@ -16,88 +16,10 @@ import test.typinganndata.ann_module as ann_module
 import typing
 from test.typinganndata import ann_module2
 import test
-
+from test.support.numbers import (
-# These are shared with test_tokenize and other test modules.
+    VALID_UNDERSCORE_LITERALS,
-#
+    INVALID_UNDERSCORE_LITERALS,
-# Note: since several test cases filter out floats by looking for "e" and ".",
+)
-# don't add hexadecimal literals that contain "e" or "E".
-VALID_UNDERSCORE_LITERALS = [
-    '0_0_0',
-    '4_2',
-    '1_0000_0000',
-    '0b1001_0100',
-    '0xffff_ffff',
-    '0o5_7_7',
-    '1_00_00.5',
-    '1_00_00.5e5',
-    '1_00_00e5_1',
-    '1e1_0',
-    '.1_4',
-    '.1_4e1',
-    '0b_0',
-    '0x_f',
-    '0o_5',
-    '1_00_00j',
-    '1_00_00.5j',
-    '1_00_00e5_1j',
-    '.1_4j',
-    '(1_2.5+3_3j)',
-    '(.5_6j)',
-]
-INVALID_UNDERSCORE_LITERALS = [
-    # Trailing underscores:
-    '0_',
-    '42_',
-    '1.4j_',
-    '0x_',
-    '0b1_',
-    '0xf_',
-    '0o5_',
-    '0 if 1_Else 1',
-    # Underscores in the base selector:
-    '0_b0',
-    '0_xf',
-    '0_o5',
-    # Old-style octal, still disallowed:
-    '0_7',
-    '09_99',
-    # Multiple consecutive underscores:
-    '4_______2',
-    '0.1__4',
-    '0.1__4j',
-    '0b1001__0100',
-    '0xffff__ffff',
-    '0x___',
-    '0o5__77',
-    '1e1__0',
-    '1e1__0j',
-    # Underscore right before a dot:
-    '1_.4',
-    '1_.4j',
-    # Underscore right after a dot:
-    '1._4',
-    '1._4j',
-    '._5',
-    '._5j',
-    # Underscore right after a sign:
-    '1.0e+_1',
-    '1.0e+_1j',
-    # Underscore right before j:
-    '1.4_j',
-    '1.4e5_j',
-    # Underscore right before e:
-    '1_e1',
-    '1.4_e1',
-    '1.4_e1j',
-    # Underscore right after e:
-    '1e_1',
-    '1.4e_1',
-    '1.4e_1j',
-    # Complex cases with parens:
-    '(1+1.5_j_)',
-    '(1+1.5_j)',
-]
-
 
 class TokenTests(unittest.TestCase):
 

Lib/test/test_hashlib.py

@@ -10,6 +10,7 @@ import hashlib
 import importlib
 import io
 import itertools
+import logging
 import os
 import sys
 import sysconfig
@@ -113,7 +114,11 @@ class HashLibTestCase(unittest.TestCase):
             return importlib.import_module(module_name)
         except ModuleNotFoundError as error:
             if self._warn_on_extension_import and module_name in builtin_hashes:
-                warnings.warn(f'Did a C extension fail to compile? {error}')
+                logging.getLogger(__name__).warning(
+                    'Did a C extension fail to compile? %s',
+                    error,
+                    exc_info=error,
+                )
         return None
 
     def __init__(self, *args, **kwargs):

Lib/test/test_int.py

@@ -4,8 +4,10 @@ import time
 import unittest
 from unittest import mock
 from test import support
-from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
+from test.support.numbers import (
-                               INVALID_UNDERSCORE_LITERALS)
+    VALID_UNDERSCORE_LITERALS,
+    INVALID_UNDERSCORE_LITERALS,
+)
 
 try:
     import _pylong

Lib/test/test_interpreters/utils.py

@@ -2,6 +2,7 @@ from collections import namedtuple
 import contextlib
 import json
 import io
+import logging
 import os
 import os.path
 import pickle
@@ -69,8 +70,8 @@ def pack_exception(exc=None):
 def unpack_exception(packed):
     try:
         data = json.loads(packed)
-    except json.decoder.JSONDecodeError:
+    except json.decoder.JSONDecodeError as e:
-        warnings.warn('incomplete exception data', RuntimeWarning)
+        logging.getLogger(__name__).warning('incomplete exception data', exc_info=e)
         print(packed if isinstance(packed, str) else packed.decode('utf-8'))
         return None
     exc = types.SimpleNamespace(**data)

Lib/test/test_pty.py

@@ -135,8 +135,10 @@ class PtyTest(unittest.TestCase):
                 new_dim = tty.tcgetwinsize(pty.STDIN_FILENO)
                 self.assertEqual(new_dim, target_dim,
                                  "pty.STDIN_FILENO window size unchanged")
-            except OSError:
+            except OSError as e:
-                warnings.warn("Failed to set pty.STDIN_FILENO window size.")
+                logging.getLogger(__name__).warning(
+                    "Failed to set pty.STDIN_FILENO window size.", exc_info=e,
+                )
                 pass
 
         try:

Lib/test/test_socket.py

@@ -27,6 +27,7 @@ import tempfile
 import threading
 import time
 import traceback
+import warnings
 from weakref import proxy
 try:
     import multiprocessing
@@ -199,6 +200,24 @@ def socket_setdefaulttimeout(timeout):
         socket.setdefaulttimeout(old_timeout)
 
 
+@contextlib.contextmanager
+def downgrade_malformed_data_warning():
+    # This warning happens on macos and win, but does not always happen on linux.
+    if sys.platform not in {"win32", "darwin"}:
+        yield
+        return
+
+    with warnings.catch_warnings():
+        # TODO: gh-110012, we should investigate why this warning is happening
+        # and fix it properly.
+        warnings.filterwarnings(
+            action="always",
+            message="received malformed or improperly-truncated ancillary data",
+            category=RuntimeWarning,
+        )
+        yield
+
+
 HAVE_SOCKET_CAN = _have_socket_can()
 
 HAVE_SOCKET_CAN_ISOTP = _have_socket_can_isotp()
@@ -3943,8 +3962,9 @@ class SCMRightsTest(SendrecvmsgServerTimeoutBase):
         # mindata and maxdata bytes when received with buffer size
         # ancbuf, and that any complete file descriptor numbers are
         # valid.
-        msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+        with downgrade_malformed_data_warning():  # TODO: gh-110012
-                                                   len(MSG), ancbuf)
+            msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock,
+                                                       len(MSG), ancbuf)
         self.assertEqual(msg, MSG)
         self.checkRecvmsgAddress(addr, self.cli_addr)
         self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC)
@@ -4295,8 +4315,9 @@ class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase):
         self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
                                   socket.IPV6_RECVHOPLIMIT, 1)
         self.misc_event.set()
-        msg, ancdata, flags, addr = self.doRecvmsg(
+        with downgrade_malformed_data_warning():  # TODO: gh-110012
-            self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1)
+            msg, ancdata, flags, addr = self.doRecvmsg(
+                self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1)
 
         self.assertEqual(msg, MSG)
         self.checkRecvmsgAddress(addr, self.cli_addr)
@@ -4399,9 +4420,10 @@ class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase):
         self.serv_sock.setsockopt(socket.IPPROTO_IPV6,
                                   socket.IPV6_RECVTCLASS, 1)
         self.misc_event.set()
-        msg, ancdata, flags, addr = self.doRecvmsg(
+        with downgrade_malformed_data_warning():  # TODO: gh-110012
-            self.serv_sock, len(MSG),
+            msg, ancdata, flags, addr = self.doRecvmsg(
-            socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1)
+                self.serv_sock, len(MSG),
+                socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1)
 
         self.assertEqual(msg, MSG)
         self.checkRecvmsgAddress(addr, self.cli_addr)

Lib/test/test_support.py

@@ -1,6 +1,8 @@
+import contextlib
 import errno
 import importlib
 import io
+import logging
 import os
 import shutil
 import socket
@@ -19,11 +21,37 @@ from test.support import os_helper
 from test.support import script_helper
 from test.support import socket_helper
 from test.support import warnings_helper
+from test.support.testcase import ExtraAssertions
 
 TESTFN = os_helper.TESTFN
 
 
-class TestSupport(unittest.TestCase):
+class LogCaptureHandler(logging.StreamHandler):
+    # Inspired by pytest's caplog
+    def __init__(self):
+        super().__init__(io.StringIO())
+        self.records = []
+
+    def emit(self, record) -> None:
+        self.records.append(record)
+        super().emit(record)
+
+    def handleError(self, record):
+        raise
+
+
+@contextlib.contextmanager
+def _caplog():
+    handler = LogCaptureHandler()
+    root_logger = logging.getLogger()
+    root_logger.addHandler(handler)
+    try:
+        yield handler
+    finally:
+        root_logger.removeHandler(handler)
+
+
+class TestSupport(unittest.TestCase, ExtraAssertions):
     @classmethod
     def setUpClass(cls):
         orig_filter_len = len(warnings.filters)
@@ -186,7 +214,7 @@ class TestSupport(unittest.TestCase):
         path = os.path.realpath(path)
 
         try:
-            with warnings_helper.check_warnings() as recorder:
+            with warnings_helper.check_warnings() as recorder, _caplog() as caplog:
                 with os_helper.temp_dir(path, quiet=True) as temp_path:
                     self.assertEqual(path, temp_path)
                 warnings = [str(w.message) for w in recorder.warnings]
@@ -195,11 +223,14 @@ class TestSupport(unittest.TestCase):
         finally:
             shutil.rmtree(path)
 
-        self.assertEqual(len(warnings), 1, warnings)
+        self.assertListEqual(warnings, [])
-        warn = warnings[0]
+        self.assertEqual(len(caplog.records), 1)
-        self.assertTrue(warn.startswith(f'tests may fail, unable to create '
+        record = caplog.records[0]
-                                        f'temporary directory {path!r}: '),
+        self.assertStartsWith(
-                        warn)
+            record.getMessage(),
+            f'tests may fail, unable to create '
+            f'temporary directory {path!r}: '
+        )
 
     @support.requires_fork()
     def test_temp_dir__forked_child(self):
@@ -259,35 +290,41 @@ class TestSupport(unittest.TestCase):
 
         with os_helper.temp_dir() as parent_dir:
             bad_dir = os.path.join(parent_dir, 'does_not_exist')
-            with warnings_helper.check_warnings() as recorder:
+            with warnings_helper.check_warnings() as recorder, _caplog() as caplog:
                 with os_helper.change_cwd(bad_dir, quiet=True) as new_cwd:
                     self.assertEqual(new_cwd, original_cwd)
                     self.assertEqual(os.getcwd(), new_cwd)
                 warnings = [str(w.message) for w in recorder.warnings]
 
-        self.assertEqual(len(warnings), 1, warnings)
+        self.assertListEqual(warnings, [])
-        warn = warnings[0]
+        self.assertEqual(len(caplog.records), 1)
-        self.assertTrue(warn.startswith(f'tests may fail, unable to change '
+        record = caplog.records[0]
-                                        f'the current working directory '
+        self.assertStartsWith(
-                                        f'to {bad_dir!r}: '),
+            record.getMessage(),
-                        warn)
+            f'tests may fail, unable to change '
+            f'the current working directory '
+            f'to {bad_dir!r}: '
+        )
 
     # Tests for change_cwd()
 
     def test_change_cwd__chdir_warning(self):
         """Check the warning message when os.chdir() fails."""
         path = TESTFN + '_does_not_exist'
-        with warnings_helper.check_warnings() as recorder:
+        with warnings_helper.check_warnings() as recorder, _caplog() as caplog:
             with os_helper.change_cwd(path=path, quiet=True):
                 pass
             messages = [str(w.message) for w in recorder.warnings]
 
-        self.assertEqual(len(messages), 1, messages)
+        self.assertListEqual(messages, [])
-        msg = messages[0]
+        self.assertEqual(len(caplog.records), 1)
-        self.assertTrue(msg.startswith(f'tests may fail, unable to change '
+        record = caplog.records[0]
-                                       f'the current working directory '
+        self.assertStartsWith(
-                                       f'to {path!r}: '),
+            record.getMessage(),
-                        msg)
+            f'tests may fail, unable to change '
+            f'the current working directory '
+            f'to {path!r}: ',
+        )
 
     # Tests for temp_cwd()
 

Lib/test/test_tokenize.py

@@ -7,10 +7,13 @@ from io import BytesIO, StringIO
 from textwrap import dedent
 from unittest import TestCase, mock
 from test import support
-from test.test_grammar import (VALID_UNDERSCORE_LITERALS,
-                               INVALID_UNDERSCORE_LITERALS)
 from test.support import os_helper
 from test.support.script_helper import run_test_script, make_script, run_python_until_end
+from test.support.numbers import (
+    VALID_UNDERSCORE_LITERALS,
+    INVALID_UNDERSCORE_LITERALS,
+)
+
 
 # Converts a source string into a list of textual representation
 # of the tokens such as: