[3.12] gh-109972: Enhance test_gdb (#110026) (#110351)

* gh-109972: Enhance test_gdb (#110026)

* Split test_pycfunction.py: add test_cfunction_full.py.
  Split the function into the following 6 functions. In verbose
  mode, these "pycfunction" tests now log each tested call.

  * test_pycfunction_noargs()
  * test_pycfunction_o()
  * test_pycfunction_varargs()
  * test_pycfunction_varargs_keywords()
  * test_pycfunction_fastcall()
  * test_pycfunction_fastcall_keywords()

* Move get_gdb_repr() to PrettyPrintTests.
* Replace DebuggerTests.get_sample_script() with SAMPLE_SCRIPT.
* Rename checkout_hook_path to CHECKOUT_HOOK_PATH.
* Rename gdb_version to GDB_VERSION_TEXT.
* Replace (gdb_major_version, gdb_minor_version) with GDB_VERSION.
* run_gdb() uses "backslashreplace" error handler instead of "replace".
* Add check_gdb() function to util.py.
* Enhance support.check_cflags_pgo(): check also for sysconfig
  PGO_PROF_USE_FLAG (if available) in compiler flags.
* Move some SkipTest checks to test_gdb/__init__.py.
* Elaborate why gdb cannot be tested on Windows: gdb doesn't support
  PDB debug symbol files.

(cherry picked from commit 757cbd4f29)

* gh-104736: Fix test_gdb tests on ppc64le with clang (#109360)

Fix test_gdb on Python built with LLVM clang 16 on Linux ppc64le (ex:
Fedora 38). Search patterns in gdb "bt" command output to detect
when gdb fails to retrieve the traceback. For example, skip a test if
"Backtrace stopped: frame did not save the PC" is found.

(cherry picked from commit 44d9a71ea2)

* gh-110166: Fix gdb CFunctionFullTests on ppc64le clang build (#110331)

CFunctionFullTests now also runs "bt" command before "py-bt-full",
similar to CFunctionTests which also runs "bt" command before
"py-bt". So test_gdb can skip the test if patterns like "?? ()" are
found in the gdb output.

(cherry picked from commit 1de9406f91)

Lib/test/support/__init__.py

@@ -785,14 +785,17 @@ def check_cflags_pgo():
     # Check if Python was built with ./configure --enable-optimizations:
     # with Profile Guided Optimization (PGO).
     cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or ''
-    pgo_options = (
+    pgo_options = [
         # GCC
         '-fprofile-use',
         # clang: -fprofile-instr-use=code.profclangd
         '-fprofile-instr-use',
         # ICC
         "-prof-use",
-    )
+    ]
+    PGO_PROF_USE_FLAG = sysconfig.get_config_var('PGO_PROF_USE_FLAG')
+    if PGO_PROF_USE_FLAG:
+        pgo_options.append(PGO_PROF_USE_FLAG)
     return any(option in cflags_nodist for option in pgo_options)
 
 

Lib/test/test_gdb/__init__.py

@@ -4,7 +4,27 @@
 # Lib/test/test_jit_gdb.py
 
 import os
-from test.support import load_package_tests
+import sysconfig
+import unittest
+from test import support
+
+
+MS_WINDOWS = (os.name == 'nt')
+if MS_WINDOWS:
+    # On Windows, Python is usually built by MSVC. Passing /p:DebugSymbols=true
+    # option to MSBuild produces PDB debug symbols, but gdb doesn't support PDB
+    # debug symbol files.
+    raise unittest.SkipTest("test_gdb doesn't work on Windows")
+
+if support.PGO:
+    raise unittest.SkipTest("test_gdb is not useful for PGO")
+
+if not sysconfig.is_python_build():
+    raise unittest.SkipTest("test_gdb only works on source builds at the moment.")
+
+if support.check_cflags_pgo():
+    raise unittest.SkipTest("test_gdb is not reliable on PGO builds")
+
 
 def load_tests(*args):
-    return load_package_tests(os.path.dirname(__file__), *args)
+    return support.load_package_tests(os.path.dirname(__file__), *args)

Lib/test/test_gdb/test_backtrace.py

@@ -3,7 +3,7 @@ import unittest
 from test import support
 from test.support import python_is_optimized
 
-from .util import setup_module, DebuggerTests, CET_PROTECTION
+from .util import setup_module, DebuggerTests, CET_PROTECTION, SAMPLE_SCRIPT
 
 
 def setUpModule():
@@ -15,7 +15,7 @@ class PyBtTests(DebuggerTests):
                      "Python was compiled with optimizations")
     def test_bt(self):
         'Verify that the "py-bt" command works'
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-bt'])
         self.assertMultilineMatches(bt,
                                     r'''^.*
@@ -35,7 +35,7 @@ Traceback \(most recent call first\):
                      "Python was compiled with optimizations")
     def test_bt_full(self):
         'Verify that the "py-bt-full" command works'
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-bt-full'])
         self.assertMultilineMatches(bt,
                                     r'''^.*

Lib/test/test_gdb/test_cfunction.py

@@ -1,8 +1,6 @@
-import re
 import textwrap
 import unittest
 from test import support
-from test.support import python_is_optimized
 
 from .util import setup_module, DebuggerTests
 
@@ -11,10 +9,22 @@ def setUpModule():
     setup_module()
 
 
-@unittest.skipIf(python_is_optimized(),
+@unittest.skipIf(support.python_is_optimized(),
                  "Python was compiled with optimizations")
 @support.requires_resource('cpu')
 class CFunctionTests(DebuggerTests):
+    def check(self, func_name, cmd):
+        # Verify with "py-bt":
+        gdb_output = self.get_stack_trace(
+            cmd,
+            breakpoint=func_name,
+            cmds_after_breakpoint=['bt', 'py-bt'],
+            # bpo-45207: Ignore 'Function "meth_varargs" not
+            # defined.' message in stderr.
+            ignore_stderr=True,
+        )
+        self.assertIn(f'<built-in method {func_name}', gdb_output)
+
     # Some older versions of gdb will fail with
     #  "Cannot find new threads: generic error"
     # unless we add LD_PRELOAD=PATH-TO-libpthread.so.1 as a workaround
@@ -24,60 +34,52 @@ class CFunctionTests(DebuggerTests):
     # This is because we are calling functions from an "external" module
     # (_testcapimodule) rather than compiled-in functions. It seems difficult
     # to suppress these. See also the comment in DebuggerTests.get_stack_trace
-    def test_pycfunction(self):
+    def check_pycfunction(self, func_name, args):
         'Verify that "py-bt" displays invocations of PyCFunction instances'
-        # bpo-46600: If the compiler inlines _null_to_none() in meth_varargs()
+
-        # (ex: clang -Og), _null_to_none() is the frame #1. Otherwise,
+        if support.verbose:
-        # meth_varargs() is the frame #1.
+            print()
-        expected_frame = r'#(1|2)'
+
         # Various optimizations multiply the code paths by which these are
         # called, so test a variety of calling conventions.
-        for func_name, args in (
+        for obj in (
-            ('meth_varargs', ''),
+            '_testcapi',
-            ('meth_varargs_keywords', ''),
+            '_testcapi.MethClass',
-            ('meth_o', '[]'),
+            '_testcapi.MethClass()',
-            ('meth_noargs', ''),
+            '_testcapi.MethStatic()',
-            ('meth_fastcall', ''),
+
-            ('meth_fastcall_keywords', ''),
+            # XXX: bound methods don't yet give nice tracebacks
+            # '_testcapi.MethInstance()',
         ):
-            for obj in (
+            with self.subTest(f'{obj}.{func_name}'):
-                '_testcapi',
+                call = f'{obj}.{func_name}({args})'
-                '_testcapi.MethClass',
+                cmd = textwrap.dedent(f'''
-                '_testcapi.MethClass()',
+                    import _testcapi
-                '_testcapi.MethStatic()',
+                    def foo():
+                        {call}
+                    def bar():
+                        foo()
+                    bar()
+                ''')
+                if support.verbose:
+                    print(f'  test call: {call}', flush=True)
 
-                # XXX: bound methods don't yet give nice tracebacks
+                self.check(func_name, cmd)
-                # '_testcapi.MethInstance()',
-            ):
-                with self.subTest(f'{obj}.{func_name}'):
-                    cmd = textwrap.dedent(f'''
-                        import _testcapi
-                        def foo():
-                            {obj}.{func_name}({args})
-                        def bar():
-                            foo()
-                        bar()
-                    ''')
-                    # Verify with "py-bt":
-                    gdb_output = self.get_stack_trace(
-                        cmd,
-                        breakpoint=func_name,
-                        cmds_after_breakpoint=['bt', 'py-bt'],
-                        # bpo-45207: Ignore 'Function "meth_varargs" not
-                        # defined.' message in stderr.
-                        ignore_stderr=True,
-                    )
-                    self.assertIn(f'<built-in method {func_name}', gdb_output)
 
-                    # Verify with "py-bt-full":
+    def test_pycfunction_noargs(self):
-                    gdb_output = self.get_stack_trace(
+        self.check_pycfunction('meth_noargs', '')
-                        cmd,
+
-                        breakpoint=func_name,
+    def test_pycfunction_o(self):
-                        cmds_after_breakpoint=['py-bt-full'],
+        self.check_pycfunction('meth_o', '[]')
-                        # bpo-45207: Ignore 'Function "meth_varargs" not
+
-                        # defined.' message in stderr.
+    def test_pycfunction_varargs(self):
-                        ignore_stderr=True,
+        self.check_pycfunction('meth_varargs', '')
-                    )
+
-                    regex = expected_frame
+    def test_pycfunction_varargs_keywords(self):
-                    regex += re.escape(f' <built-in method {func_name}')
+        self.check_pycfunction('meth_varargs_keywords', '')
-                    self.assertRegex(gdb_output, regex)
+
+    def test_pycfunction_fastcall(self):
+        self.check_pycfunction('meth_fastcall', '')
+
+    def test_pycfunction_fastcall_keywords(self):
+        self.check_pycfunction('meth_fastcall_keywords', '')

Lib/test/test_gdb/test_cfunction_full.py

@@ -0,0 +1,36 @@
+"""
+Similar to test_cfunction but test "py-bt-full" command.
+"""
+
+import re
+
+from .util import setup_module
+from .test_cfunction import CFunctionTests
+
+
+def setUpModule():
+    setup_module()
+
+
+class CFunctionFullTests(CFunctionTests):
+    def check(self, func_name, cmd):
+        # Verify with "py-bt-full":
+        gdb_output = self.get_stack_trace(
+            cmd,
+            breakpoint=func_name,
+            cmds_after_breakpoint=['bt', 'py-bt-full'],
+            # bpo-45207: Ignore 'Function "meth_varargs" not
+            # defined.' message in stderr.
+            ignore_stderr=True,
+        )
+
+        # bpo-46600: If the compiler inlines _null_to_none() in
+        # meth_varargs() (ex: clang -Og), _null_to_none() is the
+        # frame #1. Otherwise, meth_varargs() is the frame #1.
+        regex = r'#(1|2)'
+        regex += re.escape(f' <built-in method {func_name}')
+        self.assertRegex(gdb_output, regex)
+
+
+# Delete the test case, otherwise it's executed twice
+del CFunctionTests

Lib/test/test_gdb/test_misc.py

@@ -2,7 +2,7 @@ import re
 import unittest
 from test.support import python_is_optimized
 
-from .util import run_gdb, setup_module, DebuggerTests
+from .util import run_gdb, setup_module, DebuggerTests, SAMPLE_SCRIPT
 
 
 def setUpModule():
@@ -32,7 +32,7 @@ class PyListTests(DebuggerTests):
 
     def test_basic_command(self):
         'Verify that the "py-list" command works'
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-list'])
 
         self.assertListing('   5    \n'
@@ -47,7 +47,7 @@ class PyListTests(DebuggerTests):
 
     def test_one_abs_arg(self):
         'Verify the "py-list" command with one absolute argument'
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-list 9'])
 
         self.assertListing('   9    def baz(*args):\n'
@@ -58,7 +58,7 @@ class PyListTests(DebuggerTests):
 
     def test_two_abs_args(self):
         'Verify the "py-list" command with two absolute arguments'
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-list 1,3'])
 
         self.assertListing('   1    # Sample script for use by test_gdb\n'
@@ -101,7 +101,7 @@ $''')
     @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands")
     def test_down_at_bottom(self):
         'Verify handling of "py-down" at the bottom of the stack'
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-down'])
         self.assertEndsWith(bt,
                             'Unable to find a newer python frame\n')
@@ -109,7 +109,7 @@ $''')
     @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands")
     def test_up_at_top(self):
         'Verify handling of "py-up" at the top of the stack'
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-up'] * 5)
         self.assertEndsWith(bt,
                             'Unable to find an older python frame\n')
@@ -150,7 +150,7 @@ class PyPrintTests(DebuggerTests):
     @unittest.skipIf(python_is_optimized(),
                      "Python was compiled with optimizations")
     def test_printing_global(self):
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-up', 'py-print __name__'])
         self.assertMultilineMatches(bt,
                                     r".*\nglobal '__name__' = '__main__'\n.*")
@@ -158,7 +158,7 @@ class PyPrintTests(DebuggerTests):
     @unittest.skipIf(python_is_optimized(),
                      "Python was compiled with optimizations")
     def test_printing_builtin(self):
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-up', 'py-print len'])
         self.assertMultilineMatches(bt,
                                     r".*\nbuiltin 'len' = <built-in method len of module object at remote 0x-?[0-9a-f]+>\n.*")
@@ -167,7 +167,7 @@ class PyLocalsTests(DebuggerTests):
     @unittest.skipIf(python_is_optimized(),
                      "Python was compiled with optimizations")
     def test_basic_command(self):
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-up', 'py-locals'])
         self.assertMultilineMatches(bt,
                                     r".*\nargs = \(1, 2, 3\)\n.*")
@@ -176,7 +176,7 @@ class PyLocalsTests(DebuggerTests):
     @unittest.skipIf(python_is_optimized(),
                      "Python was compiled with optimizations")
     def test_locals_after_up(self):
-        bt = self.get_stack_trace(script=self.get_sample_script(),
+        bt = self.get_stack_trace(script=SAMPLE_SCRIPT,
                                   cmds_after_breakpoint=['py-up', 'py-up', 'py-locals'])
         self.assertMultilineMatches(bt,
                                     r'''^.*

Lib/test/test_gdb/test_pretty_print.py

@@ -3,7 +3,7 @@ import sys
 from test import support
 
 from .util import (
-    BREAKPOINT_FN, gdb_major_version, gdb_minor_version,
+    BREAKPOINT_FN, GDB_VERSION,
     run_gdb, setup_module, DebuggerTests)
 
 
@@ -12,6 +12,42 @@ def setUpModule():
 
 
 class PrettyPrintTests(DebuggerTests):
+    def get_gdb_repr(self, source,
+                     cmds_after_breakpoint=None,
+                     import_site=False):
+        # Given an input python source representation of data,
+        # run "python -c'id(DATA)'" under gdb with a breakpoint on
+        # builtin_id and scrape out gdb's representation of the "op"
+        # parameter, and verify that the gdb displays the same string
+        #
+        # Verify that the gdb displays the expected string
+        #
+        # For a nested structure, the first time we hit the breakpoint will
+        # give us the top-level structure
+
+        # NOTE: avoid decoding too much of the traceback as some
+        # undecodable characters may lurk there in optimized mode
+        # (issue #19743).
+        cmds_after_breakpoint = cmds_after_breakpoint or ["backtrace 1"]
+        gdb_output = self.get_stack_trace(source, breakpoint=BREAKPOINT_FN,
+                                          cmds_after_breakpoint=cmds_after_breakpoint,
+                                          import_site=import_site)
+        # gdb can insert additional '\n' and space characters in various places
+        # in its output, depending on the width of the terminal it's connected
+        # to (using its "wrap_here" function)
+        m = re.search(
+            # Match '#0 builtin_id(self=..., v=...)'
+            r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)'
+            # Match ' at Python/bltinmodule.c'.
+            # bpo-38239: builtin_id() is defined in Python/bltinmodule.c,
+            # but accept any "Directory\file.c" to support Link Time
+            # Optimization (LTO).
+            r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c',
+            gdb_output, re.DOTALL)
+        if not m:
+            self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output))
+        return m.group(1), gdb_output
+
     def test_getting_backtrace(self):
         gdb_output = self.get_stack_trace('id(42)')
         self.assertTrue(BREAKPOINT_FN in gdb_output)
@@ -75,15 +111,17 @@ class PrettyPrintTests(DebuggerTests):
         # as GDB might have been linked against a different version
         # of Python with a different encoding and coercion policy
         # with respect to PEP 538 and PEP 540.
-        out, err = run_gdb(
+        stdout, stderr = run_gdb(
             '--eval-command',
             'python import locale; print(locale.getpreferredencoding())')
 
-        encoding = out.rstrip()
+        encoding = stdout
-        if err or not encoding:
+        if stderr or not encoding:
             raise RuntimeError(
-                f'unable to determine the preferred encoding '
+                f'unable to determine the Python locale preferred encoding '
-                f'of embedded Python in GDB: {err}')
+                f'of embedded Python in GDB\n'
+                f'stdout={stdout!r}\n'
+                f'stderr={stderr!r}')
 
         def check_repr(text):
             try:
@@ -122,7 +160,7 @@ class PrettyPrintTests(DebuggerTests):
     @support.requires_resource('cpu')
     def test_sets(self):
         'Verify the pretty-printing of sets'
-        if (gdb_major_version, gdb_minor_version) < (7, 3):
+        if GDB_VERSION < (7, 3):
             self.skipTest("pretty-printing of sets needs gdb 7.3 or later")
         self.assertGdbRepr(set(), "set()")
         self.assertGdbRepr(set(['a']), "{'a'}")
@@ -141,7 +179,7 @@ id(s)''')
     @support.requires_resource('cpu')
     def test_frozensets(self):
         'Verify the pretty-printing of frozensets'
-        if (gdb_major_version, gdb_minor_version) < (7, 3):
+        if GDB_VERSION < (7, 3):
             self.skipTest("pretty-printing of frozensets needs gdb 7.3 or later")
         self.assertGdbRepr(frozenset(), "frozenset()")
         self.assertGdbRepr(frozenset(['a']), "frozenset({'a'})")

Lib/test/test_gdb/util.py

@@ -1,5 +1,6 @@
 import os
 import re
+import shlex
 import subprocess
 import sys
 import sysconfig
@@ -7,29 +8,74 @@ import unittest
 from test import support
 
 
-MS_WINDOWS = (sys.platform == 'win32')
+# Location of custom hooks file in a repository checkout.
-if MS_WINDOWS:
+CHECKOUT_HOOK_PATH = os.path.join(os.path.dirname(sys.executable),
-    raise unittest.SkipTest("test_gdb doesn't work on Windows")
+                                  'python-gdb.py')
+
+SAMPLE_SCRIPT = os.path.join(os.path.dirname(__file__), 'gdb_sample.py')
+BREAKPOINT_FN = 'builtin_id'
+
+PYTHONHASHSEED = '123'
+
+
+def clean_environment():
+    # Remove PYTHON* environment variables such as PYTHONHOME
+    return {name: value for name, value in os.environ.items()
+            if not name.startswith('PYTHON')}
+
+
+# Temporary value until it's initialized by get_gdb_version() below
+GDB_VERSION = (0, 0)
+
+def run_gdb(*args, exitcode=0, **env_vars):
+    """Runs gdb in --batch mode with the additional arguments given by *args.
+
+    Returns its (stdout, stderr) decoded from utf-8 using the replace handler.
+    """
+    env = clean_environment()
+    if env_vars:
+        env.update(env_vars)
+
+    cmd = ['gdb',
+           # Batch mode: Exit after processing all the command files
+           # specified with -x/--command
+           '--batch',
+            # -nx: Do not execute commands from any .gdbinit initialization
+            # files (gh-66384)
+           '-nx']
+    if GDB_VERSION >= (7, 4):
+        cmd.extend(('--init-eval-command',
+                    f'add-auto-load-safe-path {CHECKOUT_HOOK_PATH}'))
+    cmd.extend(args)
+
+    proc = subprocess.run(
+        cmd,
+        # Redirect stdin to prevent gdb from messing with the terminal settings
+        stdin=subprocess.PIPE,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        encoding="utf8", errors="backslashreplace",
+        env=env)
+
+    stdout = proc.stdout
+    stderr = proc.stderr
+    if proc.returncode != exitcode:
+        cmd_text = shlex.join(cmd)
+        raise Exception(f"{cmd_text} failed with exit code {proc.returncode}, "
+                        f"expected exit code {exitcode}:\n"
+                        f"stdout={stdout!r}\n"
+                        f"stderr={stderr!r}")
+
+    return (stdout, stderr)
 
 
 def get_gdb_version():
     try:
-        cmd = ["gdb", "-nx", "--version"]
+        stdout, stderr = run_gdb('--version')
-        proc = subprocess.Popen(cmd,
-                                stdout=subprocess.PIPE,
-                                stderr=subprocess.PIPE,
-                                universal_newlines=True)
-        with proc:
-            version, stderr = proc.communicate()
-
-        if proc.returncode:
-            raise Exception(f"Command {' '.join(cmd)!r} failed "
-                            f"with exit code {proc.returncode}: "
-                            f"stdout={version!r} stderr={stderr!r}")
     except OSError:
         # This is what "no gdb" looks like.  There may, however, be other
         # errors that manifest this way too.
-        raise unittest.SkipTest("Couldn't find gdb on the path")
+        raise unittest.SkipTest("Couldn't find gdb program on the path")
 
     # Regex to parse:
     # 'GNU gdb (GDB; SUSE Linux Enterprise 12) 7.7\n' -> 7.7
@@ -37,32 +83,48 @@ def get_gdb_version():
     # 'GNU gdb 6.1.1 [FreeBSD]\n' -> 6.1
     # 'GNU gdb (GDB) Fedora (7.5.1-37.fc18)\n' -> 7.5
     # 'HP gdb 6.7 for HP Itanium (32 or 64 bit) and target HP-UX 11iv2 and 11iv3.\n' -> 6.7
-    match = re.search(r"^(?:GNU|HP) gdb.*?\b(\d+)\.(\d+)", version)
+    match = re.search(r"^(?:GNU|HP) gdb.*?\b(\d+)\.(\d+)", stdout)
     if match is None:
-        raise Exception("unable to parse GDB version: %r" % version)
+        raise Exception("unable to parse gdb version: %r" % stdout)
-    return (version, int(match.group(1)), int(match.group(2)))
+    version_text = stdout
+    major = int(match.group(1))
+    minor = int(match.group(2))
+    version = (major, minor)
+    return (version_text, version)
 
-gdb_version, gdb_major_version, gdb_minor_version = get_gdb_version()
+GDB_VERSION_TEXT, GDB_VERSION = get_gdb_version()
-if gdb_major_version < 7:
+if GDB_VERSION < (7, 0):
-    raise unittest.SkipTest("gdb versions before 7.0 didn't support python "
+    raise unittest.SkipTest(
-                            "embedding. Saw %s.%s:\n%s"
+        f"gdb versions before 7.0 didn't support python embedding. "
-                            % (gdb_major_version, gdb_minor_version,
+        f"Saw gdb version {GDB_VERSION[0]}.{GDB_VERSION[1]}:\n"
-                               gdb_version))
+        f"{GDB_VERSION_TEXT}")
-
-if not sysconfig.is_python_build():
-    raise unittest.SkipTest("test_gdb only works on source builds at the moment.")
-
-if ((sysconfig.get_config_var('PGO_PROF_USE_FLAG') or 'xxx') in
-    (sysconfig.get_config_var('PY_CORE_CFLAGS') or '')):
-    raise unittest.SkipTest("test_gdb is not reliable on PGO builds")
-
-# Location of custom hooks file in a repository checkout.
-checkout_hook_path = os.path.join(os.path.dirname(sys.executable),
-                                  'python-gdb.py')
-
-PYTHONHASHSEED = '123'
 
 
+def check_usable_gdb():
+    # Verify that "gdb" was built with the embedded Python support enabled and
+    # verify that "gdb" can load our custom hooks, as OS security settings may
+    # disallow this without a customized .gdbinit.
+    stdout, stderr = run_gdb(
+        '--eval-command=python import sys; print(sys.version_info)',
+        '--args', sys.executable)
+
+    if "auto-loading has been declined" in stderr:
+        raise unittest.SkipTest(
+            f"gdb security settings prevent use of custom hooks; "
+            f"stderr: {stderr!r}")
+
+    if not stdout:
+        raise unittest.SkipTest(
+            f"gdb not built with embedded python support; "
+            f"stderr: {stderr!r}")
+
+    if "major=2" in stdout:
+        raise unittest.SkipTest("gdb built with Python 2")
+
+check_usable_gdb()
+
+
+# Control-flow enforcement technology
 def cet_protection():
     cflags = sysconfig.get_config_var('CFLAGS')
     if not cflags:
@@ -74,63 +136,17 @@ def cet_protection():
             and any((flag.startswith('-fcf-protection')
                      and not flag.endswith(("=none", "=return")))
                     for flag in flags))
-
-# Control-flow enforcement technology
 CET_PROTECTION = cet_protection()
 
 
-def run_gdb(*args, **env_vars):
-    """Runs gdb in --batch mode with the additional arguments given by *args.
-
-    Returns its (stdout, stderr) decoded from utf-8 using the replace handler.
-    """
-    if env_vars:
-        env = os.environ.copy()
-        env.update(env_vars)
-    else:
-        env = None
-    # -nx: Do not execute commands from any .gdbinit initialization files
-    #      (issue #22188)
-    base_cmd = ('gdb', '--batch', '-nx')
-    if (gdb_major_version, gdb_minor_version) >= (7, 4):
-        base_cmd += ('-iex', 'add-auto-load-safe-path ' + checkout_hook_path)
-    proc = subprocess.Popen(base_cmd + args,
-                            # Redirect stdin to prevent GDB from messing with
-                            # the terminal settings
-                            stdin=subprocess.PIPE,
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE,
-                            env=env)
-    with proc:
-        out, err = proc.communicate()
-    return out.decode('utf-8', 'replace'), err.decode('utf-8', 'replace')
-
-# Verify that "gdb" was built with the embedded python support enabled:
-gdbpy_version, _ = run_gdb("--eval-command=python import sys; print(sys.version_info)")
-if not gdbpy_version:
-    raise unittest.SkipTest("gdb not built with embedded python support")
-
-if "major=2" in gdbpy_version:
-    raise unittest.SkipTest("gdb built with Python 2")
-
-# Verify that "gdb" can load our custom hooks, as OS security settings may
-# disallow this without a customized .gdbinit.
-_, gdbpy_errors = run_gdb('--args', sys.executable)
-if "auto-loading has been declined" in gdbpy_errors:
-    msg = "gdb security settings prevent use of custom hooks: "
-    raise unittest.SkipTest(msg + gdbpy_errors.rstrip())
-
-BREAKPOINT_FN='builtin_id'
-
-
 def setup_module():
     if support.verbose:
-        print("GDB version %s.%s:" % (gdb_major_version, gdb_minor_version))
+        print(f"gdb version {GDB_VERSION[0]}.{GDB_VERSION[1]}:")
-        for line in gdb_version.splitlines():
+        for line in GDB_VERSION_TEXT.splitlines():
             print(" " * 4 + line)
+        print()
 
 
-@unittest.skipIf(support.PGO, "not useful for PGO")
 class DebuggerTests(unittest.TestCase):
 
     """Test that the debugger can debug Python."""
@@ -163,20 +179,22 @@ class DebuggerTests(unittest.TestCase):
         # structures
 
         # Generate a list of commands in gdb's language:
-        commands = ['set breakpoint pending yes',
+        commands = [
-                    'break %s' % breakpoint,
+            'set breakpoint pending yes',
+            'break %s' % breakpoint,
 
-                    # The tests assume that the first frame of printed
+            # The tests assume that the first frame of printed
-                    #  backtrace will not contain program counter,
+            #  backtrace will not contain program counter,
-                    #  that is however not guaranteed by gdb
+            #  that is however not guaranteed by gdb
-                    #  therefore we need to use 'set print address off' to
+            #  therefore we need to use 'set print address off' to
-                    #  make sure the counter is not there. For example:
+            #  make sure the counter is not there. For example:
-                    # #0 in PyObject_Print ...
+            # #0 in PyObject_Print ...
-                    #  is assumed, but sometimes this can be e.g.
+            #  is assumed, but sometimes this can be e.g.
-                    # #0 0x00003fffb7dd1798 in PyObject_Print ...
+            # #0 0x00003fffb7dd1798 in PyObject_Print ...
-                    'set print address off',
+            'set print address off',
 
-                    'run']
+            'run',
+        ]
 
         # GDB as of 7.4 onwards can distinguish between the
         # value of a variable at entry vs current value:
@@ -184,7 +202,7 @@ class DebuggerTests(unittest.TestCase):
         # which leads to the selftests failing with errors like this:
         #   AssertionError: 'v@entry=()' != '()'
         # Disable this:
-        if (gdb_major_version, gdb_minor_version) >= (7, 4):
+        if GDB_VERSION >= (7, 4):
             commands += ['set print entry-values no']
 
         if cmds_after_breakpoint:
@@ -237,13 +255,16 @@ class DebuggerTests(unittest.TestCase):
         for pattern in (
             '(frame information optimized out)',
             'Unable to read information on python frame',
+
             # gh-91960: On Python built with "clang -Og", gdb gets
             # "frame=<optimized out>" for _PyEval_EvalFrameDefault() parameter
             '(unable to read python frame information)',
+
             # gh-104736: On Python built with "clang -Og" on ppc64le,
             # "py-bt" displays a truncated or not traceback, but "where"
             # logs this error message:
             'Backtrace stopped: frame did not save the PC',
+
             # gh-104736: When "bt" command displays something like:
             # "#1  0x0000000000000000 in ?? ()", the traceback is likely
             # truncated or wrong.
@@ -254,42 +275,6 @@ class DebuggerTests(unittest.TestCase):
 
         return out
 
-    def get_gdb_repr(self, source,
-                     cmds_after_breakpoint=None,
-                     import_site=False):
-        # Given an input python source representation of data,
-        # run "python -c'id(DATA)'" under gdb with a breakpoint on
-        # builtin_id and scrape out gdb's representation of the "op"
-        # parameter, and verify that the gdb displays the same string
-        #
-        # Verify that the gdb displays the expected string
-        #
-        # For a nested structure, the first time we hit the breakpoint will
-        # give us the top-level structure
-
-        # NOTE: avoid decoding too much of the traceback as some
-        # undecodable characters may lurk there in optimized mode
-        # (issue #19743).
-        cmds_after_breakpoint = cmds_after_breakpoint or ["backtrace 1"]
-        gdb_output = self.get_stack_trace(source, breakpoint=BREAKPOINT_FN,
-                                          cmds_after_breakpoint=cmds_after_breakpoint,
-                                          import_site=import_site)
-        # gdb can insert additional '\n' and space characters in various places
-        # in its output, depending on the width of the terminal it's connected
-        # to (using its "wrap_here" function)
-        m = re.search(
-            # Match '#0 builtin_id(self=..., v=...)'
-            r'#0\s+builtin_id\s+\(self\=.*,\s+v=\s*(.*?)?\)'
-            # Match ' at Python/bltinmodule.c'.
-            # bpo-38239: builtin_id() is defined in Python/bltinmodule.c,
-            # but accept any "Directory\file.c" to support Link Time
-            # Optimization (LTO).
-            r'\s+at\s+\S*[A-Za-z]+/[A-Za-z0-9_-]+\.c',
-            gdb_output, re.DOTALL)
-        if not m:
-            self.fail('Unexpected gdb output: %r\n%s' % (gdb_output, gdb_output))
-        return m.group(1), gdb_output
-
     def assertEndsWith(self, actual, exp_end):
         '''Ensure that the given "actual" string ends with "exp_end"'''
         self.assertTrue(actual.endswith(exp_end),
@@ -299,6 +284,3 @@ class DebuggerTests(unittest.TestCase):
         m = re.match(pattern, actual, re.DOTALL)
         if not m:
             self.fail(msg='%r did not match %r' % (actual, pattern))
-
-    def get_sample_script(self):
-        return os.path.join(os.path.dirname(__file__), 'gdb_sample.py')

Misc/NEWS.d/next/Tests/2023-09-13-05-58-09.gh-issue-104736.lA25Fu.rst

@@ -0,0 +1,4 @@
+Fix test_gdb on Python built with LLVM clang 16 on Linux ppc64le (ex: Fedora
+38). Search patterns in gdb "bt" command output to detect when gdb fails to
+retrieve the traceback. For example, skip a test if ``Backtrace stopped: frame
+did not save the PC`` is found. Patch by Victor Stinner.