Issue #13628: python-gdb.py is now able to retrieve more frames in the Python

traceback if Python is optimized.

 * delay the lookup of the size_t type, it is not available at startup
 * The second argument of the PyFrameObjectPtr constructor is optional, as
   done in other constructors
 * iter_builtins() and iter_globals() methods of PyFrameObjectPtr returns
   an empty tuple instead of None if Python is optimized
 * Fix py-bt and py-bt-full to handle correctly "optimized" frames
 * Frame.get_pyop() tries to get the frame pointer from PyEval_EvalCodeEx()
   if the pointer is optimized out in PyEval_EvalFrameEx()

Lib/test/test_gdb.py

@@ -529,6 +529,8 @@ id(foo.__code__)''',
                                  re.DOTALL),
                         'Unexpected gdb representation: %r\n%s' % (gdb_output, gdb_output))
 
+@unittest.skipIf(python_is_optimized(),
+                 "Python was compiled with optimizations")
 class PyListTests(DebuggerTests):
     def assertListing(self, expected, actual):
         self.assertEndsWith(actual, expected)
@@ -571,6 +573,8 @@ class PyListTests(DebuggerTests):
 
 class StackNavigationTests(DebuggerTests):
     @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands")
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
     def test_pyup_command(self):
         'Verify that the "py-up" command works'
         bt = self.get_stack_trace(script=self.get_sample_script(),
@@ -598,6 +602,8 @@ $''')
                             'Unable to find an older python frame\n')
 
     @unittest.skipUnless(HAS_PYUP_PYDOWN, "test requires py-up/py-down commands")
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
     def test_up_then_down(self):
         'Verify "py-up" followed by "py-down"'
         bt = self.get_stack_trace(script=self.get_sample_script(),
@@ -611,6 +617,8 @@ $''')
 $''')
 
 class PyBtTests(DebuggerTests):
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
     def test_bt(self):
         'Verify that the "py-bt" command works'
         bt = self.get_stack_trace(script=self.get_sample_script(),
@@ -628,6 +636,8 @@ Traceback \(most recent call first\):
     foo\(1, 2, 3\)
 ''')
 
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
     def test_bt_full(self):
         'Verify that the "py-bt-full" command works'
         bt = self.get_stack_trace(script=self.get_sample_script(),
@@ -639,10 +649,12 @@ Traceback \(most recent call first\):
 #[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 4, in foo \(a=1, b=2, c=3\)
     bar\(a, b, c\)
 #[0-9]+ Frame 0x[0-9a-f]+, for file .*gdb_sample.py, line 12, in <module> \(\)
-foo\(1, 2, 3\)
+    foo\(1, 2, 3\)
 ''')
 
 class PyPrintTests(DebuggerTests):
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
     def test_basic_command(self):
         'Verify that the "py-print" command works'
         bt = self.get_stack_trace(script=self.get_sample_script(),
@@ -657,12 +669,16 @@ class PyPrintTests(DebuggerTests):
         self.assertMultilineMatches(bt,
                                     r".*\nlocal 'c' = 3\nlocal 'b' = 2\nlocal 'a' = 1\n.*")
 
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
     def test_printing_global(self):
         bt = self.get_stack_trace(script=self.get_sample_script(),
                                   cmds_after_breakpoint=['py-print __name__'])
         self.assertMultilineMatches(bt,
                                     r".*\nglobal '__name__' = '__main__'\n.*")
 
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
     def test_printing_builtin(self):
         bt = self.get_stack_trace(script=self.get_sample_script(),
                                   cmds_after_breakpoint=['py-print len'])
@@ -670,6 +686,8 @@ class PyPrintTests(DebuggerTests):
                                     r".*\nbuiltin 'len' = <built-in method len of module object at remote 0x[0-9a-f]+>\n.*")
 
 class PyLocalsTests(DebuggerTests):
+    @unittest.skipIf(python_is_optimized(),
+                     "Python was compiled with optimizations")
     def test_basic_command(self):
         bt = self.get_stack_trace(script=self.get_sample_script(),
                                   cmds_after_breakpoint=['py-locals'])
@@ -684,8 +702,6 @@ class PyLocalsTests(DebuggerTests):
                                     r".*\na = 1\nb = 2\nc = 3\n.*")
 
 def test_main():
-    if python_is_optimized():
-        raise unittest.SkipTest("Python was compiled with optimizations")
     run_unittest(PrettyPrintTests,
                  PyListTests,
                  StackNavigationTests,

Misc/NEWS

@@ -97,7 +97,7 @@ Core and Builtins
 Library
 -------
 
-- Issue #11813: Fix inspect.getattr_static for modules. Patch by Andreas 
+- Issue #11813: Fix inspect.getattr_static for modules. Patch by Andreas
   Stührk.
 
 - Issue #7502: Fix equality comparison for DocTestCase instances.  Patch by
@@ -314,6 +314,12 @@ Build
 
 - Issue #13326: Clean __pycache__ directories correctly on OpenBSD.
 
+Tools/Demos
+-----------
+
+- Issue #13628: python-gdb.py is now able to retrieve more frames in the Python
+  traceback if Python is optimized.
+
 Tests
 -----
 

Tools/gdb/libpython.py

@@ -49,7 +49,6 @@ import sys
 _type_char_ptr = gdb.lookup_type('char').pointer() # char*
 _type_unsigned_char_ptr = gdb.lookup_type('unsigned char').pointer() # unsigned char*
 _type_void_ptr = gdb.lookup_type('void').pointer() # void*
-_type_size_t = gdb.lookup_type('size_t')
 
 SIZEOF_VOID_P = _type_void_ptr.sizeof
 
@@ -435,11 +434,15 @@ class InstanceProxy(object):
                                             self.address)
 
 def _PyObject_VAR_SIZE(typeobj, nitems):
+    if _PyObject_VAR_SIZE._type_size_t is None:
+        _PyObject_VAR_SIZE._type_size_t = gdb.lookup_type('size_t')
+
     return ( ( typeobj.field('tp_basicsize') +
                nitems * typeobj.field('tp_itemsize') +
                (SIZEOF_VOID_P - 1)
              ) & ~(SIZEOF_VOID_P - 1)
-           ).cast(_type_size_t)
+           ).cast(_PyObject_VAR_SIZE._type_size_t)
+_PyObject_VAR_SIZE._type_size_t = None
 
 class HeapTypeObjectPtr(PyObjectPtr):
     _typename = 'PyObject'
@@ -806,7 +809,7 @@ class PyNoneStructPtr(PyObjectPtr):
 class PyFrameObjectPtr(PyObjectPtr):
     _typename = 'PyFrameObject'
 
-    def __init__(self, gdbval, cast_to):
+    def __init__(self, gdbval, cast_to=None):
         PyObjectPtr.__init__(self, gdbval, cast_to)
 
         if not self.is_optimized_out():
@@ -840,7 +843,7 @@ class PyFrameObjectPtr(PyObjectPtr):
         the global variables of this frame
         '''
         if self.is_optimized_out():
-            return
+            return ()
 
         pyop_globals = self.pyop_field('f_globals')
         return pyop_globals.iteritems()
@@ -851,7 +854,7 @@ class PyFrameObjectPtr(PyObjectPtr):
         the builtin variables
         '''
         if self.is_optimized_out():
-            return
+            return ()
 
         pyop_builtins = self.pyop_field('f_builtins')
         return pyop_builtins.iteritems()
@@ -938,6 +941,7 @@ class PyFrameObjectPtr(PyObjectPtr):
     def print_traceback(self):
         if self.is_optimized_out():
             sys.stdout.write('  (frame information optimized out)\n')
+            return
         visited = set()
         sys.stdout.write('  File "%s", line %i, in %s\n'
                   % (self.co_filename.proxyval(visited),
@@ -1403,7 +1407,20 @@ class Frame(object):
     def get_pyop(self):
         try:
             f = self._gdbframe.read_var('f')
-            return PyFrameObjectPtr.from_pyobject_ptr(f)
+            frame = PyFrameObjectPtr.from_pyobject_ptr(f)
+            if not frame.is_optimized_out():
+                return frame
+            # gdb is unable to get the "f" argument of PyEval_EvalFrameEx()
+            # because it was "optimized out". Try to get "f" from the frame
+            # of the caller, PyEval_EvalCodeEx().
+            orig_frame = frame
+            caller = self._gdbframe.older()
+            if caller:
+                f = caller.read_var('f')
+                frame = PyFrameObjectPtr.from_pyobject_ptr(f)
+                if not frame.is_optimized_out():
+                    return frame
+            return orig_frame
         except ValueError:
             return None
 
@@ -1434,9 +1451,10 @@ class Frame(object):
             if pyop:
                 line = pyop.get_truncated_repr(MAX_OUTPUT_LEN)
                 write_unicode(sys.stdout, '#%i %s\n' % (self.get_index(), line))
-                line = pyop.current_line()
+                if not pyop.is_optimized_out():
-                if line is not None:
+                    line = pyop.current_line()
-                    sys.stdout.write(line)
+                    if line is not None:
+                        sys.stdout.write('    %s\n' % line.strip())
             else:
                 sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index())
         else:
@@ -1447,9 +1465,10 @@ class Frame(object):
             pyop = self.get_pyop()
             if pyop:
                 pyop.print_traceback()
-                line = pyop.current_line()
+                if not pyop.is_optimized_out():
-                if line is not None:
+                    line = pyop.current_line()
-                    sys.stdout.write('    %s\n' % line.strip())
+                    if line is not None:
+                        sys.stdout.write('    %s\n' % line.strip())
             else:
                 sys.stdout.write('  (unable to read python frame information)\n')
         else:
@@ -1495,7 +1514,7 @@ class PyList(gdb.Command):
             return
 
         pyop = frame.get_pyop()
-        if not pyop:
+        if not pyop or pyop.is_optimized_out():
             print 'Unable to read information on python frame'
             return