gh-110721: Use the traceback module for PyErr_Display() and fallback to the C implementation (#110702)

Include/internal/pycore_traceback.h

@@ -95,9 +95,8 @@ extern PyObject* _PyTraceBack_FromFrame(
 
 /* Write the traceback tb to file f. Prefix each line with
    indent spaces followed by the margin (if it is not NULL). */
-extern int _PyTraceBack_Print_Indented(
+extern int _PyTraceBack_Print(
-    PyObject *tb, int indent, const char* margin,
+    PyObject *tb, const char *header, PyObject *f);
-    const char *header_margin, const char *header, PyObject *f);
 extern int _Py_WriteIndentedMargin(int, const char*, PyObject *);
 extern int _Py_WriteIndent(int, PyObject *);
 

Lib/test/test_sys.py

@@ -175,7 +175,8 @@ class ExceptHookTest(unittest.TestCase):
 
     def test_excepthook(self):
         with test.support.captured_output("stderr") as stderr:
-            sys.excepthook(1, '1', 1)
+            with test.support.catch_unraisable_exception():
+                sys.excepthook(1, '1', 1)
         self.assertTrue("TypeError: print_exception(): Exception expected for " \
                          "value, str found" in stderr.getvalue())
 

Lib/test/test_traceback.py

@@ -963,7 +963,9 @@ class CPythonTracebackLegacyErrorCaretTests(
     Same set of tests as above but with Python's legacy internal traceback printing.
     """
 
-class TracebackFormatTests(unittest.TestCase):
+
+class TracebackFormatMixin:
+    DEBUG_RANGES = True
 
     def some_exception(self):
         raise KeyError('blah')
@@ -1137,6 +1139,8 @@ class TracebackFormatTests(unittest.TestCase):
         )
         expected = (tb_line + result_g).splitlines()
         actual = stderr_g.getvalue().splitlines()
+        if not self.DEBUG_RANGES:
+            expected = [line for line in expected if not set(line.strip()) == {"^"}]
         self.assertEqual(actual, expected)
 
         # Check 2 different repetitive sections
@@ -1173,6 +1177,8 @@ class TracebackFormatTests(unittest.TestCase):
         )
         expected = (result_h + result_g).splitlines()
         actual = stderr_h.getvalue().splitlines()
+        if not self.DEBUG_RANGES:
+            expected = [line for line in expected if not set(line.strip()) == {"^"}]
         self.assertEqual(actual, expected)
 
         # Check the boundary conditions. First, test just below the cutoff.
@@ -1199,11 +1205,13 @@ class TracebackFormatTests(unittest.TestCase):
         )
         tb_line = (
             'Traceback (most recent call last):\n'
-            f'  File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n'
+            f'  File "{__file__}", line {lineno_g+81}, in _check_recursive_traceback_display\n'
             '    g(traceback._RECURSIVE_CUTOFF)\n'
         )
         expected = (tb_line + result_g).splitlines()
         actual = stderr_g.getvalue().splitlines()
+        if not self.DEBUG_RANGES:
+            expected = [line for line in expected if not set(line.strip()) == {"^"}]
         self.assertEqual(actual, expected)
 
         # Second, test just above the cutoff.
@@ -1231,24 +1239,24 @@ class TracebackFormatTests(unittest.TestCase):
         )
         tb_line = (
             'Traceback (most recent call last):\n'
-            f'  File "{__file__}", line {lineno_g+108}, in _check_recursive_traceback_display\n'
+            f'  File "{__file__}", line {lineno_g+114}, in _check_recursive_traceback_display\n'
             '    g(traceback._RECURSIVE_CUTOFF + 1)\n'
         )
         expected = (tb_line + result_g).splitlines()
         actual = stderr_g.getvalue().splitlines()
+        if not self.DEBUG_RANGES:
+            expected = [line for line in expected if not set(line.strip()) == {"^"}]
         self.assertEqual(actual, expected)
 
     @requires_debug_ranges()
-    def test_recursive_traceback_python(self):
+    def test_recursive_traceback(self):
-        self._check_recursive_traceback_display(traceback.print_exc)
+        if self.DEBUG_RANGES:
-
+            self._check_recursive_traceback_display(traceback.print_exc)
-    @cpython_only
+        else:
-    @requires_debug_ranges()
+            from _testcapi import exception_print
-    def test_recursive_traceback_cpython_internal(self):
+            def render_exc():
-        from _testcapi import exception_print
+                exception_print(sys.exception())
-        def render_exc():
+            self._check_recursive_traceback_display(render_exc)
-            exception_print(sys.exception())
-        self._check_recursive_traceback_display(render_exc)
 
     def test_format_stack(self):
         def fmt():
@@ -1321,7 +1329,8 @@ class TracebackFormatTests(unittest.TestCase):
     def test_print_exception_bad_type_capi(self):
         from _testcapi import exception_print
         with captured_output("stderr") as stderr:
-            exception_print(42)
+            with support.catch_unraisable_exception():
+                exception_print(42)
         self.assertEqual(
             stderr.getvalue(),
             ('TypeError: print_exception(): '
@@ -1345,6 +1354,24 @@ context_message = (
 boundaries = re.compile(
     '(%s|%s)' % (re.escape(cause_message), re.escape(context_message)))
 
+class TestTracebackFormat(unittest.TestCase, TracebackFormatMixin):
+    pass
+
+@cpython_only
+class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixin):
+    DEBUG_RANGES = False
+    def setUp(self) -> None:
+        self.original_unraisable_hook = sys.unraisablehook
+        sys.unraisablehook = lambda *args: None
+        self.original_hook = traceback._print_exception_bltin
+        traceback._print_exception_bltin = lambda *args: 1/0
+        return super().setUp()
+
+    def tearDown(self) -> None:
+        traceback._print_exception_bltin = self.original_hook
+        sys.unraisablehook = self.original_unraisable_hook
+        return super().tearDown()
+
 class BaseExceptionReportingTests:
 
     def get_exception(self, exception_or_callable):

Lib/traceback.py

@@ -125,6 +125,14 @@ def print_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \
     te.print(file=file, chain=chain)
 
 
+BUILTIN_EXCEPTION_LIMIT = object()
+
+
+def _print_exception_bltin(exc, /):
+    file = sys.stderr if sys.stderr is not None else sys.__stderr__
+    return print_exception(exc, limit=BUILTIN_EXCEPTION_LIMIT, file=file)
+
+
 def format_exception(exc, /, value=_sentinel, tb=_sentinel, limit=None, \
                      chain=True):
     """Format a stack trace and the exception information.
@@ -406,12 +414,16 @@ class StackSummary(list):
         # (frame, (lineno, end_lineno, colno, end_colno)) in the stack.
         # Only lineno is required, the remaining fields can be None if the
         # information is not available.
-        if limit is None:
+        builtin_limit = limit is BUILTIN_EXCEPTION_LIMIT
+        if limit is None or builtin_limit:
             limit = getattr(sys, 'tracebacklimit', None)
             if limit is not None and limit < 0:
                 limit = 0
         if limit is not None:
-            if limit >= 0:
+            if builtin_limit:
+                frame_gen = tuple(frame_gen)
+                frame_gen = frame_gen[len(frame_gen) - limit:]
+            elif limit >= 0:
                 frame_gen = itertools.islice(frame_gen, limit)
             else:
                 frame_gen = collections.deque(frame_gen, maxlen=-limit)
@@ -741,9 +753,9 @@ class TracebackException:
                 wrong_name = getattr(exc_value, "name", None)
                 if wrong_name is not None and wrong_name in sys.stdlib_module_names:
                     if suggestion:
-                        self._str += f" Or did you forget to import '{wrong_name}'"
+                        self._str += f" Or did you forget to import '{wrong_name}'?"
                     else:
-                        self._str += f". Did you forget to import '{wrong_name}'"
+                        self._str += f". Did you forget to import '{wrong_name}'?"
         if lookup_lines:
             self._load_lines()
         self.__suppress_context__ = \
@@ -904,7 +916,11 @@ class TracebackException:
             if self.offset is not None:
                 offset = self.offset
                 end_offset = self.end_offset if self.end_offset not in {None, 0} else offset
-                if offset == end_offset or end_offset == -1:
+                if self.text and offset > len(self.text):
+                    offset = len(self.text) + 1
+                if self.text and end_offset > len(self.text):
+                    end_offset = len(self.text) + 1
+                if offset >= end_offset or end_offset < 0:
                     end_offset = offset + 1
 
                 # Convert 1-based column offset to 0-based index into stripped text

Misc/NEWS.d/next/Core and Builtins/2023-10-11-16-56-54.gh-issue-110721.afcSsH.rst

@@ -0,0 +1,2 @@
+Use the :mod:`traceback` implementation for the default
+:c:func:`PyErr_Display` functionality. Patch by Pablo Galindo

Python/pythonrun.c

@@ -23,7 +23,7 @@
 #include "pycore_pystate.h"       // _PyInterpreterState_GET()
 #include "pycore_pythonrun.h"     // export _PyRun_InteractiveLoopObject()
 #include "pycore_sysmodule.h"     // _PySys_Audit()
-#include "pycore_traceback.h"     // _PyTraceBack_Print_Indented()
+#include "pycore_traceback.h"     // _PyTraceBack_Print()
 
 #include "errcode.h"              // E_EOF
 #include "marshal.h"              // PyMarshal_ReadLongFromFile()
@@ -517,204 +517,6 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags)
     return 0;
 }
 
-static int
-parse_syntax_error(PyObject *err, PyObject **message, PyObject **filename,
-                   Py_ssize_t *lineno, Py_ssize_t *offset,
-                   Py_ssize_t* end_lineno, Py_ssize_t* end_offset,
-                   PyObject **text)
-{
-    Py_ssize_t hold;
-    PyObject *v;
-
-    *message = NULL;
-    *filename = NULL;
-
-    /* new style errors.  `err' is an instance */
-    *message = PyObject_GetAttr(err, &_Py_ID(msg));
-    if (!*message)
-        goto finally;
-
-    v = PyObject_GetAttr(err, &_Py_ID(filename));
-    if (!v)
-        goto finally;
-    if (v == Py_None) {
-        Py_DECREF(v);
-        _Py_DECLARE_STR(anon_string, "<string>");
-        *filename = Py_NewRef(&_Py_STR(anon_string));
-    }
-    else {
-        *filename = v;
-    }
-
-    v = PyObject_GetAttr(err, &_Py_ID(lineno));
-    if (!v)
-        goto finally;
-    hold = PyLong_AsSsize_t(v);
-    Py_DECREF(v);
-    if (hold < 0 && PyErr_Occurred())
-        goto finally;
-    *lineno = hold;
-
-    v = PyObject_GetAttr(err, &_Py_ID(offset));
-    if (!v)
-        goto finally;
-    if (v == Py_None) {
-        *offset = -1;
-        Py_DECREF(v);
-    } else {
-        hold = PyLong_AsSsize_t(v);
-        Py_DECREF(v);
-        if (hold < 0 && PyErr_Occurred())
-            goto finally;
-        *offset = hold;
-    }
-
-    if (Py_TYPE(err) == (PyTypeObject*)PyExc_SyntaxError) {
-        v = PyObject_GetAttr(err, &_Py_ID(end_lineno));
-        if (!v) {
-            PyErr_Clear();
-            *end_lineno = *lineno;
-        }
-        else if (v == Py_None) {
-            *end_lineno = *lineno;
-            Py_DECREF(v);
-        } else {
-            hold = PyLong_AsSsize_t(v);
-            Py_DECREF(v);
-            if (hold < 0 && PyErr_Occurred())
-                goto finally;
-            *end_lineno = hold;
-        }
-
-        v = PyObject_GetAttr(err, &_Py_ID(end_offset));
-        if (!v) {
-            PyErr_Clear();
-            *end_offset = -1;
-        }
-        else if (v == Py_None) {
-            *end_offset = -1;
-            Py_DECREF(v);
-        } else {
-            hold = PyLong_AsSsize_t(v);
-            Py_DECREF(v);
-            if (hold < 0 && PyErr_Occurred())
-                goto finally;
-            *end_offset = hold;
-        }
-    } else {
-        // SyntaxError subclasses
-        *end_lineno = *lineno;
-        *end_offset = -1;
-    }
-
-    v = PyObject_GetAttr(err, &_Py_ID(text));
-    if (!v)
-        goto finally;
-    if (v == Py_None) {
-        Py_DECREF(v);
-        *text = NULL;
-    }
-    else {
-        *text = v;
-    }
-    return 1;
-
-finally:
-    Py_XDECREF(*message);
-    Py_XDECREF(*filename);
-    return 0;
-}
-
-static int
-print_error_text(PyObject *f, Py_ssize_t offset, Py_ssize_t end_offset,
-                 PyObject *text_obj)
-{
-    size_t caret_repetitions = (end_offset > 0 && end_offset > offset) ?
-                               end_offset - offset : 1;
-
-    /* Convert text to a char pointer; return if error */
-    const char *text = PyUnicode_AsUTF8(text_obj);
-    if (text == NULL) {
-        return -1;
-    }
-
-    /* Convert offset from 1-based to 0-based */
-    offset--;
-
-    /* Strip leading whitespace from text, adjusting offset as we go */
-    while (*text == ' ' || *text == '\t' || *text == '\f') {
-        text++;
-        offset--;
-    }
-
-    /* Calculate text length excluding trailing newline */
-    Py_ssize_t len = strlen(text);
-    if (len > 0 && text[len-1] == '\n') {
-        len--;
-    }
-
-    /* Clip offset to at most len */
-    if (offset > len) {
-        offset = len;
-    }
-
-    /* Skip past newlines embedded in text */
-    for (;;) {
-        const char *nl = strchr(text, '\n');
-        if (nl == NULL) {
-            break;
-        }
-        Py_ssize_t inl = nl - text;
-        if (inl >= offset) {
-            break;
-        }
-        inl += 1;
-        text += inl;
-        len -= inl;
-        offset -= (int)inl;
-    }
-
-    /* Print text */
-    if (PyFile_WriteString("    ", f) < 0) {
-        return -1;
-    }
-    if (PyFile_WriteString(text, f) < 0) {
-        return -1;
-    }
-
-    /* Make sure there's a newline at the end */
-    if (text[len] != '\n') {
-        if (PyFile_WriteString("\n", f) < 0) {
-            return -1;
-        }
-    }
-
-    /* Don't print caret if it points to the left of the text */
-    if (offset < 0) {
-        return 0;
-    }
-
-    /* Write caret line */
-    if (PyFile_WriteString("    ", f) < 0) {
-        return -1;
-    }
-    while (--offset >= 0) {
-        if (PyFile_WriteString(" ", f) < 0) {
-            return -1;
-        }
-    }
-    for (size_t caret_iter=0; caret_iter < caret_repetitions ; caret_iter++) {
-        if (PyFile_WriteString("^", f) < 0) {
-            return -1;
-        }
-    }
-    if (PyFile_WriteString("\n", f) < 0) {
-        return -1;
-    }
-    return 0;
-}
-
-
 int
 _Py_HandleSystemExit(int *exitcode_p)
 {
@@ -883,29 +685,13 @@ struct exception_print_context
 {
     PyObject *file;
     PyObject *seen;            // Prevent cycles in recursion
-    int exception_group_depth; // nesting level of current exception group
-    bool need_close;           // Need a closing bottom frame
-    int max_group_width;       // Maximum number of children of each EG
-    int max_group_depth;       // Maximum nesting level of EGs
 };
 
-#define EXC_MARGIN(ctx) ((ctx)->exception_group_depth ? "| " : "")
-#define EXC_INDENT(ctx) (2 * (ctx)->exception_group_depth)
-
-static int
-write_indented_margin(struct exception_print_context *ctx, PyObject *f)
-{
-    return _Py_WriteIndentedMargin(EXC_INDENT(ctx), EXC_MARGIN(ctx), f);
-}
-
 static int
 print_exception_invalid_type(struct exception_print_context *ctx,
                              PyObject *value)
 {
     PyObject *f = ctx->file;
-    if (_Py_WriteIndent(EXC_INDENT(ctx), f) < 0) {
-        return -1;
-    }
     const char *const msg = "TypeError: print_exception(): Exception expected "
                             "for value, ";
     if (PyFile_WriteString(msg, f) < 0) {
@@ -929,15 +715,7 @@ print_exception_traceback(struct exception_print_context *ctx, PyObject *value)
     PyObject *tb = PyException_GetTraceback(value);
     if (tb && tb != Py_None) {
         const char *header = EXCEPTION_TB_HEADER;
-        const char *header_margin = EXC_MARGIN(ctx);
+        err = _PyTraceBack_Print(tb, header, f);
-        if (_PyBaseExceptionGroup_Check(value)) {
-            header = EXCEPTION_GROUP_TB_HEADER;
-            if (ctx->exception_group_depth == 1) {
-                header_margin = "+ ";
-            }
-        }
-        err = _PyTraceBack_Print_Indented(
-            tb, EXC_INDENT(ctx), EXC_MARGIN(ctx), header_margin, header, f);
     }
     Py_XDECREF(tb);
     return err;
@@ -959,16 +737,20 @@ print_exception_file_and_line(struct exception_print_context *ctx,
     }
     Py_DECREF(tmp);
 
-    PyObject *message, *filename, *text;
+    PyObject *filename = NULL;
-    Py_ssize_t lineno, offset, end_lineno, end_offset;
+    Py_ssize_t lineno = 0;
-    if (!parse_syntax_error(*value_p, &message, &filename,
+    PyObject* v = PyObject_GetAttr(*value_p, &_Py_ID(filename));
-                            &lineno, &offset,
+    if (!v) {
-                            &end_lineno, &end_offset, &text)) {
+        return -1;
-        PyErr_Clear();
+    }
-        return 0;
+    if (v == Py_None) {
+        Py_DECREF(v);
+        _Py_DECLARE_STR(anon_string, "<string>");
+        filename = Py_NewRef(&_Py_STR(anon_string));
+    }
+    else {
+        filename = v;
     }
-
-    Py_SETREF(*value_p, message);
 
     PyObject *line = PyUnicode_FromFormat("  File \"%S\", line %zd\n",
                                           filename, lineno);
@@ -976,40 +758,16 @@ print_exception_file_and_line(struct exception_print_context *ctx,
     if (line == NULL) {
         goto error;
     }
-    if (write_indented_margin(ctx, f) < 0) {
-        goto error;
-    }
     if (PyFile_WriteObject(line, f, Py_PRINT_RAW) < 0) {
         goto error;
     }
     Py_CLEAR(line);
 
-    if (text != NULL) {
-        Py_ssize_t line_size;
-        const char *error_line = PyUnicode_AsUTF8AndSize(text, &line_size);
-        // If the location of the error spawn multiple lines, we want
-        // to just print the first one and highlight everything until
-        // the end of that one since we don't support multi-line error
-        // messages.
-        if (end_lineno > lineno) {
-            end_offset = (error_line != NULL) ? line_size : -1;
-        }
-        // Limit the amount of '^' that we can display to
-        // the size of the text in the source line.
-        if (error_line != NULL && end_offset > line_size + 1) {
-            end_offset = line_size + 1;
-        }
-        if (print_error_text(f, offset, end_offset, text) < 0) {
-            goto error;
-        }
-        Py_DECREF(text);
-    }
     assert(!PyErr_Occurred());
     return 0;
 
 error:
     Py_XDECREF(line);
-    Py_XDECREF(text);
     return -1;
 }
 
@@ -1020,11 +778,14 @@ print_exception_message(struct exception_print_context *ctx, PyObject *type,
 {
     PyObject *f = ctx->file;
 
-    assert(PyExceptionClass_Check(type));
+    if (PyErr_GivenExceptionMatches(value, PyExc_MemoryError)) {
-
+        // The Python APIs in this function require allocating memory
-    if (write_indented_margin(ctx, f) < 0) {
+        // for various objects. If we're out of memory, we can't do that,
         return -1;
     }
+
+    assert(PyExceptionClass_Check(type));
+
     PyObject *modulename = PyObject_GetAttr(type, &_Py_ID(__module__));
     if (modulename == NULL || !PyUnicode_Check(modulename)) {
         Py_XDECREF(modulename);
@@ -1098,104 +859,9 @@ print_exception_message(struct exception_print_context *ctx, PyObject *type,
     return 0;
 }
 
-static int
-print_exception_suggestions(struct exception_print_context *ctx,
-                            PyObject *value)
-{
-    PyObject *f = ctx->file;
-    PyObject *suggestions = _Py_Offer_Suggestions(value);
-    if (suggestions) {
-        if (PyFile_WriteObject(suggestions, f, Py_PRINT_RAW) < 0) {
-            goto error;
-        }
-        Py_DECREF(suggestions);
-    }
-    else if (PyErr_Occurred()) {
-        PyErr_Clear();
-    }
-    return 0;
-error:
-    Py_XDECREF(suggestions);
-    return -1;
-}
-
-static int
-print_exception_notes(struct exception_print_context *ctx, PyObject *notes)
-{
-    PyObject *f = ctx->file;
-
-    if (notes == NULL) {
-        return 0;
-    }
-
-    if (!PySequence_Check(notes) || PyUnicode_Check(notes) || PyBytes_Check(notes)) {
-        int res = 0;
-        if (write_indented_margin(ctx, f) < 0) {
-            res = -1;
-        }
-        PyObject *s = PyObject_Repr(notes);
-        if (s == NULL) {
-            PyErr_Clear();
-            res = PyFile_WriteString("<__notes__ repr() failed>", f);
-        }
-        else {
-            res = PyFile_WriteObject(s, f, Py_PRINT_RAW);
-            Py_DECREF(s);
-        }
-        if (PyFile_WriteString("\n", f) < 0) {
-            res = -1;
-        }
-        return res;
-    }
-    Py_ssize_t num_notes = PySequence_Length(notes);
-    PyObject *lines = NULL;
-    for (Py_ssize_t ni = 0; ni < num_notes; ni++) {
-        PyObject *note = PySequence_GetItem(notes, ni);
-        PyObject *note_str = PyObject_Str(note);
-        Py_DECREF(note);
-
-        if (note_str == NULL) {
-            PyErr_Clear();
-            if (PyFile_WriteString("<note str() failed>", f) < 0) {
-                goto error;
-            }
-        }
-        else {
-            lines = PyUnicode_Splitlines(note_str, 1);
-            Py_DECREF(note_str);
-
-            if (lines == NULL) {
-                goto error;
-            }
-
-            Py_ssize_t n = PyList_GET_SIZE(lines);
-            for (Py_ssize_t i = 0; i < n; i++) {
-                PyObject *line = PyList_GET_ITEM(lines, i);
-                assert(PyUnicode_Check(line));
-                if (write_indented_margin(ctx, f) < 0) {
-                    goto error;
-                }
-                if (PyFile_WriteObject(line, f, Py_PRINT_RAW) < 0) {
-                    goto error;
-                }
-            }
-            Py_CLEAR(lines);
-        }
-        if (PyFile_WriteString("\n", f) < 0) {
-            goto error;
-        }
-    }
-
-    return 0;
-error:
-    Py_XDECREF(lines);
-    return -1;
-}
-
 static int
 print_exception(struct exception_print_context *ctx, PyObject *value)
 {
-    PyObject *notes = NULL;
     PyObject *f = ctx->file;
 
     if (!PyExceptionInstance_Check(value)) {
@@ -1211,9 +877,6 @@ print_exception(struct exception_print_context *ctx, PyObject *value)
 
     /* grab the type and notes now because value can change below */
     PyObject *type = (PyObject *) Py_TYPE(value);
-    if (PyObject_GetOptionalAttr(value, &_Py_ID(__notes__), &notes) < 0) {
-        goto error;
-    }
 
     if (print_exception_file_and_line(ctx, &value) < 0) {
         goto error;
@@ -1221,22 +884,13 @@ print_exception(struct exception_print_context *ctx, PyObject *value)
     if (print_exception_message(ctx, type, value) < 0) {
         goto error;
     }
-    if (print_exception_suggestions(ctx, value) < 0) {
-        goto error;
-    }
     if (PyFile_WriteString("\n", f) < 0) {
         goto error;
     }
-    if (print_exception_notes(ctx, notes) < 0) {
-        goto error;
-    }
-
-    Py_XDECREF(notes);
     Py_DECREF(value);
     assert(!PyErr_Occurred());
     return 0;
 error:
-    Py_XDECREF(notes);
     Py_DECREF(value);
     return -1;
 }
@@ -1260,29 +914,18 @@ print_chained(struct exception_print_context* ctx, PyObject *value,
     if (_Py_EnterRecursiveCall(" in print_chained")) {
         return -1;
     }
-    bool need_close = ctx->need_close;
     int res = print_exception_recursive(ctx, value);
-    ctx->need_close = need_close;
     _Py_LeaveRecursiveCall();
     if (res < 0) {
         return -1;
     }
 
-    if (write_indented_margin(ctx, f) < 0) {
-        return -1;
-    }
     if (PyFile_WriteString("\n", f) < 0) {
         return -1;
     }
-    if (write_indented_margin(ctx, f) < 0) {
-        return -1;
-    }
     if (PyFile_WriteString(message, f) < 0) {
         return -1;
     }
-    if (write_indented_margin(ctx, f) < 0) {
-        return -1;
-    }
     if (PyFile_WriteString("\n", f) < 0) {
         return -1;
     }
@@ -1359,133 +1002,6 @@ print_exception_cause_and_context(struct exception_print_context *ctx,
     return 0;
 }
 
-static int
-print_exception_group(struct exception_print_context *ctx, PyObject *value)
-{
-    PyObject *f = ctx->file;
-
-    if (ctx->exception_group_depth > ctx->max_group_depth) {
-        /* depth exceeds limit */
-
-        if (write_indented_margin(ctx, f) < 0) {
-            return -1;
-        }
-
-        PyObject *line = PyUnicode_FromFormat("... (max_group_depth is %d)\n",
-                                              ctx->max_group_depth);
-        if (line == NULL) {
-            return -1;
-        }
-        int err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
-        Py_DECREF(line);
-        return err;
-    }
-
-    if (ctx->exception_group_depth == 0) {
-        ctx->exception_group_depth += 1;
-    }
-    if (print_exception(ctx, value) < 0) {
-        return -1;
-    }
-
-    PyObject *excs = ((PyBaseExceptionGroupObject *)value)->excs;
-    assert(excs && PyTuple_Check(excs));
-    Py_ssize_t num_excs = PyTuple_GET_SIZE(excs);
-    assert(num_excs > 0);
-    Py_ssize_t n;
-    if (num_excs <= ctx->max_group_width) {
-        n = num_excs;
-    }
-    else {
-        n = ctx->max_group_width + 1;
-    }
-
-    ctx->need_close = false;
-    for (Py_ssize_t i = 0; i < n; i++) {
-        bool last_exc = (i == n - 1);
-        if (last_exc) {
-            // The closing frame may be added in a recursive call
-            ctx->need_close = true;
-        }
-
-        if (_Py_WriteIndent(EXC_INDENT(ctx), f) < 0) {
-            return -1;
-        }
-        bool truncated = (i >= ctx->max_group_width);
-        PyObject *line;
-        if (!truncated) {
-            line = PyUnicode_FromFormat(
-                "%s+---------------- %zd ----------------\n",
-                (i == 0) ? "+-" : "  ", i + 1);
-        }
-        else {
-            line = PyUnicode_FromFormat(
-                "%s+---------------- ... ----------------\n",
-                (i == 0) ? "+-" : "  ");
-        }
-        if (line == NULL) {
-            return -1;
-        }
-        int err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
-        Py_DECREF(line);
-        if (err < 0) {
-            return -1;
-        }
-
-        ctx->exception_group_depth += 1;
-        PyObject *exc = PyTuple_GET_ITEM(excs, i);
-
-        if (!truncated) {
-            if (_Py_EnterRecursiveCall(" in print_exception_group")) {
-                return -1;
-            }
-            int res = print_exception_recursive(ctx, exc);
-            _Py_LeaveRecursiveCall();
-            if (res < 0) {
-                return -1;
-            }
-        }
-        else {
-            Py_ssize_t excs_remaining = num_excs - ctx->max_group_width;
-
-            if (write_indented_margin(ctx, f) < 0) {
-                return -1;
-            }
-
-            PyObject *line = PyUnicode_FromFormat(
-                "and %zd more exception%s\n",
-                excs_remaining, excs_remaining > 1 ? "s" : "");
-
-            if (line == NULL) {
-                return -1;
-            }
-
-            int err = PyFile_WriteObject(line, f, Py_PRINT_RAW);
-            Py_DECREF(line);
-            if (err < 0) {
-                return -1;
-            }
-        }
-
-        if (last_exc && ctx->need_close) {
-            if (_Py_WriteIndent(EXC_INDENT(ctx), f) < 0) {
-                return -1;
-            }
-            if (PyFile_WriteString(
-                    "+------------------------------------\n", f) < 0) {
-                return -1;
-            }
-            ctx->need_close = false;
-        }
-        ctx->exception_group_depth -= 1;
-    }
-
-    if (ctx->exception_group_depth == 1) {
-        ctx->exception_group_depth -= 1;
-    }
-    return 0;
-}
-
 static int
 print_exception_recursive(struct exception_print_context *ctx, PyObject *value)
 {
@@ -1498,12 +1014,7 @@ print_exception_recursive(struct exception_print_context *ctx, PyObject *value)
             goto error;
         }
     }
-    if (!_PyBaseExceptionGroup_Check(value)) {
+    if (print_exception(ctx, value) < 0) {
-        if (print_exception(ctx, value) < 0) {
-            goto error;
-        }
-    }
-    else if (print_exception_group(ctx, value) < 0) {
         goto error;
     }
     assert(!PyErr_Occurred());
@@ -1515,9 +1026,6 @@ error:
     return -1;
 }
 
-#define PyErr_MAX_GROUP_WIDTH 15
-#define PyErr_MAX_GROUP_DEPTH 10
-
 void
 _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb)
 {
@@ -1535,12 +1043,45 @@ _PyErr_Display(PyObject *file, PyObject *unused, PyObject *value, PyObject *tb)
         }
     }
 
+    int unhandled_keyboard_interrupt = _PyRuntime.signals.unhandled_keyboard_interrupt;
+
+    if (!value || PyErr_GivenExceptionMatches(value, PyExc_MemoryError)) {
+        goto fallback;
+    }
+
+    // Try first with the stdlib traceback module
+    PyObject *traceback_module = PyImport_ImportModule("traceback");
+
+    if (traceback_module == NULL) {
+        goto fallback;
+    }
+
+    PyObject *print_exception_fn = PyObject_GetAttrString(traceback_module, "_print_exception_bltin");
+
+    if (print_exception_fn == NULL || !PyCallable_Check(print_exception_fn)) {
+        Py_DECREF(traceback_module);
+        goto fallback;
+    }
+
+    PyObject* result = PyObject_CallOneArg(print_exception_fn, value);
+
+    Py_DECREF(traceback_module);
+    Py_XDECREF(print_exception_fn);
+    if (result) {
+        Py_DECREF(result);
+        _PyRuntime.signals.unhandled_keyboard_interrupt = unhandled_keyboard_interrupt;
+        return;
+    }
+fallback:
+    _PyRuntime.signals.unhandled_keyboard_interrupt = unhandled_keyboard_interrupt;
+#ifdef Py_DEBUG
+     if (PyErr_Occurred()) {
+        _PyErr_WriteUnraisableMsg("in the internal traceback machinery", NULL);
+     }
+#endif
+    PyErr_Clear();
     struct exception_print_context ctx;
     ctx.file = file;
-    ctx.exception_group_depth = 0;
-    ctx.need_close = false;
-    ctx.max_group_width = PyErr_MAX_GROUP_WIDTH;
-    ctx.max_group_depth = PyErr_MAX_GROUP_DEPTH;
 
     /* We choose to ignore seen being possibly NULL, and report
        at least the main exception (it could be a MemoryError).

Python/traceback.c

@@ -397,27 +397,9 @@ _Py_WriteIndent(int indent, PyObject *f)
     return 0;
 }
 
-/* Writes indent spaces, followed by the margin if it is not `\0`.
-   Returns 0 on success and non-zero on failure.
- */
-int
-_Py_WriteIndentedMargin(int indent, const char *margin, PyObject *f)
-{
-    if (_Py_WriteIndent(indent, f) < 0) {
-        return -1;
-    }
-    if (margin) {
-        if (PyFile_WriteString(margin, f) < 0) {
-            return -1;
-        }
-    }
-    return 0;
-}
-
 static int
-display_source_line_with_margin(PyObject *f, PyObject *filename, int lineno, int indent,
+display_source_line(PyObject *f, PyObject *filename, int lineno, int indent,
-                                int margin_indent, const char *margin,
+                    int *truncation, PyObject **line)
-                                int *truncation, PyObject **line)
 {
     int fd;
     int i;
@@ -545,10 +527,6 @@ display_source_line_with_margin(PyObject *f, PyObject *filename, int lineno, int
         *truncation = i - indent;
     }
 
-    if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) {
-        goto error;
-    }
-
     /* Write some spaces before the line */
     if (_Py_WriteIndent(indent, f) < 0) {
         goto error;
@@ -574,161 +552,11 @@ int
 _Py_DisplaySourceLine(PyObject *f, PyObject *filename, int lineno, int indent,
                       int *truncation, PyObject **line)
 {
-    return display_source_line_with_margin(f, filename, lineno, indent, 0,
+    return display_source_line(f, filename, lineno, indent, truncation, line);
-                                           NULL, truncation, line);
 }
 
-/* AST based Traceback Specialization
- *
- * When displaying a new traceback line, for certain syntactical constructs
- * (e.g a subscript, an arithmetic operation) we try to create a representation
- * that separates the primary source of error from the rest.
- *
- * Example specialization of BinOp nodes:
- *  Traceback (most recent call last):
- *    File "/home/isidentical/cpython/cpython/t.py", line 10, in <module>
- *      add_values(1, 2, 'x', 3, 4)
- *    File "/home/isidentical/cpython/cpython/t.py", line 2, in add_values
- *      return a + b + c + d + e
- *             ~~~~~~^~~
- *  TypeError: 'NoneType' object is not subscriptable
- */
 
 #define IS_WHITESPACE(c) (((c) == ' ') || ((c) == '\t') || ((c) == '\f'))
-
-static int
-extract_anchors_from_expr(const char *segment_str, expr_ty expr, Py_ssize_t *left_anchor, Py_ssize_t *right_anchor,
-                          char** primary_error_char, char** secondary_error_char)
-{
-    switch (expr->kind) {
-        case BinOp_kind: {
-            expr_ty left = expr->v.BinOp.left;
-            expr_ty right = expr->v.BinOp.right;
-            for (int i = left->end_col_offset; i < right->col_offset; i++) {
-                if (IS_WHITESPACE(segment_str[i])) {
-                    continue;
-                }
-
-                *left_anchor = i;
-                *right_anchor = i + 1;
-
-                // Check whether if this a two-character operator (e.g //)
-                if (i + 1 < right->col_offset && !IS_WHITESPACE(segment_str[i + 1])) {
-                    ++*right_anchor;
-                }
-
-                // Keep going if the current char is not ')'
-                if (i+1 < right->col_offset && (segment_str[i] == ')')) {
-                    continue;
-                }
-
-                // Set the error characters
-                *primary_error_char = "~";
-                *secondary_error_char = "^";
-                break;
-            }
-            return 1;
-        }
-        case Subscript_kind: {
-            *left_anchor = expr->v.Subscript.value->end_col_offset;
-            *right_anchor = expr->v.Subscript.slice->end_col_offset + 1;
-            Py_ssize_t str_len = strlen(segment_str);
-
-            // Move right_anchor and left_anchor forward to the first non-whitespace character that is not ']' and '['
-            while (*left_anchor < str_len && (IS_WHITESPACE(segment_str[*left_anchor]) || segment_str[*left_anchor] != '[')) {
-                ++*left_anchor;
-            }
-            while (*right_anchor < str_len && (IS_WHITESPACE(segment_str[*right_anchor]) || segment_str[*right_anchor] != ']')) {
-                ++*right_anchor;
-            }
-            if (*right_anchor < str_len){
-                *right_anchor += 1;
-            }
-
-            // Set the error characters
-            *primary_error_char = "~";
-            *secondary_error_char = "^";
-            return 1;
-        }
-        default:
-            return 0;
-    }
-}
-
-static int
-extract_anchors_from_stmt(const char *segment_str, stmt_ty statement, Py_ssize_t *left_anchor, Py_ssize_t *right_anchor,
-                          char** primary_error_char, char** secondary_error_char)
-{
-    switch (statement->kind) {
-        case Expr_kind: {
-            return extract_anchors_from_expr(segment_str, statement->v.Expr.value, left_anchor, right_anchor,
-                                             primary_error_char, secondary_error_char);
-        }
-        default:
-            return 0;
-    }
-}
-
-static int
-extract_anchors_from_line(PyObject *filename, PyObject *line,
-                          Py_ssize_t start_offset, Py_ssize_t end_offset,
-                          Py_ssize_t *left_anchor, Py_ssize_t *right_anchor,
-                          char** primary_error_char, char** secondary_error_char)
-{
-    int res = -1;
-    PyArena *arena = NULL;
-    PyObject *segment = PyUnicode_Substring(line, start_offset, end_offset);
-    if (!segment) {
-        goto done;
-    }
-
-    const char *segment_str = PyUnicode_AsUTF8(segment);
-    if (!segment_str) {
-        goto done;
-    }
-
-    arena = _PyArena_New();
-    if (!arena) {
-        goto done;
-    }
-
-    PyCompilerFlags flags = _PyCompilerFlags_INIT;
-
-    mod_ty module = _PyParser_ASTFromString(segment_str, filename, Py_file_input,
-                                            &flags, arena);
-    if (!module) {
-        goto done;
-    }
-    if (!_PyAST_Optimize(module, arena, _Py_GetConfig()->optimization_level, 0)) {
-        goto done;
-    }
-
-    assert(module->kind == Module_kind);
-    if (asdl_seq_LEN(module->v.Module.body) == 1) {
-        stmt_ty statement = asdl_seq_GET(module->v.Module.body, 0);
-        res = extract_anchors_from_stmt(segment_str, statement, left_anchor, right_anchor,
-                                        primary_error_char, secondary_error_char);
-    } else {
-        res = 0;
-    }
-
-done:
-    if (res > 0) {
-        // Normalize the AST offsets to byte offsets and adjust them with the
-        // start of the actual line (instead of the source code segment).
-        assert(segment != NULL);
-        assert(*left_anchor >= 0);
-        assert(*right_anchor >= 0);
-        *left_anchor = _PyPegen_byte_offset_to_character_offset(segment, *left_anchor) + start_offset;
-        *right_anchor = _PyPegen_byte_offset_to_character_offset(segment, *right_anchor) + start_offset;
-    }
-    Py_XDECREF(segment);
-    if (arena) {
-        _PyArena_Free(arena);
-    }
-    return res;
-}
-
 #define _TRACEBACK_SOURCE_LINE_INDENT 4
 
 static inline int
@@ -742,42 +570,14 @@ ignore_source_errors(void) {
     return 0;
 }
 
-static inline int
-print_error_location_carets(PyObject *f, int offset, Py_ssize_t start_offset, Py_ssize_t end_offset,
-                            Py_ssize_t right_start_offset, Py_ssize_t left_end_offset,
-                            const char *primary, const char *secondary) {
-    int special_chars = (left_end_offset != -1 || right_start_offset != -1);
-    const char *str;
-    while (++offset <= end_offset) {
-        if (offset <= start_offset) {
-            str = " ";
-        } else if (special_chars && left_end_offset < offset && offset <= right_start_offset) {
-            str = secondary;
-        } else {
-            str = primary;
-        }
-        if (PyFile_WriteString(str, f) < 0) {
-            return -1;
-        }
-    }
-    if (PyFile_WriteString("\n", f) < 0) {
-        return -1;
-    }
-    return 0;
-}
-
 static int
 tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int lineno,
-               PyFrameObject *frame, PyObject *name, int margin_indent, const char *margin)
+               PyFrameObject *frame, PyObject *name)
 {
     if (filename == NULL || name == NULL) {
         return -1;
     }
 
-    if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) {
-        return -1;
-    }
-
     PyObject *line = PyUnicode_FromFormat("  File \"%U\", line %d, in %U\n",
                                           filename, lineno, name);
     if (line == NULL) {
@@ -794,9 +594,9 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen
 
     int truncation = _TRACEBACK_SOURCE_LINE_INDENT;
     PyObject* source_line = NULL;
-    int rc = display_source_line_with_margin(
+    int rc = display_source_line(
             f, filename, lineno, _TRACEBACK_SOURCE_LINE_INDENT,
-            margin_indent, margin, &truncation, &source_line);
+            &truncation, &source_line);
     if (rc != 0 || !source_line) {
         /* ignore errors since we can't report them, can we? */
         err = ignore_source_errors();
@@ -823,87 +623,19 @@ tb_displayline(PyTracebackObject* tb, PyObject *f, PyObject *filename, int linen
         goto done;
     }
 
-    // When displaying errors, we will use the following generic structure:
+    // If this is a multi-line expression, then we will highlight until
-    //
+    // the last non-whitespace character.
-    //  ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE ERROR LINE
+    const char *source_line_str = PyUnicode_AsUTF8(source_line);
-    //        ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^~~~~~~~~~~~~~~~~~~~
+    if (!source_line_str) {
-    //        |              |-> left_end_offset     |                  |-> end_offset
-    //        |-> start_offset                       |-> right_start_offset
-    //
-    // In general we will only have (start_offset, end_offset) but we can gather more information
-    // by analyzing the AST of the text between *start_offset* and *end_offset*. If this succeeds
-    // we could get *left_end_offset* and *right_start_offset* and some selection of characters for
-    // the different ranges (primary_error_char and secondary_error_char). If we cannot obtain the
-    // AST information or we cannot identify special ranges within it, then left_end_offset and
-    // right_end_offset will be set to -1.
-    //
-    // To keep the column indicators pertinent, they are not shown when the primary character
-    // spans the whole line.
-
-    // Convert the utf-8 byte offset to the actual character offset so we print the right number of carets.
-    assert(source_line);
-    Py_ssize_t start_offset = _PyPegen_byte_offset_to_character_offset(source_line, start_col_byte_offset);
-    if (start_offset < 0) {
-        err = ignore_source_errors() < 0;
         goto done;
     }
 
-    Py_ssize_t end_offset = _PyPegen_byte_offset_to_character_offset(source_line, end_col_byte_offset);
+    Py_ssize_t i = source_line_len;
-    if (end_offset < 0) {
+    while (--i >= 0) {
-        err = ignore_source_errors() < 0;
+        if (!IS_WHITESPACE(source_line_str[i])) {
-        goto done;
+            break;
-    }
-
-    Py_ssize_t left_end_offset = -1;
-    Py_ssize_t right_start_offset = -1;
-
-    char *primary_error_char = "^";
-    char *secondary_error_char = primary_error_char;
-
-    if (start_line == end_line) {
-        int res = extract_anchors_from_line(filename, source_line, start_offset, end_offset,
-                                            &left_end_offset, &right_start_offset,
-                                            &primary_error_char, &secondary_error_char);
-        if (res < 0 && ignore_source_errors() < 0) {
-            goto done;
         }
     }
-    else {
-        // If this is a multi-line expression, then we will highlight until
-        // the last non-whitespace character.
-        const char *source_line_str = PyUnicode_AsUTF8(source_line);
-        if (!source_line_str) {
-            goto done;
-        }
-
-        Py_ssize_t i = source_line_len;
-        while (--i >= 0) {
-            if (!IS_WHITESPACE(source_line_str[i])) {
-                break;
-            }
-        }
-
-        end_offset = i + 1;
-    }
-
-    // Elide indicators if primary char spans the frame line
-    Py_ssize_t stripped_line_len = source_line_len - truncation - _TRACEBACK_SOURCE_LINE_INDENT;
-    bool has_secondary_ranges = (left_end_offset != -1 || right_start_offset != -1);
-    if (end_offset - start_offset == stripped_line_len && !has_secondary_ranges) {
-        goto done;
-    }
-
-    if (_Py_WriteIndentedMargin(margin_indent, margin, f) < 0) {
-        err = -1;
-        goto done;
-    }
-
-    if (print_error_location_carets(f, truncation, start_offset, end_offset,
-                                    right_start_offset, left_end_offset,
-                                    primary_error_char, secondary_error_char) < 0) {
-        err = -1;
-        goto done;
-    }
 
 done:
     Py_XDECREF(source_line);
@@ -930,8 +662,7 @@ tb_print_line_repeated(PyObject *f, long cnt)
 }
 
 static int
-tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit,
+tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit)
-                 int indent, const char *margin)
 {
     PyCodeObject *code = NULL;
     Py_ssize_t depth = 0;
@@ -967,7 +698,7 @@ tb_printinternal(PyTracebackObject *tb, PyObject *f, long limit,
         cnt++;
         if (cnt <= TB_RECURSIVE_CUTOFF) {
             if (tb_displayline(tb, f, code->co_filename, tb->tb_lineno,
-                               tb->tb_frame, code->co_name, indent, margin) < 0) {
+                               tb->tb_frame, code->co_name) < 0) {
                 goto error;
             }
 
@@ -992,8 +723,7 @@ error:
 #define PyTraceBack_LIMIT 1000
 
 int
-_PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin,
+_PyTraceBack_Print(PyObject *v, const char *header, PyObject *f)
-                            const char *header_margin, const char *header, PyObject *f)
 {
     PyObject *limitv;
     long limit = PyTraceBack_LIMIT;
@@ -1016,15 +746,12 @@ _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin,
             return 0;
         }
     }
-    if (_Py_WriteIndentedMargin(indent, header_margin, f) < 0) {
-        return -1;
-    }
 
     if (PyFile_WriteString(header, f) < 0) {
         return -1;
     }
 
-    if (tb_printinternal((PyTracebackObject *)v, f, limit, indent, margin) < 0) {
+    if (tb_printinternal((PyTracebackObject *)v, f, limit) < 0) {
         return -1;
     }
 
@@ -1034,12 +761,8 @@ _PyTraceBack_Print_Indented(PyObject *v, int indent, const char *margin,
 int
 PyTraceBack_Print(PyObject *v, PyObject *f)
 {
-    int indent = 0;
-    const char *margin = NULL;
-    const char *header_margin = NULL;
     const char *header = EXCEPTION_TB_HEADER;
-
+    return _PyTraceBack_Print(v, header, f);
-    return _PyTraceBack_Print_Indented(v, indent, margin, header_margin, header, f);
 }
 
 /* Format an integer in range [0; 0xffffffff] to decimal and write it