gh-67224: Show source lines in tracebacks when using the -c option when running Python (#111200)

Include/internal/pycore_pylifecycle.h

@@ -114,6 +114,9 @@ extern int _Py_LegacyLocaleDetected(int warn);
 // Export for 'readline' shared extension
 PyAPI_FUNC(char*) _Py_SetLocaleFromEnv(int category);
 
+// Export for special main.c string compiling with source tracebacks
+int _PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags);
+
 #ifdef __cplusplus
 }
 #endif

Lib/linecache.py

@@ -5,10 +5,8 @@ is not found, it will look down the module search path for a file by
 that name.
 """
 
-import functools
 import sys
 import os
-import tokenize
 
 __all__ = ["getline", "clearcache", "checkcache", "lazycache"]
 
@@ -82,6 +80,8 @@ def updatecache(filename, module_globals=None):
     If something's wrong, print a message, discard the cache entry,
     and return an empty list."""
 
+    import tokenize
+
     if filename in cache:
         if len(cache[filename]) != 1:
             cache.pop(filename, None)
@@ -176,11 +176,13 @@ def lazycache(filename, module_globals):
         get_source = getattr(loader, 'get_source', None)
 
         if name and get_source:
-            get_lines = functools.partial(get_source, name)
+            def get_lines(name=name, *args, **kwargs):
+                return get_source(name, *args, **kwargs)
             cache[filename] = (get_lines,)
             return True
     return False
 
+
 def _register_code(code, string, name):
     cache[code] = (
             len(string),

Lib/test/test_cmd_line_script.py

@@ -684,6 +684,16 @@ class CmdLineTest(unittest.TestCase):
                     ]
                 )
 
+    def test_source_lines_are_shown_when_running_source(self):
+        _, _, stderr = assert_python_failure("-c", "1/0")
+        expected_lines = [
+            b'Traceback (most recent call last):',
+            b'  File "<string>", line 1, in <module>',
+            b'    1/0',
+            b'    ~^~',
+            b'ZeroDivisionError: division by zero']
+        self.assertEqual(stderr.splitlines(), expected_lines)
+
     def test_syntaxerror_does_not_crash(self):
         script = "nonlocal x\n"
         with os_helper.temp_dir() as script_dir:

Lib/test/test_io.py

@@ -4396,11 +4396,11 @@ class MiscIOTest(unittest.TestCase):
         ''')
         proc = assert_python_ok('-X', 'warn_default_encoding', '-c', code)
         warnings = proc.err.splitlines()
-        self.assertEqual(len(warnings), 2)
+        self.assertEqual(len(warnings), 4)
         self.assertTrue(
             warnings[0].startswith(b"<string>:5: EncodingWarning: "))
         self.assertTrue(
-            warnings[1].startswith(b"<string>:8: EncodingWarning: "))
+            warnings[2].startswith(b"<string>:8: EncodingWarning: "))
 
     def test_text_encoding(self):
         # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8"

Lib/test/test_repl.py

@@ -184,7 +184,7 @@ class TestInteractiveInterpreter(unittest.TestCase):
         p.stdin.write(user_input)
         user_input2 = dedent("""
         import linecache
-        print(linecache.cache['<python-input-1>'])
+        print(linecache.cache['<stdin>-1'])
         """)
         p.stdin.write(user_input2)
         output = kill_python(p)

Lib/test/test_subprocess.py

@@ -1769,9 +1769,9 @@ class RunFuncTestCase(BaseTestCase):
         cp = subprocess.run([sys.executable, "-Xwarn_default_encoding", "-c", code],
                             capture_output=True)
         lines = cp.stderr.splitlines()
-        self.assertEqual(len(lines), 2, lines)
+        self.assertEqual(len(lines), 4, lines)
         self.assertTrue(lines[0].startswith(b"<string>:2: EncodingWarning: "))
-        self.assertTrue(lines[1].startswith(b"<string>:3: EncodingWarning: "))
+        self.assertTrue(lines[2].startswith(b"<string>:3: EncodingWarning: "))
 
 
 def _get_test_grp_name():

Lib/test/test_sys.py

@@ -1114,14 +1114,18 @@ class SysModuleTest(unittest.TestCase):
         traceback = [
             b'Traceback (most recent call last):',
             b'  File "<string>", line 8, in <module>',
+            b'    f2()',
             b'  File "<string>", line 6, in f2',
+            b'    f1()',
             b'  File "<string>", line 4, in f1',
+            b'    1 / 0',
+            b'    ~~^~~',
             b'ZeroDivisionError: division by zero'
         ]
         check(10, traceback)
         check(3, traceback)
-        check(2, traceback[:1] + traceback[2:])
+        check(2, traceback[:1] + traceback[3:])
-        check(1, traceback[:1] + traceback[3:])
+        check(1, traceback[:1] + traceback[5:])
         check(0, [traceback[-1]])
         check(-1, [traceback[-1]])
         check(1<<1000, traceback)

Lib/test/test_traceback.py

@@ -313,6 +313,8 @@ class TracebackCases(unittest.TestCase):
         rc, stdout, stderr = assert_python_ok('-c', code)
         expected = [b'Traceback (most recent call last):',
                     b'  File "<string>", line 8, in __init__',
+                    b'    x = 1 / 0',
+                    b'        ^^^^^',
                     b'ZeroDivisionError: division by zero']
         self.assertEqual(stderr.splitlines(), expected)
 

Lib/test/test_warnings/__init__.py

@@ -1233,6 +1233,10 @@ class EnvironmentVariableTests(BaseTest):
         self.assertEqual(stderr.splitlines(),
             [b"Traceback (most recent call last):",
              b"  File \"<string>\", line 1, in <module>",
+             b'    import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w'
+             b"arn('Message', DeprecationWarning)",
+             b'                                                                  ^^^^^^^^^^'
+             b'^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^',
              b"DeprecationWarning: Message"])
 
     def test_default_filter_configuration(self):

Lib/traceback.py

@@ -476,12 +476,11 @@ class StackSummary(list):
         gets called for every frame to be printed in the stack summary.
         """
         row = []
-        if frame_summary.filename.startswith("<python-input"):
+        filename = frame_summary.filename
-            row.append('  File "<stdin>", line {}, in {}\n'.format(
+        if frame_summary.filename.startswith("<stdin>-"):
-                frame_summary.lineno, frame_summary.name))
+            filename = "<stdin>"
-        else:
+        row.append('  File "{}", line {}, in {}\n'.format(
-            row.append('  File "{}", line {}, in {}\n'.format(
+            filename, frame_summary.lineno, frame_summary.name))
-                frame_summary.filename, frame_summary.lineno, frame_summary.name))
         if frame_summary.line:
             stripped_line = frame_summary.line.strip()
             row.append('    {}\n'.format(stripped_line))

Misc/NEWS.d/next/Core and Builtins/2023-10-23-15-44-47.gh-issue-67224.S4D6CR.rst

@@ -0,0 +1,2 @@
+Show source lines in tracebacks when using the ``-c`` option when running
+Python. Patch by Pablo Galindo

Modules/main.c

@@ -249,7 +249,7 @@ pymain_run_command(wchar_t *command)
 
     PyCompilerFlags cf = _PyCompilerFlags_INIT;
     cf.cf_flags |= PyCF_IGNORE_COOKIE;
-    ret = PyRun_SimpleStringFlags(PyBytes_AsString(bytes), &cf);
+    ret = _PyRun_SimpleStringFlagsWithName(PyBytes_AsString(bytes), "<string>", &cf);
     Py_DECREF(bytes);
     return (ret != 0);
 

Python/pythonrun.c

@@ -40,14 +40,17 @@
 /* Forward */
 static void flush_io(void);
 static PyObject *run_mod(mod_ty, PyObject *, PyObject *, PyObject *,
-                          PyCompilerFlags *, PyArena *, PyObject*);
+                          PyCompilerFlags *, PyArena *, PyObject*, int);
 static PyObject *run_pyc_file(FILE *, PyObject *, PyObject *,
                               PyCompilerFlags *);
 static int PyRun_InteractiveOneObjectEx(FILE *, PyObject *, PyCompilerFlags *);
 static PyObject* pyrun_file(FILE *fp, PyObject *filename, int start,
                             PyObject *globals, PyObject *locals, int closeit,
                             PyCompilerFlags *flags);
-
+static PyObject *
+_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start,
+                           PyObject *globals, PyObject *locals, PyCompilerFlags *flags,
+                           int generate_new_source);
 
 int
 _PyRun_AnyFileObject(FILE *fp, PyObject *filename, int closeit,
@@ -281,7 +284,7 @@ PyRun_InteractiveOneObjectEx(FILE *fp, PyObject *filename,
     }
     PyObject *main_dict = PyModule_GetDict(main_module);  // borrowed ref
 
-    PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src);
+    PyObject *res = run_mod(mod, filename, main_dict, main_dict, flags, arena, interactive_src, 1);
     _PyArena_Free(arena);
     Py_DECREF(main_module);
     if (res == NULL) {
@@ -499,16 +502,25 @@ PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit,
 
 
 int
-PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags)
+_PyRun_SimpleStringFlagsWithName(const char *command, const char* name, PyCompilerFlags *flags) {
-{
     PyObject *main_module = PyImport_AddModuleRef("__main__");
     if (main_module == NULL) {
         return -1;
     }
     PyObject *dict = PyModule_GetDict(main_module);  // borrowed ref
 
-    PyObject *res = PyRun_StringFlags(command, Py_file_input,
+    PyObject *res = NULL;
-                                      dict, dict, flags);
+    if (name == NULL) {
+        res = PyRun_StringFlags(command, Py_file_input, dict, dict, flags);
+    } else {
+        PyObject* the_name = PyUnicode_FromString(name);
+        if (!the_name) {
+            PyErr_Print();
+            return -1;
+        }
+        res = _PyRun_StringFlagsWithName(command, the_name, Py_file_input, dict, dict, flags, 0);
+        Py_DECREF(the_name);
+    }
     Py_DECREF(main_module);
     if (res == NULL) {
         PyErr_Print();
@@ -519,6 +531,12 @@ PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags)
     return 0;
 }
 
+int
+PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags)
+{
+    return _PyRun_SimpleStringFlagsWithName(command, NULL, flags);
+}
+
 int
 _Py_HandleSystemExit(int *exitcode_p)
 {
@@ -1131,9 +1149,10 @@ void PyErr_DisplayException(PyObject *exc)
     PyErr_Display(NULL, exc, NULL);
 }
 
-PyObject *
+static PyObject *
-PyRun_StringFlags(const char *str, int start, PyObject *globals,
+_PyRun_StringFlagsWithName(const char *str, PyObject* name, int start,
-                  PyObject *locals, PyCompilerFlags *flags)
+                           PyObject *globals, PyObject *locals, PyCompilerFlags *flags,
+                           int generate_new_source)
 {
     PyObject *ret = NULL;
     mod_ty mod;
@@ -1143,17 +1162,36 @@ PyRun_StringFlags(const char *str, int start, PyObject *globals,
     if (arena == NULL)
         return NULL;
 
+    PyObject* source = NULL;
     _Py_DECLARE_STR(anon_string, "<string>");
-    mod = _PyParser_ASTFromString(
-            str, &_Py_STR(anon_string), start, flags, arena);
 
-    if (mod != NULL)
+    if (name) {
-        ret = run_mod(mod, &_Py_STR(anon_string), globals, locals, flags, arena, NULL);
+        source = PyUnicode_FromString(str);
+        if (!source) {
+            PyErr_Clear();
+        }
+    } else {
+        name = &_Py_STR(anon_string);
+    }
+
+    mod = _PyParser_ASTFromString(str, name, start, flags, arena);
+
+   if (mod != NULL) {
+        ret = run_mod(mod, name, globals, locals, flags, arena, source, generate_new_source);
+    }
+    Py_XDECREF(source);
     _PyArena_Free(arena);
     return ret;
 }
 
 
+PyObject *
+PyRun_StringFlags(const char *str, int start, PyObject *globals,
+                     PyObject *locals, PyCompilerFlags *flags) {
+
+    return _PyRun_StringFlagsWithName(str, NULL, start, globals, locals, flags, 0);
+}
+
 static PyObject *
 pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals,
            PyObject *locals, int closeit, PyCompilerFlags *flags)
@@ -1173,7 +1211,7 @@ pyrun_file(FILE *fp, PyObject *filename, int start, PyObject *globals,
 
     PyObject *ret;
     if (mod != NULL) {
-        ret = run_mod(mod, filename, globals, locals, flags, arena, NULL);
+        ret = run_mod(mod, filename, globals, locals, flags, arena, NULL, 0);
     }
     else {
         ret = NULL;
@@ -1261,15 +1299,19 @@ run_eval_code_obj(PyThreadState *tstate, PyCodeObject *co, PyObject *globals, Py
 
 static PyObject *
 run_mod(mod_ty mod, PyObject *filename, PyObject *globals, PyObject *locals,
-            PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src)
+            PyCompilerFlags *flags, PyArena *arena, PyObject* interactive_src,
+            int generate_new_source)
 {
     PyThreadState *tstate = _PyThreadState_GET();
     PyObject* interactive_filename = filename;
     if (interactive_src) {
         PyInterpreterState *interp = tstate->interp;
-        interactive_filename = PyUnicode_FromFormat(
+        if (generate_new_source) {
-            "<python-input-%d>", interp->_interactive_src_count++
+            interactive_filename = PyUnicode_FromFormat(
-        );
+                "%U-%d", filename, interp->_interactive_src_count++);
+        } else {
+            Py_INCREF(interactive_filename);
+        }
         if (interactive_filename == NULL) {
             return NULL;
         }