try to use the same str object for all code filenames when compiling or unmarshalling (#12190)

This should reduce memory usage.

Lib/test/test_compile.py

@@ -1,6 +1,7 @@
 import unittest
 import sys
 import _ast
+import types
 from test import support
 
 class TestSpecifics(unittest.TestCase):
@@ -433,6 +434,14 @@ if 1:
         ast.body = [_ast.BoolOp()]
         self.assertRaises(TypeError, compile, ast, '<ast>', 'exec')
 
+    @support.cpython_only
+    def test_same_filename_used(self):
+        s = """def f(): pass\ndef g(): pass"""
+        c = compile(s, "myfile", "exec")
+        for obj in c.co_consts:
+            if isinstance(obj, types.CodeType):
+                self.assertIs(obj.co_filename, c.co_filename)
+
 
 def test_main():
     support.run_unittest(TestSpecifics)

Lib/test/test_marshal.py

@@ -5,6 +5,7 @@ import marshal
 import sys
 import unittest
 import os
+import types
 
 class HelperMixin:
     def helper(self, sample, *extra):
@@ -113,6 +114,22 @@ class CodeTestCase(unittest.TestCase):
         codes = (ExceptionTestCase.test_exceptions.__code__,) * count
         marshal.loads(marshal.dumps(codes))
 
+    def test_different_filenames(self):
+        co1 = compile("x", "f1", "exec")
+        co2 = compile("y", "f2", "exec")
+        co1, co2 = marshal.loads(marshal.dumps((co1, co2)))
+        self.assertEqual(co1.co_filename, "f1")
+        self.assertEqual(co2.co_filename, "f2")
+
+    @support.cpython_only
+    def test_same_filename_used(self):
+        s = """def f(): pass\ndef g(): pass"""
+        co = compile(s, "myfile", "exec")
+        co = marshal.loads(marshal.dumps(co))
+        for obj in co.co_consts:
+            if isinstance(obj, types.CodeType):
+                self.assertIs(co.co_filename, obj.co_filename)
+
 class ContainerTestCase(unittest.TestCase, HelperMixin):
     d = {'astring': 'foo@bar.baz.spam',
          'afloat': 7283.43,

Misc/NEWS

@@ -10,6 +10,9 @@ What's New in Python 3.3 Alpha 1?
 Core and Builtins
 -----------------
 
+- Issue #12190: Try to use the same filename object when compiling unmarshalling
+  a code objects in the same file.
+
 - Issue #12166: Move implementations of dir() specialized for various types into
   the __dir__() methods of those types.
 

Python/compile.c

@@ -135,6 +135,7 @@ managed by compiler_enter_scope() and compiler_exit_scope().
 
 struct compiler {
     const char *c_filename;
+    PyObject *c_filename_obj;
     struct symtable *c_st;
     PyFutureFeatures *c_future; /* pointer to module's __future__ */
     PyCompilerFlags *c_flags;
@@ -272,6 +273,9 @@ PyAST_CompileEx(mod_ty mod, const char *filename, PyCompilerFlags *flags,
     if (!compiler_init(&c))
         return NULL;
     c.c_filename = filename;
+    c.c_filename_obj = PyUnicode_DecodeFSDefault(filename);
+    if (!c.c_filename_obj)
+        goto finally;
     c.c_arena = arena;
     c.c_future = PyFuture_FromAST(mod, filename);
     if (c.c_future == NULL)
@@ -324,6 +328,8 @@ compiler_free(struct compiler *c)
         PySymtable_Free(c->c_st);
     if (c->c_future)
         PyObject_Free(c->c_future);
+    if (c->c_filename_obj)
+        Py_DECREF(c->c_filename_obj);
     Py_DECREF(c->c_stack);
 }
 
@@ -3361,7 +3367,7 @@ compiler_in_loop(struct compiler *c) {
 static int
 compiler_error(struct compiler *c, const char *errstr)
 {
-    PyObject *loc, *filename;
+    PyObject *loc;
     PyObject *u = NULL, *v = NULL;
 
     loc = PyErr_ProgramText(c->c_filename, c->u->u_lineno);
@@ -3369,16 +3375,7 @@ compiler_error(struct compiler *c, const char *errstr)
         Py_INCREF(Py_None);
         loc = Py_None;
     }
-    if (c->c_filename != NULL) {
+    u = Py_BuildValue("(OiiO)", c->c_filename_obj, c->u->u_lineno,
-        filename = PyUnicode_DecodeFSDefault(c->c_filename);
-        if (!filename)
-            goto exit;
-    }
-    else {
-        Py_INCREF(Py_None);
-        filename = Py_None;
-    }
-    u = Py_BuildValue("(NiiO)", filename, c->u->u_lineno,
                       c->u->u_col_offset, loc);
     if (!u)
         goto exit;
@@ -3927,7 +3924,6 @@ makecode(struct compiler *c, struct assembler *a)
     PyObject *consts = NULL;
     PyObject *names = NULL;
     PyObject *varnames = NULL;
-    PyObject *filename = NULL;
     PyObject *name = NULL;
     PyObject *freevars = NULL;
     PyObject *cellvars = NULL;
@@ -3951,10 +3947,6 @@ makecode(struct compiler *c, struct assembler *a)
     freevars = dict_keys_inorder(c->u->u_freevars, PyTuple_Size(cellvars));
     if (!freevars)
         goto error;
-    filename = PyUnicode_DecodeFSDefault(c->c_filename);
-    if (!filename)
-        goto error;
-
     nlocals = PyDict_Size(c->u->u_varnames);
     flags = compute_code_flags(c);
     if (flags < 0)
@@ -3974,14 +3966,13 @@ makecode(struct compiler *c, struct assembler *a)
                     nlocals, stackdepth(c), flags,
                     bytecode, consts, names, varnames,
                     freevars, cellvars,
-                    filename, c->u->u_name,
+                    c->c_filename_obj, c->u->u_name,
                     c->u->u_firstlineno,
                     a->a_lnotab);
  error:
     Py_XDECREF(consts);
     Py_XDECREF(names);
     Py_XDECREF(varnames);
-    Py_XDECREF(filename);
     Py_XDECREF(name);
     Py_XDECREF(freevars);
     Py_XDECREF(cellvars);

Python/marshal.c

@@ -58,6 +58,7 @@ typedef struct {
     int depth;
     /* If fp == NULL, the following are valid: */
     PyObject *str;
+    PyObject *current_filename;
     char *ptr;
     char *end;
     int version;
@@ -976,6 +977,18 @@ r_object(RFILE *p)
             filename = r_object(p);
             if (filename == NULL)
                 goto code_error;
+            if (PyUnicode_CheckExact(filename)) {
+                if (p->current_filename != NULL) {
+                    if (!PyUnicode_Compare(filename, p->current_filename)) {
+                        Py_DECREF(filename);
+                        Py_INCREF(p->current_filename);
+                        filename = p->current_filename;
+                    }
+                }
+                else {
+                    p->current_filename = filename;
+                }
+            }
             name = r_object(p);
             if (name == NULL)
                 goto code_error;
@@ -1037,6 +1050,7 @@ PyMarshal_ReadShortFromFile(FILE *fp)
     RFILE rf;
     assert(fp);
     rf.fp = fp;
+    rf.current_filename = NULL;
     rf.end = rf.ptr = NULL;
     return r_short(&rf);
 }
@@ -1046,6 +1060,7 @@ PyMarshal_ReadLongFromFile(FILE *fp)
 {
     RFILE rf;
     rf.fp = fp;
+    rf.current_filename = NULL;
     rf.ptr = rf.end = NULL;
     return r_long(&rf);
 }
@@ -1106,6 +1121,7 @@ PyMarshal_ReadObjectFromFile(FILE *fp)
     RFILE rf;
     PyObject *result;
     rf.fp = fp;
+    rf.current_filename = NULL;
     rf.depth = 0;
     rf.ptr = rf.end = NULL;
     result = r_object(&rf);
@@ -1118,6 +1134,7 @@ PyMarshal_ReadObjectFromString(char *str, Py_ssize_t len)
     RFILE rf;
     PyObject *result;
     rf.fp = NULL;
+    rf.current_filename = NULL;
     rf.ptr = str;
     rf.end = str + len;
     rf.depth = 0;
@@ -1214,6 +1231,7 @@ marshal_load(PyObject *self, PyObject *f)
     if (data == NULL)
         return NULL;
     rf.fp = NULL;
+    rf.current_filename = NULL;
     if (PyBytes_Check(data)) {
         rf.ptr = PyBytes_AS_STRING(data);
         rf.end = rf.ptr + PyBytes_GET_SIZE(data);
@@ -1282,6 +1300,7 @@ marshal_loads(PyObject *self, PyObject *args)
     s = p.buf;
     n = p.len;
     rf.fp = NULL;
+    rf.current_filename = NULL;
     rf.ptr = s;
     rf.end = s + n;
     rf.depth = 0;