Patch #1953

I implemented the function sys._compact_freelists() and C API functions PyInt_/PyFloat_CompactFreeList() to compact the pre-allocated blocks of ints and floats. They allow the user to reduce the memory usage of a Python process that deals with lots of numbers.
The patch also renames sys._cleartypecache to sys._clear_type_cache

Doc/c-api/float.rst

@@ -84,3 +84,12 @@ Floating Point Objects
    Return the minimum normalized positive float *DBL_MIN* as C :ctype:`double`.
 
    .. versionadded:: 2.6
+
+
+.. cfunction:: void PyFloat_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
+
+   Compact the float free list. *bc* is the number of allocated blocks before
+   blocks are freed, *bf* is the number of freed blocks and *sum* is the number
+   of remaining objects in the blocks.
+
+   .. versionadded:: 2.6

Doc/c-api/int.rst

@@ -120,3 +120,12 @@ Plain Integer Objects
 
    Return the system's idea of the largest integer it can handle
    (:const:`LONG_MAX`, as defined in the system header files).
+
+
+.. cfunction:: void PyInt_CompactFreeList(size_t *bc, size_t *bf, size_t *sum)
+
+   Compact the integer free list. *bc* is the number of allocated blocks before
+   blocks are freed, *bf* is the number of freed blocks and *sum* is the number
+   of remaining objects in the blocks.
+
+   .. versionadded:: 2.6

Doc/library/sys.rst

@@ -58,9 +58,29 @@ always available.
    A string containing the copyright pertaining to the Python interpreter.
 
 
-.. function:: _cleartypecache()
+.. function:: _compact_freelists()
 
-   Clear the internal type lookup cache.
+   Compact the free lists of integers and floats by deallocating unused blocks.
+   It can reduce the memory usage of the Python process several tenth of
+   thousands of integers or floats have been allocated at once.
+
+   The return value is a tuple of tuples each containing three elements,
+   amount of used objects, total block count before the blocks are deallocated
+   and amount of freed blocks. The first tuple refers to ints, the second to
+   floats.
+
+   This function should be used for specialized purposes only.
+
+   .. versionadded:: 2.6
+
+
+.. function:: _clear_type_cache()
+
+   Clear the internal type cache. The type cache is used to speed up attribute
+   and method lookups. Use the function *only* to drop unnecessary references
+   during reference leak debugging.
+
+   This function should be used for internal and specialized purposes only.
 
    .. versionadded:: 2.6
 

Include/floatobject.h

@@ -101,6 +101,8 @@ PyAPI_FUNC(void) _PyFloat_DigitsInit(void);
 PyAPI_FUNC(double) _PyFloat_Unpack4(const unsigned char *p, int le);
 PyAPI_FUNC(double) _PyFloat_Unpack8(const unsigned char *p, int le);
 
+/* free list api */
+PyAPI_FUNC(void) PyFloat_CompactFreeList(size_t *, size_t *, size_t *);
 
 #ifdef __cplusplus
 }

Include/intobject.h

@@ -59,6 +59,9 @@ PyAPI_FUNC(long) PyInt_GetMax(void);
 PyAPI_FUNC(unsigned long) PyOS_strtoul(char *, char **, int);
 PyAPI_FUNC(long) PyOS_strtol(char *, char **, int);
 
+/* free list api */
+PyAPI_FUNC(void) PyInt_CompactFreeList(size_t *, size_t *, size_t *);
+
 #ifdef __cplusplus
 }
 #endif

Lib/test/regrtest.py

@@ -710,7 +710,7 @@ def dash_R_cleanup(fs, ps, pic, abcs):
     sys.path_importer_cache.update(pic)
 
     # clear type cache
-    sys._cleartypecache()
+    sys._clear_type_cache()
 
     # Clear ABC registries, restoring previously saved ABC registries.
     for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]:

Lib/test/test_sys.py

@@ -363,6 +363,24 @@ class SysModuleTest(unittest.TestCase):
             self.assertEqual(type(getattr(sys.flags, attr)), int, attr)
         self.assert_(repr(sys.flags))
 
+    def test_clear_type_cache(self):
+        sys._clear_type_cache()
+
+    def test_compact_freelists(self):
+        sys._compact_freelists()
+        r = sys._compact_freelists()
+        # freed blocks shouldn't change
+        self.assertEqual(r[0][2], 0)
+        self.assertEqual(r[1][2], 0)
+        # fill freelists
+        ints = list(range(12000))
+        floats = [float(i) for i in ints]
+        del ints
+        del floats
+        # should free more than 200 blocks each
+        r = sys._compact_freelists()
+        self.assert_(r[0][2] > 200, r[0][2])
+        self.assert_(r[1][2] > 200, r[1][2])
 
 def test_main():
     test.test_support.run_unittest(SysModuleTest)

Misc/NEWS

@@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1?
 Core and builtins
 -----------------
 
+- Patch #1953: Added ´´sys._compact_freelists()´´ and the C API functions
+  ´´PyInt_CompactFreeList´´ and ´´PyFloat_CompactFreeList´´
+  to compact the internal free lists of pre-allocted ints and floats.
+
 - Bug #1983: Fixed return type of fork(), fork1() and forkpty() calls.
   Python expected the return type int but the fork familie returns pi_t.
 
@@ -21,7 +25,7 @@ Core and builtins
 - Patch #1970 by Antoine Pitrou: Speedup unicode whitespace and linebreak
   detection
 
-- Added ``PyType_ClearCache()`` and ``sys._cleartypecache`` to clear the
+- Added ``PyType_ClearCache()`` and ``sys._clear_type_cache`` to clear the
   internal lookup cache for ref leak tests.
 
 - Patch #1473257: generator objects gain a gi_code attribute. This is the

Objects/floatobject.c

@@ -1609,17 +1609,15 @@ _PyFloat_Init(void)
 }
 
 void
-PyFloat_Fini(void)
+PyFloat_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
 {
 	PyFloatObject *p;
 	PyFloatBlock *list, *next;
 	unsigned i;
-	int bc, bf;	/* block count, number of freed blocks */
+	size_t bc = 0, bf = 0;	/* block count, number of freed blocks */
-	int frem, fsum;	/* remaining unfreed floats per block, total */
+	size_t fsum = 0;	/* total unfreed ints */
+	int frem;		/* remaining unfreed ints per block */
 
-	bc = 0;
-	bf = 0;
-	fsum = 0;
 	list = block_list;
 	block_list = NULL;
 	free_list = NULL;
@@ -1654,6 +1652,22 @@ PyFloat_Fini(void)
 		fsum += frem;
 		list = next;
 	}
+	*pbc = bc;
+	*pbf = bf;
+	*bsum = fsum;
+}
+
+void
+PyFloat_Fini(void)
+{
+	PyFloatObject *p;
+	PyFloatBlock *list;
+	unsigned i;
+	size_t bc, bf;	/* block count, number of freed blocks */
+	size_t fsum;	/* total unfreed floats per block */
+
+	PyFloat_CompactFreeList(&bc, &bf, &fsum);
+
 	if (!Py_VerboseFlag)
 		return;
 	fprintf(stderr, "# cleanup floats");
@@ -1662,7 +1676,9 @@ PyFloat_Fini(void)
 	}
 	else {
 		fprintf(stderr,
-			": %d unfreed float%s in %d out of %d block%s\n",
+			": %" PY_FORMAT_SIZE_T "d unfreed floats%s in %"
+			PY_FORMAT_SIZE_T "d out of %"
+			PY_FORMAT_SIZE_T "d block%s\n",
 			fsum, fsum == 1 ? "" : "s",
 			bc - bf, bc, bc == 1 ? "" : "s");
 	}

Objects/intobject.c

@@ -1202,28 +1202,15 @@ _PyInt_Init(void)
 }
 
 void
-PyInt_Fini(void)
+PyInt_CompactFreeList(size_t *pbc, size_t *pbf, size_t *bsum)
 {
 	PyIntObject *p;
 	PyIntBlock *list, *next;
-	int i;
 	unsigned int ctr;
-	int bc, bf;	/* block count, number of freed blocks */
+	size_t bc = 0, bf = 0;	/* block count, number of freed blocks */
-	int irem, isum;	/* remaining unfreed ints per block, total */
+	size_t isum = 0;	/* total unfreed ints */
+	int irem;		/* remaining unfreed ints per block */
 
-#if NSMALLNEGINTS + NSMALLPOSINTS > 0
-        PyIntObject **q;
-
-        i = NSMALLNEGINTS + NSMALLPOSINTS;
-        q = small_ints;
-        while (--i >= 0) {
-                Py_XDECREF(*q);
-                *q++ = NULL;
-        }
-#endif
-	bc = 0;
-	bf = 0;
-	isum = 0;
 	list = block_list;
 	block_list = NULL;
 	free_list = NULL;
@@ -1268,6 +1255,33 @@ PyInt_Fini(void)
 		isum += irem;
 		list = next;
 	}
+
+	*pbc = bc;
+	*pbf = bf;
+	*bsum = isum;
+}
+
+void
+PyInt_Fini(void)
+{
+	PyIntObject *p;
+	PyIntBlock *list;
+	unsigned int ctr;
+	size_t bc, bf;	/* block count, number of freed blocks */
+	size_t isum;	/* total unfreed ints per block */
+
+#if NSMALLNEGINTS + NSMALLPOSINTS > 0
+	int i;
+	PyIntObject **q;
+
+	i = NSMALLNEGINTS + NSMALLPOSINTS;
+	q = small_ints;
+	while (--i >= 0) {
+		Py_XDECREF(*q);
+		*q++ = NULL;
+	}
+#endif
+	PyInt_CompactFreeList(&bc, &bf, &isum);
 	if (!Py_VerboseFlag)
 		return;
 	fprintf(stderr, "# cleanup ints");
@@ -1276,7 +1290,9 @@ PyInt_Fini(void)
 	}
 	else {
 		fprintf(stderr,
-			": %d unfreed int%s in %d out of %d block%s\n",
+			": %" PY_FORMAT_SIZE_T "d unfreed ints%s in %"
+			PY_FORMAT_SIZE_T "d out of %"
+			PY_FORMAT_SIZE_T "d block%s\n",
 			isum, isum == 1 ? "" : "s",
 			bc - bf, bc, bc == 1 ? "" : "s");
 	}

Python/sysmodule.c

@@ -754,17 +754,6 @@ a 11-tuple where the entries in the tuple are counts of:\n\
 10. Number of stack pops performed by call_function()"
 );
 
-static PyObject *
-sys_cleartypecache(PyObject* self, PyObject* args)
-{
-	PyType_ClearCache();
-	Py_RETURN_NONE;
-}
-
-PyDoc_STRVAR(cleartypecache_doc,
-"_cleartypecache() -> None\n\
-Clear the internal type lookup cache.");
-
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -783,12 +772,44 @@ extern PyObject *_Py_GetDXProfile(PyObject *,  PyObject *);
 }
 #endif
 
+static PyObject *
+sys_clear_type_cache(PyObject* self, PyObject* args)
+{
+	PyType_ClearCache();
+	Py_RETURN_NONE;
+}
+
+PyDoc_STRVAR(sys_clear_type_cache__doc__,
+"_clear_type_cache() -> None\n\
+Clear the internal type lookup cache.");
+
+
+static PyObject *
+sys_compact_freelists(PyObject* self, PyObject* args)
+{
+	size_t isum, ibc, ibf;
+	size_t fsum, fbc, fbf;
+
+	PyInt_CompactFreeList(&ibc, &ibf, &isum);
+	PyFloat_CompactFreeList(&fbc, &fbf, &fsum);
+
+	return Py_BuildValue("(kkk)(kkk)", isum, ibc, ibf,
+					   fsum, fbc, fbf);
+
+}
+
+PyDoc_STRVAR(sys_compact_freelists__doc__,
+"_compact_freelists() -> ((remaing_objects, total_blocks, freed_blocks), ...)\n\
+Compact the free lists of ints and floats.");
+
 static PyMethodDef sys_methods[] = {
 	/* Might as well keep this in alphabetic order */
 	{"callstats", (PyCFunction)PyEval_GetCallStats, METH_NOARGS,
 	 callstats_doc},
-	{"_cleartypecache", sys_cleartypecache, METH_NOARGS,
+	{"_clear_type_cache",	sys_clear_type_cache,	  METH_NOARGS,
-	 cleartypecache_doc},
+	 sys_clear_type_cache__doc__},
+	{"_compact_freelists",	sys_compact_freelists,	  METH_NOARGS,
+	 sys_compact_freelists__doc__},
 	{"_current_frames", sys_current_frames, METH_NOARGS,
 	 current_frames_doc},
 	{"displayhook",	sys_displayhook, METH_O, displayhook_doc},