Implemented Martin's suggestion to clear the free lists during the garbage collection of the highest generation.

This commit is contained in:
Christian Heimes 2008-02-14 12:47:33 +00:00
parent 50361d4d9b
commit 3b718a79af
15 changed files with 131 additions and 25 deletions

View file

@ -63,3 +63,10 @@ There are some useful functions that are useful for working with method objects.
.. cfunction:: PyObject* PyMethod_GET_SELF(PyObject *meth) .. cfunction:: PyObject* PyMethod_GET_SELF(PyObject *meth)
Macro version of :cfunc:`PyMethod_Self` which avoids error checking. Macro version of :cfunc:`PyMethod_Self` which avoids error checking.
.. cfunction:: int PyMethod_ClearFreeList(void)
Clear the free list. Return the total number of freed items.
.. versionadded:: 2.6

View file

@ -115,3 +115,10 @@ Tuple Objects
.. versionchanged:: 2.2 .. versionchanged:: 2.2
Removed unused third parameter, *last_is_sticky*. Removed unused third parameter, *last_is_sticky*.
.. cfunction:: int PyMethod_ClearFreeList(void)
Clear the free list. Return the total number of freed items.
.. versionadded:: 2.6

View file

@ -89,6 +89,13 @@ access internal read-only data of Unicode objects:
Return a pointer to the internal buffer of the object. *o* has to be a Return a pointer to the internal buffer of the object. *o* has to be a
:ctype:`PyUnicodeObject` (not checked). :ctype:`PyUnicodeObject` (not checked).
.. cfunction:: int PyUnicode_ClearFreeList(void)
Clear the free list. Return the total number of freed items.
.. versionadded:: 2.6
Unicode provides many different character properties. The most often needed ones Unicode provides many different character properties. The most often needed ones
are available through these macros which are mapped to C functions depending on are available through these macros which are mapped to C functions depending on
the Python configuration. the Python configuration.

View file

@ -74,6 +74,7 @@ PyAPI_FUNC(PyObject *) _PyInstance_Lookup(PyObject *pinst, PyObject *name);
PyAPI_FUNC(int) PyClass_IsSubclass(PyObject *, PyObject *); PyAPI_FUNC(int) PyClass_IsSubclass(PyObject *, PyObject *);
PyAPI_FUNC(int) PyMethod_ClearFreeList(void);
#ifdef __cplusplus #ifdef __cplusplus
} }

View file

@ -75,6 +75,8 @@ PyAPI_FUNC(PyObject **) PyFrame_ExtendStack(PyFrameObject *, int, int);
PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int); PyAPI_FUNC(void) PyFrame_LocalsToFast(PyFrameObject *, int);
PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *); PyAPI_FUNC(void) PyFrame_FastToLocals(PyFrameObject *);
PyAPI_FUNC(int) PyFrame_ClearFreeList(void);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View file

@ -85,6 +85,8 @@ typedef struct {
PyObject *m_module; /* The __module__ attribute, can be anything */ PyObject *m_module; /* The __module__ attribute, can be anything */
} PyCFunctionObject; } PyCFunctionObject;
PyAPI_FUNC(int) PyCFunction_ClearFreeList(void);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View file

@ -52,6 +52,8 @@ PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...);
/* Macro, *only* to be used to fill in brand new tuples */ /* Macro, *only* to be used to fill in brand new tuples */
#define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v) #define PyTuple_SET_ITEM(op, i, v) (((PyTupleObject *)(op))->ob_item[i] = v)
PyAPI_FUNC(int) PyTuple_ClearFreeList(void);
#ifdef __cplusplus #ifdef __cplusplus
} }
#endif #endif

View file

@ -209,6 +209,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS2_AsDefaultEncodedString
# define _PyUnicode_Fini _PyUnicodeUCS2_Fini # define _PyUnicode_Fini _PyUnicodeUCS2_Fini
# define _PyUnicode_Init _PyUnicodeUCS2_Init # define _PyUnicode_Init _PyUnicodeUCS2_Init
# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
# define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha # define _PyUnicode_IsAlpha _PyUnicodeUCS2_IsAlpha
# define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS2_IsDecimalDigit
# define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit # define _PyUnicode_IsDigit _PyUnicodeUCS2_IsDigit
@ -295,6 +296,7 @@ typedef PY_UNICODE_TYPE Py_UNICODE;
# define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString # define _PyUnicode_AsDefaultEncodedString _PyUnicodeUCS4_AsDefaultEncodedString
# define _PyUnicode_Fini _PyUnicodeUCS4_Fini # define _PyUnicode_Fini _PyUnicodeUCS4_Fini
# define _PyUnicode_Init _PyUnicodeUCS4_Init # define _PyUnicode_Init _PyUnicodeUCS4_Init
# define PyUnicode_ClearFreeList PyUnicodeUCS2_ClearFreelist
# define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha # define _PyUnicode_IsAlpha _PyUnicodeUCS4_IsAlpha
# define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit # define _PyUnicode_IsDecimalDigit _PyUnicodeUCS4_IsDecimalDigit
# define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit # define _PyUnicode_IsDigit _PyUnicodeUCS4_IsDigit
@ -403,6 +405,8 @@ extern const unsigned char _Py_ascii_whitespace[];
extern "C" { extern "C" {
#endif #endif
PyAPI_FUNC(int) PyUnicode_ClearFreeList(void);
/* --- Unicode Type ------------------------------------------------------- */ /* --- Unicode Type ------------------------------------------------------- */
typedef struct { typedef struct {

View file

@ -12,6 +12,10 @@ What's New in Python 2.6 alpha 1?
Core and builtins Core and builtins
----------------- -----------------
- Clear all free list during a gc.collect() of the highest generation in order
to allow pymalloc to free more arenas. Python may give back memory to the
OS earlier.
- Issue #2045: Fix an infinite recursion triggered when printing a subclass of - Issue #2045: Fix an infinite recursion triggered when printing a subclass of
collections.defaultdict, if its default_factory is set to a bound method. collections.defaultdict, if its default_factory is set to a bound method.

View file

@ -19,6 +19,7 @@
*/ */
#include "Python.h" #include "Python.h"
#include "frameobject.h" /* for PyFrame_ClearFreeList */
/* Get an object's GC head */ /* Get an object's GC head */
#define AS_GC(o) ((PyGC_Head *)(o)-1) #define AS_GC(o) ((PyGC_Head *)(o)-1)
@ -722,6 +723,21 @@ delete_garbage(PyGC_Head *collectable, PyGC_Head *old)
} }
} }
/* Clear all free lists
* All free lists are cleared during the collection of the highest generation.
* Allocated items in the free list may keep a pymalloc arena occupied.
* Clearing the free lists may give back memory to the OS earlier.
*/
static void
clear_freelists(void)
{
(void)PyMethod_ClearFreeList();
(void)PyFrame_ClearFreeList();
(void)PyCFunction_ClearFreeList();
(void)PyTuple_ClearFreeList();
(void)PyUnicode_ClearFreeList();
}
/* This is the main function. Read this to understand how the /* This is the main function. Read this to understand how the
* collection process works. */ * collection process works. */
static Py_ssize_t static Py_ssize_t
@ -874,6 +890,12 @@ collect(int generation)
*/ */
(void)handle_finalizers(&finalizers, old); (void)handle_finalizers(&finalizers, old);
/* Clear free list only during the collection of the higest
* generation */
if (generation == NUM_GENERATIONS-1) {
clear_freelists();
}
if (PyErr_Occurred()) { if (PyErr_Occurred()) {
if (gc_str == NULL) if (gc_str == NULL)
gc_str = PyString_FromString("garbage collection"); gc_str = PyString_FromString("garbage collection");

View file

@ -2626,9 +2626,11 @@ PyTypeObject PyMethod_Type = {
/* Clear out the free list */ /* Clear out the free list */
void int
PyMethod_Fini(void) PyMethod_ClearFreeList(void)
{ {
int freelist_size = numfree;
while (free_list) { while (free_list) {
PyMethodObject *im = free_list; PyMethodObject *im = free_list;
free_list = (PyMethodObject *)(im->im_self); free_list = (PyMethodObject *)(im->im_self);
@ -2636,4 +2638,11 @@ PyMethod_Fini(void)
numfree--; numfree--;
} }
assert(numfree == 0); assert(numfree == 0);
return freelist_size;
}
void
PyMethod_Fini(void)
{
(void)PyMethod_ClearFreeList();
} }

View file

@ -889,10 +889,11 @@ PyFrame_LocalsToFast(PyFrameObject *f, int clear)
} }
/* Clear out the free list */ /* Clear out the free list */
int
void PyFrame_ClearFreeList(void)
PyFrame_Fini(void)
{ {
int freelist_size = numfree;
while (free_list != NULL) { while (free_list != NULL) {
PyFrameObject *f = free_list; PyFrameObject *f = free_list;
free_list = free_list->f_back; free_list = free_list->f_back;
@ -900,6 +901,13 @@ PyFrame_Fini(void)
--numfree; --numfree;
} }
assert(numfree == 0); assert(numfree == 0);
return freelist_size;
}
void
PyFrame_Fini(void)
{
(void)PyFrame_ClearFreeList();
Py_XDECREF(builtin_object); Py_XDECREF(builtin_object);
builtin_object = NULL; builtin_object = NULL;
} }

View file

@ -353,9 +353,11 @@ Py_FindMethod(PyMethodDef *methods, PyObject *self, const char *name)
/* Clear out the free list */ /* Clear out the free list */
void int
PyCFunction_Fini(void) PyCFunction_ClearFreeList(void)
{ {
int freelist_size = numfree;
while (free_list) { while (free_list) {
PyCFunctionObject *v = free_list; PyCFunctionObject *v = free_list;
free_list = (PyCFunctionObject *)(v->m_self); free_list = (PyCFunctionObject *)(v->m_self);
@ -363,6 +365,13 @@ PyCFunction_Fini(void)
numfree--; numfree--;
} }
assert(numfree == 0); assert(numfree == 0);
return freelist_size;
}
void
PyCFunction_Fini(void)
{
(void)PyCFunction_ClearFreeList();
} }
/* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(), /* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(),

View file

@ -832,25 +832,38 @@ _PyTuple_Resize(PyObject **pv, Py_ssize_t newsize)
return 0; return 0;
} }
void int
PyTuple_Fini(void) PyTuple_ClearFreeList(void)
{ {
int freelist_size = 0;
#if PyTuple_MAXSAVESIZE > 0 #if PyTuple_MAXSAVESIZE > 0
int i; int i;
Py_XDECREF(free_list[0]);
free_list[0] = NULL;
for (i = 1; i < PyTuple_MAXSAVESIZE; i++) { for (i = 1; i < PyTuple_MAXSAVESIZE; i++) {
PyTupleObject *p, *q; PyTupleObject *p, *q;
p = free_list[i]; p = free_list[i];
freelist_size += numfree[i];
free_list[i] = NULL; free_list[i] = NULL;
numfree[i] = 0;
while (p) { while (p) {
q = p; q = p;
p = (PyTupleObject *)(p->ob_item[0]); p = (PyTupleObject *)(p->ob_item[0]);
PyObject_GC_Del(q); PyObject_GC_Del(q);
} }
} }
#endif
return freelist_size;
}
void
PyTuple_Fini(void)
{
#if PyTuple_MAXSAVESIZE > 0
/* empty tuples are used all over the place and applications may
* rely on the fact that an empty tuple is a singleton. */
Py_XDECREF(free_list[0]);
free_list[0] = NULL;
(void)PyTuple_ClearFreeList();
#endif #endif
} }

View file

@ -8853,10 +8853,29 @@ void _PyUnicode_Init(void)
/* Finalize the Unicode implementation */ /* Finalize the Unicode implementation */
int
PyUnicode_ClearFreeList(void)
{
int freelist_size = numfree;
PyUnicodeObject *u;
for (u = free_list; u != NULL;) {
PyUnicodeObject *v = u;
u = *(PyUnicodeObject **)u;
if (v->str)
PyMem_DEL(v->str);
Py_XDECREF(v->defenc);
PyObject_Del(v);
numfree--;
}
free_list = NULL;
assert(numfree == 0);
return freelist_size;
}
void void
_PyUnicode_Fini(void) _PyUnicode_Fini(void)
{ {
PyUnicodeObject *u;
int i; int i;
Py_XDECREF(unicode_empty); Py_XDECREF(unicode_empty);
@ -8868,17 +8887,7 @@ _PyUnicode_Fini(void)
unicode_latin1[i] = NULL; unicode_latin1[i] = NULL;
} }
} }
(void)PyUnicode_ClearFreeList();
for (u = free_list; u != NULL;) {
PyUnicodeObject *v = u;
u = *(PyUnicodeObject **)u;
if (v->str)
PyMem_DEL(v->str);
Py_XDECREF(v->defenc);
PyObject_Del(v);
}
free_list = NULL;
numfree = 0;
} }
#ifdef __cplusplus #ifdef __cplusplus