mirror of
https://github.com/python/cpython.git
synced 2025-11-03 19:34:08 +00:00
Better resolution for issue #11849: Ensure that free()d memory arenas are really released
on POSIX systems supporting anonymous memory mappings. Patch by Charles-François Natali.
This commit is contained in:
Antoine Pitrou
parent
61ea8a0d28
commit
f0effe6379
5 changed files with 35 additions and 65 deletions
|
|
@ -10,6 +10,10 @@ What's New in Python 3.3 Alpha 1?
|
|||
Core and Builtins
|
||||
-----------------
|
||||
|
||||
- Issue #11849: Ensure that free()d memory arenas are really released
|
||||
on POSIX systems supporting anonymous memory mappings. Patch by
|
||||
Charles-François Natali.
|
||||
|
||||
- Issue #13436: Fix a bogus error message when an AST object was passed
|
||||
an invalid integer value.
|
||||
|
||||
|
|
@ -260,10 +264,6 @@ Core and Builtins
|
|||
interpreter is shutting down; instead, these threads are now killed when
|
||||
they try to take the GIL.
|
||||
|
||||
- Issue #11849: Make it more likely for the system allocator to release
|
||||
free()d memory arenas on glibc-based systems. Patch by Charles-François
|
||||
Natali.
|
||||
|
||||
- Issue #9756: When calling a method descriptor or a slot wrapper descriptor,
|
||||
the check of the object type doesn't read the __class__ attribute anymore.
|
||||
Fix a crash if a class override its __class__ attribute (e.g. a proxy of the
|
||||
|
|
|
|||
|
|
@ -2,8 +2,11 @@
|
|||
|
||||
#ifdef WITH_PYMALLOC
|
||||
|
||||
#ifdef HAVE_MALLOPT_MMAP_THRESHOLD
|
||||
#include <malloc.h>
|
||||
#ifdef HAVE_MMAP
|
||||
#include <sys/mman.h>
|
||||
#ifdef MAP_ANONYMOUS
|
||||
#define ARENAS_USE_MMAP
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#ifdef WITH_VALGRIND
|
||||
|
|
@ -183,15 +186,15 @@ static int running_on_valgrind = -1;
|
|||
/*
|
||||
* The allocator sub-allocates <Big> blocks of memory (called arenas) aligned
|
||||
* on a page boundary. This is a reserved virtual address space for the
|
||||
* current process (obtained through a malloc call). In no way this means
|
||||
* that the memory arenas will be used entirely. A malloc(<Big>) is usually
|
||||
* an address range reservation for <Big> bytes, unless all pages within this
|
||||
* space are referenced subsequently. So malloc'ing big blocks and not using
|
||||
* them does not mean "wasting memory". It's an addressable range wastage...
|
||||
* current process (obtained through a malloc()/mmap() call). In no way this
|
||||
* means that the memory arenas will be used entirely. A malloc(<Big>) is
|
||||
* usually an address range reservation for <Big> bytes, unless all pages within
|
||||
* this space are referenced subsequently. So malloc'ing big blocks and not
|
||||
* using them does not mean "wasting memory". It's an addressable range
|
||||
* wastage...
|
||||
*
|
||||
* Therefore, allocating arenas with malloc is not optimal, because there is
|
||||
* some address space wastage, but this is the most portable way to request
|
||||
* memory from the system across various platforms.
|
||||
* Arenas are allocated with mmap() on systems supporting anonymous memory
|
||||
* mappings to reduce heap fragmentation.
|
||||
*/
|
||||
#define ARENA_SIZE (256 << 10) /* 256KB */
|
||||
|
||||
|
|
@ -556,11 +559,6 @@ new_arena(void)
|
|||
#if SIZEOF_SIZE_T <= SIZEOF_INT
|
||||
if (numarenas > PY_SIZE_MAX / sizeof(*arenas))
|
||||
return NULL; /* overflow */
|
||||
#endif
|
||||
#ifdef HAVE_MALLOPT_MMAP_THRESHOLD
|
||||
/* Ensure arenas are allocated by mmap to avoid heap fragmentation. */
|
||||
if (numarenas == INITIAL_ARENA_OBJECTS)
|
||||
mallopt(M_MMAP_THRESHOLD, ARENA_SIZE);
|
||||
#endif
|
||||
nbytes = numarenas * sizeof(*arenas);
|
||||
arenaobj = (struct arena_object *)realloc(arenas, nbytes);
|
||||
|
|
@ -594,7 +592,12 @@ new_arena(void)
|
|||
arenaobj = unused_arena_objects;
|
||||
unused_arena_objects = arenaobj->nextarena;
|
||||
assert(arenaobj->address == 0);
|
||||
#ifdef ARENAS_USE_MMAP
|
||||
arenaobj->address = (uptr)mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE,
|
||||
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
|
||||
#else
|
||||
arenaobj->address = (uptr)malloc(ARENA_SIZE);
|
||||
#endif
|
||||
if (arenaobj->address == 0) {
|
||||
/* The allocation failed: return NULL after putting the
|
||||
* arenaobj back.
|
||||
|
|
@ -1071,7 +1074,11 @@ PyObject_Free(void *p)
|
|||
unused_arena_objects = ao;
|
||||
|
||||
/* Free the entire arena. */
|
||||
#ifdef ARENAS_USE_MMAP
|
||||
munmap((void *)ao->address, ARENA_SIZE);
|
||||
#else
|
||||
free((void *)ao->address);
|
||||
#endif
|
||||
ao->address = 0; /* mark unassociated */
|
||||
--narenas_currently_allocated;
|
||||
|
||||
|
|
|
|||
36
configure
vendored
36
configure
vendored
|
|
@ -9401,8 +9401,8 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
|
|||
getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
|
||||
getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \
|
||||
if_nameindex \
|
||||
initgroups kill killpg lchmod lchown lockf linkat lstat lutimes memrchr \
|
||||
mbrtowc mkdirat mkfifo \
|
||||
initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
|
||||
memrchr mbrtowc mkdirat mkfifo \
|
||||
mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
|
||||
posix_fallocate posix_fadvise pread \
|
||||
pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \
|
||||
|
|
@ -9784,34 +9784,6 @@ $as_echo "no" >&6; }
|
|||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether mallopt can set malloc mmap threshold" >&5
|
||||
$as_echo_n "checking whether mallopt can set malloc mmap threshold... " >&6; }
|
||||
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
|
||||
/* end confdefs.h. */
|
||||
|
||||
#include <malloc.h>
|
||||
|
||||
int
|
||||
main ()
|
||||
{
|
||||
mallopt(M_MMAP_THRESHOLD, 256 * 1024)
|
||||
;
|
||||
return 0;
|
||||
}
|
||||
_ACEOF
|
||||
if ac_fn_c_try_compile "$LINENO"; then :
|
||||
|
||||
$as_echo "#define HAVE_MALLOPT_MMAP_THRESHOLD 1" >>confdefs.h
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
|
||||
$as_echo "yes" >&6; }
|
||||
else
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
|
||||
$as_echo "no" >&6; }
|
||||
|
||||
fi
|
||||
rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
|
||||
|
||||
{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for broken unsetenv" >&5
|
||||
$as_echo_n "checking for broken unsetenv... " >&6; }
|
||||
cat confdefs.h - <<_ACEOF >conftest.$ac_ext
|
||||
|
|
@ -14591,8 +14563,8 @@ esac
|
|||
|
||||
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
|
||||
# Files that config.status was made for.
|
||||
config_files="$ac_config_files"
|
||||
config_headers="$ac_config_headers"
|
||||
config_files="`echo $ac_config_files`"
|
||||
config_headers="`echo $ac_config_headers`"
|
||||
|
||||
_ACEOF
|
||||
|
||||
|
|
|
|||
13
configure.in
13
configure.in
|
|
@ -2567,8 +2567,8 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
|
|||
getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
|
||||
getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \
|
||||
if_nameindex \
|
||||
initgroups kill killpg lchmod lchown lockf linkat lstat lutimes memrchr \
|
||||
mbrtowc mkdirat mkfifo \
|
||||
initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
|
||||
memrchr mbrtowc mkdirat mkfifo \
|
||||
mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
|
||||
posix_fallocate posix_fadvise pread \
|
||||
pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \
|
||||
|
|
@ -2679,15 +2679,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
|||
[AC_MSG_RESULT(no)
|
||||
])
|
||||
|
||||
AC_MSG_CHECKING(whether mallopt can set malloc mmap threshold)
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
||||
#include <malloc.h>
|
||||
]], [[mallopt(M_MMAP_THRESHOLD, 256 * 1024)]])],
|
||||
[AC_DEFINE(HAVE_MALLOPT_MMAP_THRESHOLD, 1, Define if mallopt can set malloc mmap threshold.)
|
||||
AC_MSG_RESULT(yes)],
|
||||
[AC_MSG_RESULT(no)
|
||||
])
|
||||
|
||||
AC_MSG_CHECKING(for broken unsetenv)
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
||||
#include <stdlib.h>
|
||||
|
|
|
|||
|
|
@ -512,9 +512,6 @@
|
|||
/* Define this if you have the makedev macro. */
|
||||
#undef HAVE_MAKEDEV
|
||||
|
||||
/* Define if mallopt can set malloc mmap threshold. */
|
||||
#undef HAVE_MALLOPT_MMAP_THRESHOLD
|
||||
|
||||
/* Define to 1 if you have the `mbrtowc' function. */
|
||||
#undef HAVE_MBRTOWC
|
||||
|
||||
|
|
@ -545,6 +542,9 @@
|
|||
/* Define to 1 if you have the `mktime' function. */
|
||||
#undef HAVE_MKTIME
|
||||
|
||||
/* Define to 1 if you have the `mmap' function. */
|
||||
#undef HAVE_MMAP
|
||||
|
||||
/* Define to 1 if you have the `mremap' function. */
|
||||
#undef HAVE_MREMAP
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue