Better resolution for issue #11849: Ensure that free()d memory arenas are really released

on POSIX systems supporting anonymous memory mappings. Patch by Charles-François Natali.
2011-11-26 01:11:02 +01:00 · 2011-11-26 01:11:02 +01:00 · f0effe6379
parent 61ea8a0d28
commit f0effe6379
5 changed files with 35 additions and 65 deletions
--- a/Misc/NEWS
+++ b/Misc/NEWS
@ -10,6 +10,10 @@ What's New in Python 3.3 Alpha 1?
 Core and Builtins
 -----------------

+- Issue #11849: Ensure that free()d memory arenas are really released
+  on POSIX systems supporting anonymous memory mappings.  Patch by
+  Charles-François Natali.
+
 - Issue #13436: Fix a bogus error message when an AST object was passed
  an invalid integer value.

@ -260,10 +264,6 @@ Core and Builtins
  interpreter is shutting down; instead, these threads are now killed when
  they try to take the GIL.

- Issue #11849: Make it more likely for the system allocator to release
-  free()d memory arenas on glibc-based systems.  Patch by Charles-François
-  Natali.
-
 - Issue #9756: When calling a method descriptor or a slot wrapper descriptor,
  the check of the object type doesn't read the __class__ attribute anymore.
  Fix a crash if a class override its __class__ attribute (e.g. a proxy of the
--- a/Objects/obmalloc.c
+++ b/Objects/obmalloc.c
@ -2,8 +2,11 @@

 #ifdef WITH_PYMALLOC

-#ifdef HAVE_MALLOPT_MMAP_THRESHOLD
-  #include <malloc.h>
+#ifdef HAVE_MMAP
+ #include <sys/mman.h>
+ #ifdef MAP_ANONYMOUS
+  #define ARENAS_USE_MMAP
+ #endif
 #endif

 #ifdef WITH_VALGRIND
@ -183,15 +186,15 @@ static int running_on_valgrind = -1;
 /*
 * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned
 * on a page boundary. This is a reserved virtual address space for the
- * current process (obtained through a malloc call). In no way this means
- * that the memory arenas will be used entirely. A malloc(<Big>) is usually
- * an address range reservation for <Big> bytes, unless all pages within this
- * space are referenced subsequently. So malloc'ing big blocks and not using
- * them does not mean "wasting memory". It's an addressable range wastage...
+ * current process (obtained through a malloc()/mmap() call). In no way this
+ * means that the memory arenas will be used entirely. A malloc(<Big>) is
+ * usually an address range reservation for <Big> bytes, unless all pages within
+ * this space are referenced subsequently. So malloc'ing big blocks and not
+ * using them does not mean "wasting memory". It's an addressable range
+ * wastage...
 *
- * Therefore, allocating arenas with malloc is not optimal, because there is
- * some address space wastage, but this is the most portable way to request
- * memory from the system across various platforms.
+ * Arenas are allocated with mmap() on systems supporting anonymous memory
+ * mappings to reduce heap fragmentation.
 */
 #define ARENA_SIZE              (256 << 10)     /* 256KB */

@ -556,11 +559,6 @@ new_arena(void)
 #if SIZEOF_SIZE_T <= SIZEOF_INT
        if (numarenas > PY_SIZE_MAX / sizeof(*arenas))
            return NULL;                /* overflow */
-#endif
-#ifdef HAVE_MALLOPT_MMAP_THRESHOLD
-        /* Ensure arenas are allocated by mmap to avoid heap fragmentation. */
-        if (numarenas == INITIAL_ARENA_OBJECTS)
-            mallopt(M_MMAP_THRESHOLD, ARENA_SIZE);
 #endif
        nbytes = numarenas * sizeof(*arenas);
        arenaobj = (struct arena_object *)realloc(arenas, nbytes);
@ -594,7 +592,12 @@ new_arena(void)
    arenaobj = unused_arena_objects;
    unused_arena_objects = arenaobj->nextarena;
    assert(arenaobj->address == 0);
+#ifdef ARENAS_USE_MMAP
+    arenaobj->address = (uptr)mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE,
+                                   MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
+#else
    arenaobj->address = (uptr)malloc(ARENA_SIZE);
+#endif
    if (arenaobj->address == 0) {
        /* The allocation failed: return NULL after putting the
         * arenaobj back.
@ -1071,7 +1074,11 @@ PyObject_Free(void *p)
                unused_arena_objects = ao;

                /* Free the entire arena. */
+#ifdef ARENAS_USE_MMAP
+                munmap((void *)ao->address, ARENA_SIZE);
+#else
                free((void *)ao->address);
+#endif
                ao->address = 0;                        /* mark unassociated */
                --narenas_currently_allocated;

--- a/36
+++ b/36
@ -9401,8 +9401,8 @@ for ac_func in alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
 getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
 getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \
 if_nameindex \
- initgroups kill killpg lchmod lchown lockf linkat lstat lutimes memrchr \
- mbrtowc mkdirat mkfifo \
+ initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
+ memrchr mbrtowc mkdirat mkfifo \
 mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
 posix_fallocate posix_fadvise pread \
 pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \
@ -9784,34 +9784,6 @@ $as_echo "no" >&6; }
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext

-{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether mallopt can set malloc mmap threshold" >&5
-$as_echo_n "checking whether mallopt can set malloc mmap threshold... " >&6; }
-cat confdefs.h - <<_ACEOF >conftest.$ac_ext
-/* end confdefs.h.  */
-
-#include <malloc.h>
-
-int
-main ()
-{
-mallopt(M_MMAP_THRESHOLD, 256 * 1024)
-  ;
-  return 0;
-}
-_ACEOF
-if ac_fn_c_try_compile "$LINENO"; then :
-
-$as_echo "#define HAVE_MALLOPT_MMAP_THRESHOLD 1" >>confdefs.h
-
-   { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5
-$as_echo "yes" >&6; }
-else
-  { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5
-$as_echo "no" >&6; }
-
-fi
-rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
-
 { $as_echo "$as_me:${as_lineno-$LINENO}: checking for broken unsetenv" >&5
 $as_echo_n "checking for broken unsetenv... " >&6; }
 cat confdefs.h - <<_ACEOF >conftest.$ac_ext
@ -14591,8 +14563,8 @@ esac

 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 # Files that config.status was made for.
-config_files="$ac_config_files"
-config_headers="$ac_config_headers"
+config_files="`echo $ac_config_files`"
+config_headers="`echo $ac_config_headers`"

 _ACEOF

--- a/configure.in
+++ b/configure.in
@ -2567,8 +2567,8 @@ AC_CHECK_FUNCS(alarm accept4 setitimer getitimer bind_textdomain_codeset chown \
 getgrouplist getgroups getlogin getloadavg getpeername getpgid getpid \
 getpriority getresuid getresgid getpwent getspnam getspent getsid getwd \
 if_nameindex \
- initgroups kill killpg lchmod lchown lockf linkat lstat lutimes memrchr \
- mbrtowc mkdirat mkfifo \
+ initgroups kill killpg lchmod lchown lockf linkat lstat lutimes mmap \
+ memrchr mbrtowc mkdirat mkfifo \
 mkfifoat mknod mknodat mktime mremap nice openat pathconf pause pipe2 plock poll \
 posix_fallocate posix_fadvise pread \
 pthread_init pthread_kill putenv pwrite readlink readlinkat readv realpath renameat \
@ -2679,15 +2679,6 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
  [AC_MSG_RESULT(no)
 ])

-AC_MSG_CHECKING(whether mallopt can set malloc mmap threshold)
-AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
-#include <malloc.h>
-]], [[mallopt(M_MMAP_THRESHOLD, 256 * 1024)]])],
-  [AC_DEFINE(HAVE_MALLOPT_MMAP_THRESHOLD, 1, Define if mallopt can set malloc mmap threshold.)
-   AC_MSG_RESULT(yes)],
-  [AC_MSG_RESULT(no)
-])
-
 AC_MSG_CHECKING(for broken unsetenv)
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
 #include <stdlib.h>
--- a/pyconfig.h.in
+++ b/pyconfig.h.in
@ -512,9 +512,6 @@
 /* Define this if you have the makedev macro. */
 #undef HAVE_MAKEDEV

-/* Define if mallopt can set malloc mmap threshold. */
-#undef HAVE_MALLOPT_MMAP_THRESHOLD
-
 /* Define to 1 if you have the `mbrtowc' function. */
 #undef HAVE_MBRTOWC

@ -545,6 +542,9 @@
 /* Define to 1 if you have the `mktime' function. */
 #undef HAVE_MKTIME

+/* Define to 1 if you have the `mmap' function. */
+#undef HAVE_MMAP
+
 /* Define to 1 if you have the `mremap' function. */
 #undef HAVE_MREMAP