Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(204454)

Side by Side Diff: Objects/obmalloc.c

Issue 11849: glibc allocator doesn't release all free()ed memory
Patch Set: Created 8 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « configure.in ('k') | pyconfig.h.in » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 #include "Python.h" 1 #include "Python.h"
2 2
3 #ifdef WITH_PYMALLOC 3 #ifdef WITH_PYMALLOC
4 4
5 #ifdef HAVE_MALLOPT_MMAP_THRESHOLD 5 #ifdef HAVE_MMAP
6 #include <malloc.h> 6 #include <sys/mman.h>
7 #ifdef MAP_ANONYMOUS
8 #define ARENAS_USE_MMAP
9 #endif
7 #endif 10 #endif
8 11
9 #ifdef WITH_VALGRIND 12 #ifdef WITH_VALGRIND
10 #include <valgrind/valgrind.h> 13 #include <valgrind/valgrind.h>
11 14
12 /* If we're using GCC, use __builtin_expect() to reduce overhead of 15 /* If we're using GCC, use __builtin_expect() to reduce overhead of
13 the valgrind checks */ 16 the valgrind checks */
14 #if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__) 17 #if defined(__GNUC__) && (__GNUC__ > 2) && defined(__OPTIMIZE__)
15 # define UNLIKELY(value) __builtin_expect((value), 0) 18 # define UNLIKELY(value) __builtin_expect((value), 0)
16 #else 19 #else
(...skipping 159 matching lines...) Expand 10 before | Expand all | Expand 10 after
176 */ 179 */
177 #ifdef WITH_MEMORY_LIMITS 180 #ifdef WITH_MEMORY_LIMITS
178 #ifndef SMALL_MEMORY_LIMIT 181 #ifndef SMALL_MEMORY_LIMIT
179 #define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */ 182 #define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */
180 #endif 183 #endif
181 #endif 184 #endif
182 185
183 /* 186 /*
184 * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned 187 * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned
185 * on a page boundary. This is a reserved virtual address space for the 188 * on a page boundary. This is a reserved virtual address space for the
186 * current process (obtained through a malloc call). In no way this means 189 * current process (obtained through a malloc()/mmap() call). In no way this
187 * that the memory arenas will be used entirely. A malloc(<Big>) is usually 190 * means that the memory arenas will be used entirely. A malloc(<Big>) is
188 * an address range reservation for <Big> bytes, unless all pages within this 191 * usually an address range reservation for <Big> bytes, unless all pages within
189 * space are referenced subsequently. So malloc'ing big blocks and not using 192 * this space are referenced subsequently. So malloc'ing big blocks and not
190 * them does not mean "wasting memory". It's an addressable range wastage... 193 * using them does not mean "wasting memory". It's an addressable range
194 * wastage...
191 * 195 *
192 * Therefore, allocating arenas with malloc is not optimal, because there is 196 * Arenas are allocated with mmap() on systems supporting anonymous memory
193 * some address space wastage, but this is the most portable way to request 197 * mappings to reduce heap fragmentation.
194 * memory from the system across various platforms.
195 */ 198 */
196 #define ARENA_SIZE (256 << 10) /* 256KB */ 199 #define ARENA_SIZE (256 << 10) /* 256KB */
197 200
198 #ifdef WITH_MEMORY_LIMITS 201 #ifdef WITH_MEMORY_LIMITS
199 #define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE) 202 #define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE)
200 #endif 203 #endif
201 204
202 /* 205 /*
203 * Size of the pools used for small blocks. Should be a power of 2, 206 * Size of the pools used for small blocks. Should be a power of 2,
204 * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k. 207 * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k.
(...skipping 344 matching lines...) Expand 10 before | Expand all | Expand 10 after
549 552
550 /* Double the number of arena objects on each allocation. 553 /* Double the number of arena objects on each allocation.
551 * Note that it's possible for `numarenas` to overflow. 554 * Note that it's possible for `numarenas` to overflow.
552 */ 555 */
553 numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS; 556 numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS;
554 if (numarenas <= maxarenas) 557 if (numarenas <= maxarenas)
555 return NULL; /* overflow */ 558 return NULL; /* overflow */
556 #if SIZEOF_SIZE_T <= SIZEOF_INT 559 #if SIZEOF_SIZE_T <= SIZEOF_INT
557 if (numarenas > PY_SIZE_MAX / sizeof(*arenas)) 560 if (numarenas > PY_SIZE_MAX / sizeof(*arenas))
558 return NULL; /* overflow */ 561 return NULL; /* overflow */
559 #endif
560 #ifdef HAVE_MALLOPT_MMAP_THRESHOLD
561 /* Ensure arenas are allocated by mmap to avoid heap fragmentation. */
562 if (numarenas == INITIAL_ARENA_OBJECTS)
563 mallopt(M_MMAP_THRESHOLD, ARENA_SIZE);
564 #endif 562 #endif
565 nbytes = numarenas * sizeof(*arenas); 563 nbytes = numarenas * sizeof(*arenas);
566 arenaobj = (struct arena_object *)realloc(arenas, nbytes); 564 arenaobj = (struct arena_object *)realloc(arenas, nbytes);
567 if (arenaobj == NULL) 565 if (arenaobj == NULL)
568 return NULL; 566 return NULL;
569 arenas = arenaobj; 567 arenas = arenaobj;
570 568
571 /* We might need to fix pointers that were copied. However, 569 /* We might need to fix pointers that were copied. However,
572 * new_arena only gets called when all the pages in the 570 * new_arena only gets called when all the pages in the
573 * previous arenas are full. Thus, there are *no* pointers 571 * previous arenas are full. Thus, there are *no* pointers
(...skipping 13 matching lines...) Expand all
587 /* Update globals. */ 585 /* Update globals. */
588 unused_arena_objects = &arenas[maxarenas]; 586 unused_arena_objects = &arenas[maxarenas];
589 maxarenas = numarenas; 587 maxarenas = numarenas;
590 } 588 }
591 589
592 /* Take the next available arena object off the head of the list. */ 590 /* Take the next available arena object off the head of the list. */
593 assert(unused_arena_objects != NULL); 591 assert(unused_arena_objects != NULL);
594 arenaobj = unused_arena_objects; 592 arenaobj = unused_arena_objects;
595 unused_arena_objects = arenaobj->nextarena; 593 unused_arena_objects = arenaobj->nextarena;
596 assert(arenaobj->address == 0); 594 assert(arenaobj->address == 0);
595 #ifdef ARENAS_USE_MMAP
596 arenaobj->address = (uptr)mmap(NULL, ARENA_SIZE, PROT_READ|PROT_WRITE,
597 MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
598 #else
597 arenaobj->address = (uptr)malloc(ARENA_SIZE); 599 arenaobj->address = (uptr)malloc(ARENA_SIZE);
600 #endif
598 if (arenaobj->address == 0) { 601 if (arenaobj->address == 0) {
599 /* The allocation failed: return NULL after putting the 602 /* The allocation failed: return NULL after putting the
600 * arenaobj back. 603 * arenaobj back.
601 */ 604 */
602 arenaobj->nextarena = unused_arena_objects; 605 arenaobj->nextarena = unused_arena_objects;
603 unused_arena_objects = arenaobj; 606 unused_arena_objects = arenaobj;
604 return NULL; 607 return NULL;
605 } 608 }
606 609
607 ++narenas_currently_allocated; 610 ++narenas_currently_allocated;
(...skipping 456 matching lines...) Expand 10 before | Expand all | Expand 10 after
1064 ao->nextarena->prevarena = 1067 ao->nextarena->prevarena =
1065 ao->prevarena; 1068 ao->prevarena;
1066 } 1069 }
1067 /* Record that this arena_object slot is 1070 /* Record that this arena_object slot is
1068 * available to be reused. 1071 * available to be reused.
1069 */ 1072 */
1070 ao->nextarena = unused_arena_objects; 1073 ao->nextarena = unused_arena_objects;
1071 unused_arena_objects = ao; 1074 unused_arena_objects = ao;
1072 1075
1073 /* Free the entire arena. */ 1076 /* Free the entire arena. */
1077 #ifdef ARENAS_USE_MMAP
1078 munmap((void *)ao->address, ARENA_SIZE);
1079 #else
1074 free((void *)ao->address); 1080 free((void *)ao->address);
1081 #endif
1075 ao->address = 0; /* mark unassociated */ 1082 ao->address = 0; /* mark unassociated */
1076 --narenas_currently_allocated; 1083 --narenas_currently_allocated;
1077 1084
1078 UNLOCK(); 1085 UNLOCK();
1079 return; 1086 return;
1080 } 1087 }
1081 if (nf == 1) { 1088 if (nf == 1) {
1082 /* Case 2. Put ao at the head of 1089 /* Case 2. Put ao at the head of
1083 * usable_arenas. Note that because 1090 * usable_arenas. Note that because
1084 * ao->nfreepools was 0 before, ao isn't 1091 * ao->nfreepools was 0 before, ao isn't
(...skipping 811 matching lines...) Expand 10 before | Expand all | Expand 10 after
1896 int 1903 int
1897 Py_ADDRESS_IN_RANGE(void *P, poolp pool) 1904 Py_ADDRESS_IN_RANGE(void *P, poolp pool)
1898 { 1905 {
1899 uint arenaindex_temp = pool->arenaindex; 1906 uint arenaindex_temp = pool->arenaindex;
1900 1907
1901 return arenaindex_temp < maxarenas && 1908 return arenaindex_temp < maxarenas &&
1902 (uptr)P - arenas[arenaindex_temp].address < (uptr)ARENA_SIZE && 1909 (uptr)P - arenas[arenaindex_temp].address < (uptr)ARENA_SIZE &&
1903 arenas[arenaindex_temp].address != 0; 1910 arenas[arenaindex_temp].address != 0;
1904 } 1911 }
1905 #endif 1912 #endif
OLDNEW
« no previous file with comments | « configure.in ('k') | pyconfig.h.in » ('j') | no next file with comments »

RSS Feeds Recent Issues | This issue
This is Rietveld 894c83f36cb7+