### Bug Summary

File: | Objects/obmalloc.c
Location: | line 1122, column 13
Description: | Access to field 'nextarena' results in a dereference of a null pointer (loaded from field 'prevarena')

### Annotated Source Code

1 | #include "Python.h"
2 |
3 | #ifdef WITH_PYMALLOC1
4 |
5 | #ifdef WITH_VALGRIND
6 | #include <valgrind/valgrind.h>
7 |
8 | /* If we're using GCC, use __builtin_expect() to reduce overhead of
9 | the valgrind checks */
10 | #if defined(__GNUC__4) && (__GNUC__4 > 2) && defined(__OPTIMIZE__)
11 | # define UNLIKELY(value) __builtin_expect((value), 0)
12 | #else
13 | # define UNLIKELY(value) (value)
14 | #endif
15 |
16 | /* -1 indicates that we haven't checked that we're running on valgrind yet. */
17 | static int running_on_valgrind = -1;
18 | #endif
19 |
20 | /* An object allocator for Python.
21 |
22 | Here is an introduction to the layers of the Python memory architecture,
23 | showing where the object allocator is actually used (layer +2), It is
24 | called for every object allocation and deallocation (PyObject_New/Del),
25 | unless the object-specific allocators implement a proprietary allocation
26 | scheme (ex.: ints use a simple free list). This is also the place where
27 | the cyclic garbage collector operates selectively on container objects.
28 |
29 |
30 | Object-specific allocators
31 | _____ ______ ______ ________
32 | [ int ] [ dict ] [ list ] ... [ string ] Python core |
33 | +3 | <----- Object-specific memory -----> | <-- Non-object memory --> |
34 | _______________________________ | |
35 | [ Python's object allocator ] | |
36 | +2 | ####### Object memory ####### | <------ Internal buffers ------> |
37 | ______________________________________________________________ |
38 | [ Python's raw memory allocator (PyMem_ API) ] |
39 | +1 | <----- Python memory (under PyMem manager's control) ------> | |
40 | __________________________________________________________________
41 | [ Underlying general-purpose allocator (ex: C library malloc) ]
42 | 0 | <------ Virtual memory allocated for the python process -------> |
43 |
44 | =========================================================================
45 | _______________________________________________________________________
46 | [ OS-specific Virtual Memory Manager (VMM) ]
47 | -1 | <--- Kernel dynamic storage allocation & management (page-based) ---> |
48 | __________________________________ __________________________________
49 | [ ] [ ]
50 | -2 | <-- Physical memory: ROM/RAM --> | | <-- Secondary storage (swap) --> |
51 |
52 | */
53 | /*==========================================================================*/
54 |
55 | /* A fast, special-purpose memory allocator for small blocks, to be used
56 | on top of a general-purpose malloc -- heavily based on previous art. */
57 |
58 | /* Vladimir Marangozov -- August 2000 */
59 |
60 | /*
61 | * "Memory management is where the rubber meets the road -- if we do the wrong
62 | * thing at any level, the results will not be good. And if we don't make the
63 | * levels work well together, we are in serious trouble." (1)
64 | *
65 | * (1) Paul R. Wilson, Mark S. Johnstone, Michael Neely, and David Boles,
66 | * "Dynamic Storage Allocation: A Survey and Critical Review",
67 | * in Proc. 1995 Int'l. Workshop on Memory Management, September 1995.
68 | */
69 |
70 | /* #undef WITH_MEMORY_LIMITS */ /* disable mem limit checks */
71 |
72 | /*==========================================================================*/
73 |
74 | /*
75 | * Allocation strategy abstract:
76 | *
77 | * For small requests, the allocator sub-allocates <Big> blocks of memory.
78 | * Requests greater than 256 bytes are routed to the system's allocator.
79 | *
80 | * Small requests are grouped in size classes spaced 8 bytes apart, due
81 | * to the required valid alignment of the returned address. Requests of
82 | * a particular size are serviced from memory pools of 4K (one VMM page).
83 | * Pools are fragmented on demand and contain free lists of blocks of one
84 | * particular size class. In other words, there is a fixed-size allocator
85 | * for each size class. Free pools are shared by the different allocators
86 | * thus minimizing the space reserved for a particular size class.
87 | *
88 | * This allocation strategy is a variant of what is known as "simple
89 | * segregated storage based on array of free lists". The main drawback of
90 | * simple segregated storage is that we might end up with lot of reserved
91 | * memory for the different free lists, which degenerate in time. To avoid
92 | * this, we partition each free list in pools and we share dynamically the
93 | * reserved space between all free lists. This technique is quite efficient
94 | * for memory intensive programs which allocate mainly small-sized blocks.
95 | *
96 | * For small requests we have the following table:
97 | *
98 | * Request in bytes Size of allocated block Size class idx
99 | * ----------------------------------------------------------------
100 | * 1-8 8 0
101 | * 9-16 16 1
102 | * 17-24 24 2
103 | * 25-32 32 3
104 | * 33-40 40 4
105 | * 41-48 48 5
106 | * 49-56 56 6
107 | * 57-64 64 7
108 | * 65-72 72 8
109 | * ... ... ...
110 | * 241-248 248 30
111 | * 249-256 256 31
112 | *
113 | * 0, 257 and up: routed to the underlying allocator.
114 | */
115 |
116 | /*==========================================================================*/
117 |
118 | /*
119 | * -- Main tunable settings section --
120 | */
121 |
122 | /*
123 | * Alignment of addresses returned to the user. 8-bytes alignment works
124 | * on most current architectures (with 32-bit or 64-bit address busses).
125 | * The alignment value is also used for grouping small requests in size
126 | * classes spaced ALIGNMENT bytes apart.
127 | *
128 | * You shouldn't change this unless you know what you are doing.
129 | */
130 | #define ALIGNMENT8 8 /* must be 2^N */
131 | #define ALIGNMENT_SHIFT3 3
132 | #define ALIGNMENT_MASK(8 - 1) (ALIGNMENT8 - 1)
133 |
134 | /* Return the number of bytes in size class I, as a uint. */
135 | #define INDEX2SIZE(I)(((unsigned int)(I) + 1) << 3) (((uintunsigned int)(I) + 1) << ALIGNMENT_SHIFT3)
136 |
137 | /*
138 | * Max size threshold below which malloc requests are considered to be
139 | * small enough in order to use preallocated memory pools. You can tune
140 | * this value according to your application behaviour and memory needs.
141 | *
142 | * The following invariants must hold:
143 | * 1) ALIGNMENT <= SMALL_REQUEST_THRESHOLD <= 256
144 | * 2) SMALL_REQUEST_THRESHOLD is evenly divisible by ALIGNMENT
145 | *
146 | * Although not required, for better performance and space efficiency,
147 | * it is recommended that SMALL_REQUEST_THRESHOLD is set to a power of 2.
148 | */
149 | #define SMALL_REQUEST_THRESHOLD256 256
150 | #define NB_SMALL_SIZE_CLASSES(256 / 8) (SMALL_REQUEST_THRESHOLD256 / ALIGNMENT8)
151 |
152 | /*
153 | * The system's VMM page size can be obtained on most unices with a
154 | * getpagesize() call or deduced from various header files. To make
155 | * things simpler, we assume that it is 4K, which is OK for most systems.
156 | * It is probably better if this is the native page size, but it doesn't
157 | * have to be. In theory, if SYSTEM_PAGE_SIZE is larger than the native page
158 | * size, then `POOL_ADDR(p)->arenaindex' could rarely cause a segmentation
159 | * violation fault. 4K is apparently OK for all the platforms that python
160 | * currently targets.
161 | */
162 | #define SYSTEM_PAGE_SIZE(4 * 1024) (4 * 1024)
163 | #define SYSTEM_PAGE_SIZE_MASK((4 * 1024) - 1) (SYSTEM_PAGE_SIZE(4 * 1024) - 1)
164 |
165 | /*
166 | * Maximum amount of memory managed by the allocator for small requests.
167 | */
168 | #ifdef WITH_MEMORY_LIMITS
169 | #ifndef SMALL_MEMORY_LIMIT
170 | #define SMALL_MEMORY_LIMIT (64 * 1024 * 1024) /* 64 MB -- more? */
171 | #endif
172 | #endif
173 |
174 | /*
175 | * The allocator sub-allocates <Big> blocks of memory (called arenas) aligned
176 | * on a page boundary. This is a reserved virtual address space for the
177 | * current process (obtained through a malloc call). In no way this means
178 | * that the memory arenas will be used entirely. A malloc(<Big>) is usually
179 | * an address range reservation for <Big> bytes, unless all pages within this
180 | * space are referenced subsequently. So malloc'ing big blocks and not using
181 | * them does not mean "wasting memory". It's an addressable range wastage...
182 | *
183 | * Therefore, allocating arenas with malloc is not optimal, because there is
184 | * some address space wastage, but this is the most portable way to request
185 | * memory from the system across various platforms.
186 | */
187 | #define ARENA_SIZE(256 << 10) (256 << 10) /* 256KB */
188 |
189 | #ifdef WITH_MEMORY_LIMITS
190 | #define MAX_ARENAS (SMALL_MEMORY_LIMIT / ARENA_SIZE(256 << 10))
191 | #endif
192 |
193 | /*
194 | * Size of the pools used for small blocks. Should be a power of 2,
195 | * between 1K and SYSTEM_PAGE_SIZE, that is: 1k, 2k, 4k.
196 | */
197 | #define POOL_SIZE(4 * 1024) SYSTEM_PAGE_SIZE(4 * 1024) /* must be 2^N */
198 | #define POOL_SIZE_MASK((4 * 1024) - 1) SYSTEM_PAGE_SIZE_MASK((4 * 1024) - 1)
199 |
200 | /*
201 | * -- End of tunable settings section --
202 | */
203 |
204 | /*==========================================================================*/
205 |
206 | /*
207 | * Locking
208 | *
209 | * To reduce lock contention, it would probably be better to refine the
210 | * crude function locking with per size class locking. I'm not positive
211 | * however, whether it's worth switching to such locking policy because
212 | * of the performance penalty it might introduce.
213 | *
214 | * The following macros describe the simplest (should also be the fastest)
215 | * lock object on a particular platform and the init/fini/lock/unlock
216 | * operations on it. The locks defined here are not expected to be recursive
217 | * because it is assumed that they will always be called in the order:
218 | * INIT, [LOCK, UNLOCK]*, FINI.
219 | */
220 |
221 | /*
222 | * Python's threads are serialized, so object malloc locking is disabled.
223 | */
224 | #define SIMPLELOCK_DECL(lock) /* simple lock declaration */
225 | #define SIMPLELOCK_INIT(lock) /* allocate (if needed) and initialize */
226 | #define SIMPLELOCK_FINI(lock) /* free/destroy an existing lock */
227 | #define SIMPLELOCK_LOCK(lock) /* acquire released lock */
228 | #define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */
229 |
230 | /*
231 | * Basic types
232 | * I don't care if these are defined in <sys/types.h> or elsewhere. Axiom.
233 | */
234 | #undef ucharunsigned char
235 | #define ucharunsigned char unsigned char /* assuming == 8 bits */
236 |
237 | #undef uintunsigned int
238 | #define uintunsigned int unsigned int /* assuming >= 16 bits */
239 |
240 | #undef ulongunsigned long
241 | #define ulongunsigned long unsigned long /* assuming >= 32 bits */
242 |
243 | #undef uptrPy_uintptr_t
244 | #define uptrPy_uintptr_t Py_uintptr_t
245 |
246 | /* When you say memory, my mind reasons in terms of (pointers to) blocks */
247 | typedef ucharunsigned char block;
248 |
249 | /* Pool for small blocks. */
250 | struct pool_header {
251 | union { block *_padding;
252 | uintunsigned int count; } ref; /* number of allocated blocks */
253 | block *freeblock; /* pool's free list head */
254 | struct pool_header *nextpool; /* next pool of this size class */
255 | struct pool_header *prevpool; /* previous pool "" */
256 | uintunsigned int arenaindex; /* index into arenas of base adr */
257 | uintunsigned int szidx; /* block size class index */
258 | uintunsigned int nextoffset; /* bytes to virgin block */
259 | uintunsigned int maxnextoffset; /* largest valid nextoffset */
260 | };
261 |
262 | typedef struct pool_header *poolp;
263 |
264 | /* Record keeping for arenas. */
265 | struct arena_object {
266 | /* The address of the arena, as returned by malloc. Note that 0
267 | * will never be returned by a successful malloc, and is used
268 | * here to mark an arena_object that doesn't correspond to an
269 | * allocated arena.
270 | */
271 | uptrPy_uintptr_t address;
272 |
273 | /* Pool-aligned pointer to the next pool to be carved off. */
274 | block* pool_address;
275 |
276 | /* The number of available pools in the arena: free pools + never-
277 | * allocated pools.
278 | */
279 | uintunsigned int nfreepools;
280 |
281 | /* The total number of pools in the arena, whether or not available. */
282 | uintunsigned int ntotalpools;
283 |
284 | /* Singly-linked list of available pools. */
285 | struct pool_header* freepools;
286 |
287 | /* Whenever this arena_object is not associated with an allocated
288 | * arena, the nextarena member is used to link all unassociated
289 | * arena_objects in the singly-linked `unused_arena_objects` list.
290 | * The prevarena member is unused in this case.
291 | *
292 | * When this arena_object is associated with an allocated arena
293 | * with at least one available pool, both members are used in the
294 | * doubly-linked `usable_arenas` list, which is maintained in
295 | * increasing order of `nfreepools` values.
296 | *
297 | * Else this arena_object is associated with an allocated arena
298 | * all of whose pools are in use. `nextarena` and `prevarena`
299 | * are both meaningless in this case.
300 | */
301 | struct arena_object* nextarena;
302 | struct arena_object* prevarena;
303 | };
304 |
305 | #undef ROUNDUP
306 | #define ROUNDUP(x)(((x) + (8 - 1)) & ~(8 - 1)) (((x) + ALIGNMENT_MASK(8 - 1)) & ~ALIGNMENT_MASK(8 - 1))
307 | #define POOL_OVERHEAD(((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1)) ROUNDUP(sizeof(struct pool_header))(((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1))
308 |
309 | #define DUMMY_SIZE_IDX0xffff 0xffff /* size class of newly cached pools */
310 |
311 | /* Round pointer P down to the closest pool-aligned address <= P, as a poolp */
312 | #define POOL_ADDR(P)((poolp)((Py_uintptr_t)(P) & ~(Py_uintptr_t)((4 * 1024) - 1))) ((poolp)((uptrPy_uintptr_t)(P) & ~(uptrPy_uintptr_t)POOL_SIZE_MASK((4 * 1024) - 1)))
313 |
314 | /* Return total number of blocks in pool of size index I, as a uint. */
315 | #define NUMBLOCKS(I)((unsigned int)((4 * 1024) - (((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1))) / (((unsigned int)(I) + 1) << 3)) ((uintunsigned int)(POOL_SIZE(4 * 1024) - POOL_OVERHEAD(((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1))) / INDEX2SIZE(I)(((unsigned int)(I) + 1) << 3))
316 |
317 | /*==========================================================================*/
318 |
319 | /*
320 | * This malloc lock
321 | */
322 | SIMPLELOCK_DECL(_malloc_lock)
323 | #define LOCK() SIMPLELOCK_LOCK(_malloc_lock)
324 | #define UNLOCK() SIMPLELOCK_UNLOCK(_malloc_lock)
325 | #define LOCK_INIT() SIMPLELOCK_INIT(_malloc_lock)
326 | #define LOCK_FINI() SIMPLELOCK_FINI(_malloc_lock)
327 |
328 | /*
329 | * Pool table -- headed, circular, doubly-linked lists of partially used pools.
330 |
331 | This is involved. For an index i, usedpools[i+i] is the header for a list of
332 | all partially used pools holding small blocks with "size class idx" i. So
333 | usedpools[0] corresponds to blocks of size 8, usedpools[2] to blocks of size
334 | 16, and so on: index 2*i <-> blocks of size (i+1)<<ALIGNMENT_SHIFT.
335 |
336 | Pools are carved off an arena's highwater mark (an arena_object's pool_address
337 | member) as needed. Once carved off, a pool is in one of three states forever
338 | after:
339 |
340 | used == partially used, neither empty nor full
341 | At least one block in the pool is currently allocated, and at least one
342 | block in the pool is not currently allocated (note this implies a pool
343 | has room for at least two blocks).
344 | This is a pool's initial state, as a pool is created only when malloc
345 | needs space.
346 | The pool holds blocks of a fixed size, and is in the circular list headed
347 | at usedpools[i] (see above). It's linked to the other used pools of the
348 | same size class via the pool_header's nextpool and prevpool members.
349 | If all but one block is currently allocated, a malloc can cause a
350 | transition to the full state. If all but one block is not currently
351 | allocated, a free can cause a transition to the empty state.
352 |
353 | full == all the pool's blocks are currently allocated
354 | On transition to full, a pool is unlinked from its usedpools[] list.
355 | It's not linked to from anything then anymore, and its nextpool and
356 | prevpool members are meaningless until it transitions back to used.
357 | A free of a block in a full pool puts the pool back in the used state.
358 | Then it's linked in at the front of the appropriate usedpools[] list, so
359 | that the next allocation for its size class will reuse the freed block.
360 |
361 | empty == all the pool's blocks are currently available for allocation
362 | On transition to empty, a pool is unlinked from its usedpools[] list,
363 | and linked to the front of its arena_object's singly-linked freepools list,
364 | via its nextpool member. The prevpool member has no meaning in this case.
365 | Empty pools have no inherent size class: the next time a malloc finds
366 | an empty list in usedpools[], it takes the first pool off of freepools.
367 | If the size class needed happens to be the same as the size class the pool
368 | last had, some pool initialization can be skipped.
369 |
370 |
371 | Block Management
372 |
373 | Blocks within pools are again carved out as needed. pool->freeblock points to
374 | the start of a singly-linked list of free blocks within the pool. When a
375 | block is freed, it's inserted at the front of its pool's freeblock list. Note
376 | that the available blocks in a pool are *not* linked all together when a pool
377 | is initialized. Instead only "the first two" (lowest addresses) blocks are
378 | set up, returning the first such block, and setting pool->freeblock to a
379 | one-block list holding the second such block. This is consistent with that
380 | pymalloc strives at all levels (arena, pool, and block) never to touch a piece
381 | of memory until it's actually needed.
382 |
383 | So long as a pool is in the used state, we're certain there *is* a block
384 | available for allocating, and pool->freeblock is not NULL. If pool->freeblock
385 | points to the end of the free list before we've carved the entire pool into
386 | blocks, that means we simply haven't yet gotten to one of the higher-address
387 | blocks. The offset from the pool_header to the start of "the next" virgin
388 | block is stored in the pool_header nextoffset member, and the largest value
389 | of nextoffset that makes sense is stored in the maxnextoffset member when a
390 | pool is initialized. All the blocks in a pool have been passed out at least
391 | once when and only when nextoffset > maxnextoffset.
392 |
393 |
394 | Major obscurity: While the usedpools vector is declared to have poolp
395 | entries, it doesn't really. It really contains two pointers per (conceptual)
396 | poolp entry, the nextpool and prevpool members of a pool_header. The
397 | excruciating initialization code below fools C so that
398 |
399 | usedpool[i+i]
400 |
401 | "acts like" a genuine poolp, but only so long as you only reference its
402 | nextpool and prevpool members. The "- 2*sizeof(block *)" gibberish is
403 | compensating for that a pool_header's nextpool and prevpool members
404 | immediately follow a pool_header's first two members:
405 |
406 | union { block *_padding;
407 | uint count; } ref;
408 | block *freeblock;
409 |
410 | each of which consume sizeof(block *) bytes. So what usedpools[i+i] really
411 | contains is a fudged-up pointer p such that *if* C believes it's a poolp
412 | pointer, then p->nextpool and p->prevpool are both p (meaning that the headed
413 | circular list is empty).
414 |
415 | It's unclear why the usedpools setup is so convoluted. It could be to
416 | minimize the amount of cache required to hold this heavily-referenced table
417 | (which only *needs* the two interpool pointer members of a pool_header). OTOH,
418 | referencing code has to remember to "double the index" and doing so isn't
419 | free, usedpools[0] isn't a strictly legal pointer, and we're crucially relying
420 | on that C doesn't insert any padding anywhere in a pool_header at or before
421 | the prevpool member.
422 | **************************************************************************** */
423 |
424 | #define PTA(x)((poolp )((unsigned char *)&(usedpools[2*(x)]) - 2*sizeof (block *))) ((poolp )((ucharunsigned char *)&(usedpools[2*(x)]) - 2*sizeof(block *)))
425 | #define PT(x)((poolp )((unsigned char *)&(usedpools[2*(x)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(x) ]) - 2*sizeof(block *))) PTA(x)((poolp )((unsigned char *)&(usedpools[2*(x)]) - 2*sizeof (block *))), PTA(x)((poolp )((unsigned char *)&(usedpools[2*(x)]) - 2*sizeof (block *)))
426 |
427 | static poolp usedpools[2 * ((NB_SMALL_SIZE_CLASSES(256 / 8) + 7) / 8) * 8] = {
428 | PT(0)((poolp )((unsigned char *)&(usedpools[2*(0)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(0) ]) - 2*sizeof(block *))), PT(1)((poolp )((unsigned char *)&(usedpools[2*(1)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(1) ]) - 2*sizeof(block *))), PT(2)((poolp )((unsigned char *)&(usedpools[2*(2)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(2) ]) - 2*sizeof(block *))), PT(3)((poolp )((unsigned char *)&(usedpools[2*(3)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(3) ]) - 2*sizeof(block *))), PT(4)((poolp )((unsigned char *)&(usedpools[2*(4)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(4) ]) - 2*sizeof(block *))), PT(5)((poolp )((unsigned char *)&(usedpools[2*(5)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(5) ]) - 2*sizeof(block *))), PT(6)((poolp )((unsigned char *)&(usedpools[2*(6)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(6) ]) - 2*sizeof(block *))), PT(7)((poolp )((unsigned char *)&(usedpools[2*(7)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(7) ]) - 2*sizeof(block *)))
429 | #if NB_SMALL_SIZE_CLASSES(256 / 8) > 8
430 | , PT(8)((poolp )((unsigned char *)&(usedpools[2*(8)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(8) ]) - 2*sizeof(block *))), PT(9)((poolp )((unsigned char *)&(usedpools[2*(9)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(9) ]) - 2*sizeof(block *))), PT(10)((poolp )((unsigned char *)&(usedpools[2*(10)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(10 )]) - 2*sizeof(block *))), PT(11)((poolp )((unsigned char *)&(usedpools[2*(11)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(11 )]) - 2*sizeof(block *))), PT(12)((poolp )((unsigned char *)&(usedpools[2*(12)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(12 )]) - 2*sizeof(block *))), PT(13)((poolp )((unsigned char *)&(usedpools[2*(13)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(13 )]) - 2*sizeof(block *))), PT(14)((poolp )((unsigned char *)&(usedpools[2*(14)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(14 )]) - 2*sizeof(block *))), PT(15)((poolp )((unsigned char *)&(usedpools[2*(15)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(15 )]) - 2*sizeof(block *)))
431 | #if NB_SMALL_SIZE_CLASSES(256 / 8) > 16
432 | , PT(16)((poolp )((unsigned char *)&(usedpools[2*(16)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(16 )]) - 2*sizeof(block *))), PT(17)((poolp )((unsigned char *)&(usedpools[2*(17)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(17 )]) - 2*sizeof(block *))), PT(18)((poolp )((unsigned char *)&(usedpools[2*(18)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(18 )]) - 2*sizeof(block *))), PT(19)((poolp )((unsigned char *)&(usedpools[2*(19)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(19 )]) - 2*sizeof(block *))), PT(20)((poolp )((unsigned char *)&(usedpools[2*(20)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(20 )]) - 2*sizeof(block *))), PT(21)((poolp )((unsigned char *)&(usedpools[2*(21)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(21 )]) - 2*sizeof(block *))), PT(22)((poolp )((unsigned char *)&(usedpools[2*(22)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(22 )]) - 2*sizeof(block *))), PT(23)((poolp )((unsigned char *)&(usedpools[2*(23)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(23 )]) - 2*sizeof(block *)))
433 | #if NB_SMALL_SIZE_CLASSES(256 / 8) > 24
434 | , PT(24)((poolp )((unsigned char *)&(usedpools[2*(24)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(24 )]) - 2*sizeof(block *))), PT(25)((poolp )((unsigned char *)&(usedpools[2*(25)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(25 )]) - 2*sizeof(block *))), PT(26)((poolp )((unsigned char *)&(usedpools[2*(26)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(26 )]) - 2*sizeof(block *))), PT(27)((poolp )((unsigned char *)&(usedpools[2*(27)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(27 )]) - 2*sizeof(block *))), PT(28)((poolp )((unsigned char *)&(usedpools[2*(28)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(28 )]) - 2*sizeof(block *))), PT(29)((poolp )((unsigned char *)&(usedpools[2*(29)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(29 )]) - 2*sizeof(block *))), PT(30)((poolp )((unsigned char *)&(usedpools[2*(30)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(30 )]) - 2*sizeof(block *))), PT(31)((poolp )((unsigned char *)&(usedpools[2*(31)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(31 )]) - 2*sizeof(block *)))
435 | #if NB_SMALL_SIZE_CLASSES(256 / 8) > 32
436 | , PT(32)((poolp )((unsigned char *)&(usedpools[2*(32)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(32 )]) - 2*sizeof(block *))), PT(33)((poolp )((unsigned char *)&(usedpools[2*(33)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(33 )]) - 2*sizeof(block *))), PT(34)((poolp )((unsigned char *)&(usedpools[2*(34)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(34 )]) - 2*sizeof(block *))), PT(35)((poolp )((unsigned char *)&(usedpools[2*(35)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(35 )]) - 2*sizeof(block *))), PT(36)((poolp )((unsigned char *)&(usedpools[2*(36)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(36 )]) - 2*sizeof(block *))), PT(37)((poolp )((unsigned char *)&(usedpools[2*(37)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(37 )]) - 2*sizeof(block *))), PT(38)((poolp )((unsigned char *)&(usedpools[2*(38)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(38 )]) - 2*sizeof(block *))), PT(39)((poolp )((unsigned char *)&(usedpools[2*(39)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(39 )]) - 2*sizeof(block *)))
437 | #if NB_SMALL_SIZE_CLASSES(256 / 8) > 40
438 | , PT(40)((poolp )((unsigned char *)&(usedpools[2*(40)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(40 )]) - 2*sizeof(block *))), PT(41)((poolp )((unsigned char *)&(usedpools[2*(41)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(41 )]) - 2*sizeof(block *))), PT(42)((poolp )((unsigned char *)&(usedpools[2*(42)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(42 )]) - 2*sizeof(block *))), PT(43)((poolp )((unsigned char *)&(usedpools[2*(43)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(43 )]) - 2*sizeof(block *))), PT(44)((poolp )((unsigned char *)&(usedpools[2*(44)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(44 )]) - 2*sizeof(block *))), PT(45)((poolp )((unsigned char *)&(usedpools[2*(45)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(45 )]) - 2*sizeof(block *))), PT(46)((poolp )((unsigned char *)&(usedpools[2*(46)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(46 )]) - 2*sizeof(block *))), PT(47)((poolp )((unsigned char *)&(usedpools[2*(47)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(47 )]) - 2*sizeof(block *)))
439 | #if NB_SMALL_SIZE_CLASSES(256 / 8) > 48
440 | , PT(48)((poolp )((unsigned char *)&(usedpools[2*(48)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(48 )]) - 2*sizeof(block *))), PT(49)((poolp )((unsigned char *)&(usedpools[2*(49)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(49 )]) - 2*sizeof(block *))), PT(50)((poolp )((unsigned char *)&(usedpools[2*(50)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(50 )]) - 2*sizeof(block *))), PT(51)((poolp )((unsigned char *)&(usedpools[2*(51)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(51 )]) - 2*sizeof(block *))), PT(52)((poolp )((unsigned char *)&(usedpools[2*(52)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(52 )]) - 2*sizeof(block *))), PT(53)((poolp )((unsigned char *)&(usedpools[2*(53)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(53 )]) - 2*sizeof(block *))), PT(54)((poolp )((unsigned char *)&(usedpools[2*(54)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(54 )]) - 2*sizeof(block *))), PT(55)((poolp )((unsigned char *)&(usedpools[2*(55)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(55 )]) - 2*sizeof(block *)))
441 | #if NB_SMALL_SIZE_CLASSES(256 / 8) > 56
442 | , PT(56)((poolp )((unsigned char *)&(usedpools[2*(56)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(56 )]) - 2*sizeof(block *))), PT(57)((poolp )((unsigned char *)&(usedpools[2*(57)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(57 )]) - 2*sizeof(block *))), PT(58)((poolp )((unsigned char *)&(usedpools[2*(58)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(58 )]) - 2*sizeof(block *))), PT(59)((poolp )((unsigned char *)&(usedpools[2*(59)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(59 )]) - 2*sizeof(block *))), PT(60)((poolp )((unsigned char *)&(usedpools[2*(60)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(60 )]) - 2*sizeof(block *))), PT(61)((poolp )((unsigned char *)&(usedpools[2*(61)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(61 )]) - 2*sizeof(block *))), PT(62)((poolp )((unsigned char *)&(usedpools[2*(62)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(62 )]) - 2*sizeof(block *))), PT(63)((poolp )((unsigned char *)&(usedpools[2*(63)]) - 2*sizeof (block *))), ((poolp )((unsigned char *)&(usedpools[2*(63 )]) - 2*sizeof(block *)))
443 | #endif /* NB_SMALL_SIZE_CLASSES > 56 */
444 | #endif /* NB_SMALL_SIZE_CLASSES > 48 */
445 | #endif /* NB_SMALL_SIZE_CLASSES > 40 */
446 | #endif /* NB_SMALL_SIZE_CLASSES > 32 */
447 | #endif /* NB_SMALL_SIZE_CLASSES > 24 */
448 | #endif /* NB_SMALL_SIZE_CLASSES > 16 */
449 | #endif /* NB_SMALL_SIZE_CLASSES > 8 */
450 | };
451 |
452 | /*==========================================================================
453 | Arena management.
454 |
455 | `arenas` is a vector of arena_objects. It contains maxarenas entries, some of
456 | which may not be currently used (== they're arena_objects that aren't
457 | currently associated with an allocated arena). Note that arenas proper are
458 | separately malloc'ed.
459 |
460 | Prior to Python 2.5, arenas were never free()'ed. Starting with Python 2.5,
461 | we do try to free() arenas, and use some mild heuristic strategies to increase
462 | the likelihood that arenas eventually can be freed.
463 |
464 | unused_arena_objects
465 |
466 | This is a singly-linked list of the arena_objects that are currently not
467 | being used (no arena is associated with them). Objects are taken off the
468 | head of the list in new_arena(), and are pushed on the head of the list in
469 | PyObject_Free() when the arena is empty. Key invariant: an arena_object
470 | is on this list if and only if its .address member is 0.
471 |
472 | usable_arenas
473 |
474 | This is a doubly-linked list of the arena_objects associated with arenas
475 | that have pools available. These pools are either waiting to be reused,
476 | or have not been used before. The list is sorted to have the most-
477 | allocated arenas first (ascending order based on the nfreepools member).
478 | This means that the next allocation will come from a heavily used arena,
479 | which gives the nearly empty arenas a chance to be returned to the system.
480 | In my unscientific tests this dramatically improved the number of arenas
481 | that could be freed.
482 |
483 | Note that an arena_object associated with an arena all of whose pools are
484 | currently in use isn't on either list.
485 | */
486 |
487 | /* Array of objects used to track chunks of memory (arenas). */
488 | static struct arena_object* arenas = NULL((void *)0);
489 | /* Number of slots currently allocated in the `arenas` vector. */
490 | static uintunsigned int maxarenas = 0;
491 |
492 | /* The head of the singly-linked, NULL-terminated list of available
493 | * arena_objects.
494 | */
495 | static struct arena_object* unused_arena_objects = NULL((void *)0);
496 |
497 | /* The head of the doubly-linked, NULL-terminated at each end, list of
498 | * arena_objects associated with arenas that have pools available.
499 | */
500 | static struct arena_object* usable_arenas = NULL((void *)0);
501 |
502 | /* How many arena_objects do we initially allocate?
503 | * 16 = can allocate 16 arenas = 16 * ARENA_SIZE = 4MB before growing the
504 | * `arenas` vector.
505 | */
506 | #define INITIAL_ARENA_OBJECTS16 16
507 |
508 | /* Number of arenas allocated that haven't been free()'d. */
509 | static size_t narenas_currently_allocated = 0;
510 |
511 | #ifdef PYMALLOC_DEBUG
512 | /* Total number of times malloc() called to allocate an arena. */
513 | static size_t ntimes_arena_allocated = 0;
514 | /* High water mark (max value ever seen) for narenas_currently_allocated. */
515 | static size_t narenas_highwater = 0;
516 | #endif
517 |
518 | /* Allocate a new arena. If we run out of memory, return NULL. Else
519 | * allocate a new arena, and return the address of an arena_object
520 | * describing the new arena. It's expected that the caller will set
521 | * `usable_arenas` to the return value.
522 | */
523 | static struct arena_object*
524 | new_arena(void)
525 | {
526 | struct arena_object* arenaobj;
527 | uintunsigned int excess; /* number of bytes above pool alignment */
528 |
529 | #ifdef PYMALLOC_DEBUG
530 | if (Py_GETENV("PYTHONMALLOCSTATS")(Py_IgnoreEnvironmentFlag ? ((void *)0) : getenv("PYTHONMALLOCSTATS" )))
531 | _PyObject_DebugMallocStats();
532 | #endif
533 | if (unused_arena_objects == NULL((void *)0)) {
534 | uintunsigned int i;
535 | uintunsigned int numarenas;
536 | size_t nbytes;
537 |
538 | /* Double the number of arena objects on each allocation.
539 | * Note that it's possible for `numarenas` to overflow.
540 | */
541 | numarenas = maxarenas ? maxarenas << 1 : INITIAL_ARENA_OBJECTS16;
542 | if (numarenas <= maxarenas)
543 | return NULL((void *)0); /* overflow */
544 | #if SIZEOF_SIZE_T8 <= SIZEOF_INT4
545 | if (numarenas > PY_SIZE_MAX18446744073709551615ULL / sizeof(*arenas))
546 | return NULL((void *)0); /* overflow */
547 | #endif
548 | nbytes = numarenas * sizeof(*arenas);
549 | arenaobj = (struct arena_object *)realloc(arenas, nbytes);
550 | if (arenaobj == NULL((void *)0))
551 | return NULL((void *)0);
552 | arenas = arenaobj;
553 |
554 | /* We might need to fix pointers that were copied. However,
555 | * new_arena only gets called when all the pages in the
556 | * previous arenas are full. Thus, there are *no* pointers
557 | * into the old array. Thus, we don't have to worry about
558 | * invalid pointers. Just to be sure, some asserts:
559 | */
560 | assert(usable_arenas == NULL)(__builtin_expect(!(usable_arenas == ((void *)0)), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 560, "usable_arenas == NULL" ) : (void)0);
561 | assert(unused_arena_objects == NULL)(__builtin_expect(!(unused_arena_objects == ((void *)0)), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 561, "unused_arena_objects == NULL" ) : (void)0);
562 |
563 | /* Put the new arenas on the unused_arena_objects list. */
564 | for (i = maxarenas; i < numarenas; ++i) {
565 | arenas[i].address = 0; /* mark as unassociated */
566 | arenas[i].nextarena = i < numarenas - 1 ?
567 | &arenas[i+1] : NULL((void *)0);
568 | }
569 |
570 | /* Update globals. */
571 | unused_arena_objects = &arenas[maxarenas];
572 | maxarenas = numarenas;
573 | }
574 |
575 | /* Take the next available arena object off the head of the list. */
576 | assert(unused_arena_objects != NULL)(__builtin_expect(!(unused_arena_objects != ((void *)0)), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 576, "unused_arena_objects != NULL" ) : (void)0);
577 | arenaobj = unused_arena_objects;
578 | unused_arena_objects = arenaobj->nextarena;
579 | assert(arenaobj->address == 0)(__builtin_expect(!(arenaobj->address == 0), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 579, "arenaobj->address == 0" ) : (void)0);
580 | arenaobj->address = (uptrPy_uintptr_t)malloc(ARENA_SIZE(256 << 10));
581 | if (arenaobj->address == 0) {
582 | /* The allocation failed: return NULL after putting the
583 | * arenaobj back.
584 | */
585 | arenaobj->nextarena = unused_arena_objects;
586 | unused_arena_objects = arenaobj;
587 | return NULL((void *)0);
588 | }
589 |
590 | ++narenas_currently_allocated;
591 | #ifdef PYMALLOC_DEBUG
592 | ++ntimes_arena_allocated;
593 | if (narenas_currently_allocated > narenas_highwater)
594 | narenas_highwater = narenas_currently_allocated;
595 | #endif
596 | arenaobj->freepools = NULL((void *)0);
597 | /* pool_address <- first pool-aligned address in the arena
598 | nfreepools <- number of whole pools that fit after alignment */
599 | arenaobj->pool_address = (block*)arenaobj->address;
600 | arenaobj->nfreepools = ARENA_SIZE(256 << 10) / POOL_SIZE(4 * 1024);
601 | assert(POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE)(__builtin_expect(!((4 * 1024) * arenaobj->nfreepools == ( 256 << 10)), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 601, "POOL_SIZE * arenaobj->nfreepools == ARENA_SIZE") : (void)0);
602 | excess = (uintunsigned int)(arenaobj->address & POOL_SIZE_MASK((4 * 1024) - 1));
603 | if (excess != 0) {
604 | --arenaobj->nfreepools;
605 | arenaobj->pool_address += POOL_SIZE(4 * 1024) - excess;
606 | }
607 | arenaobj->ntotalpools = arenaobj->nfreepools;
608 |
609 | return arenaobj;
610 | }
611 |
612 | /*
613 | Py_ADDRESS_IN_RANGE(P, POOL)
614 |
615 | Return true if and only if P is an address that was allocated by pymalloc.
616 | POOL must be the pool address associated with P, i.e., POOL = POOL_ADDR(P)
617 | (the caller is asked to compute this because the macro expands POOL more than
618 | once, and for efficiency it's best for the caller to assign POOL_ADDR(P) to a
619 | variable and pass the latter to the macro; because Py_ADDRESS_IN_RANGE is
620 | called on every alloc/realloc/free, micro-efficiency is important here).
621 |
622 | Tricky: Let B be the arena base address associated with the pool, B =
623 | arenas[(POOL)->arenaindex].address. Then P belongs to the arena if and only if
624 |
625 | B <= P < B + ARENA_SIZE
626 |
627 | Subtracting B throughout, this is true iff
628 |
629 | 0 <= P-B < ARENA_SIZE
630 |
631 | By using unsigned arithmetic, the "0 <=" half of the test can be skipped.
632 |
633 | Obscure: A PyMem "free memory" function can call the pymalloc free or realloc
634 | before the first arena has been allocated. `arenas` is still NULL in that
635 | case. We're relying on that maxarenas is also 0 in that case, so that
636 | (POOL)->arenaindex < maxarenas must be false, saving us from trying to index
637 | into a NULL arenas.
638 |
639 | Details: given P and POOL, the arena_object corresponding to P is AO =
640 | arenas[(POOL)->arenaindex]. Suppose obmalloc controls P. Then (barring wild
641 | stores, etc), POOL is the correct address of P's pool, AO.address is the
642 | correct base address of the pool's arena, and P must be within ARENA_SIZE of
643 | AO.address. In addition, AO.address is not 0 (no arena can start at address 0
644 | (NULL)). Therefore Py_ADDRESS_IN_RANGE correctly reports that obmalloc
645 | controls P.
646 |
647 | Now suppose obmalloc does not control P (e.g., P was obtained via a direct
648 | call to the system malloc() or realloc()). (POOL)->arenaindex may be anything
649 | in this case -- it may even be uninitialized trash. If the trash arenaindex
650 | is >= maxarenas, the macro correctly concludes at once that obmalloc doesn't
651 | control P.
652 |
653 | Else arenaindex is < maxarena, and AO is read up. If AO corresponds to an
654 | allocated arena, obmalloc controls all the memory in slice AO.address :
655 | AO.address+ARENA_SIZE. By case assumption, P is not controlled by obmalloc,
656 | so P doesn't lie in that slice, so the macro correctly reports that P is not
657 | controlled by obmalloc.
658 |
659 | Finally, if P is not controlled by obmalloc and AO corresponds to an unused
660 | arena_object (one not currently associated with an allocated arena),
661 | AO.address is 0, and the second test in the macro reduces to:
662 |
663 | P < ARENA_SIZE
664 |
665 | If P >= ARENA_SIZE (extremely likely), the macro again correctly concludes
666 | that P is not controlled by obmalloc. However, if P < ARENA_SIZE, this part
667 | of the test still passes, and the third clause (AO.address != 0) is necessary
668 | to get the correct result: AO.address is 0 in this case, so the macro
669 | correctly reports that P is not controlled by obmalloc (despite that P lies in
670 | slice AO.address : AO.address + ARENA_SIZE).
671 |
672 | Note: The third (AO.address != 0) clause was added in Python 2.5. Before
673 | 2.5, arenas were never free()'ed, and an arenaindex < maxarena always
674 | corresponded to a currently-allocated arena, so the "P is not controlled by
675 | obmalloc, AO corresponds to an unused arena_object, and P < ARENA_SIZE" case
676 | was impossible.
677 |
678 | Note that the logic is excruciating, and reading up possibly uninitialized
679 | memory when P is not controlled by obmalloc (to get at (POOL)->arenaindex)
680 | creates problems for some memory debuggers. The overwhelming advantage is
681 | that this test determines whether an arbitrary address is controlled by
682 | obmalloc in a small constant time, independent of the number of arenas
683 | obmalloc controls. Since this test is needed at every entry point, it's
684 | extremely desirable that it be this fast.
685 |
686 | Since Py_ADDRESS_IN_RANGE may be reading from memory which was not allocated
687 | by Python, it is important that (POOL)->arenaindex is read only once, as
688 | another thread may be concurrently modifying the value without holding the
689 | GIL. To accomplish this, the arenaindex_temp variable is used to store
690 | (POOL)->arenaindex for the duration of the Py_ADDRESS_IN_RANGE macro's
691 | execution. The caller of the macro is responsible for declaring this
692 | variable.
693 | */
694 | #define Py_ADDRESS_IN_RANGE(P, POOL)((arenaindex_temp = (POOL)->arenaindex) < maxarenas && (Py_uintptr_t)(P) - arenas[arenaindex_temp].address < (Py_uintptr_t )(256 << 10) && arenas[arenaindex_temp].address != 0) \
695 | ((arenaindex_temp = (POOL)->arenaindex) < maxarenas && \
696 | (uptrPy_uintptr_t)(P) - arenas[arenaindex_temp].address < (uptrPy_uintptr_t)ARENA_SIZE(256 << 10) && \
697 | arenas[arenaindex_temp].address != 0)
698 |
699 |
700 | /* This is only useful when running memory debuggers such as
701 | * Purify or Valgrind. Uncomment to use.
702 | *
703 | #define Py_USING_MEMORY_DEBUGGER
704 | */
705 |
706 | #ifdef Py_USING_MEMORY_DEBUGGER
707 |
708 | /* Py_ADDRESS_IN_RANGE may access uninitialized memory by design
709 | * This leads to thousands of spurious warnings when using
710 | * Purify or Valgrind. By making a function, we can easily
711 | * suppress the uninitialized memory reads in this one function.
712 | * So we won't ignore real errors elsewhere.
713 | *
714 | * Disable the macro and use a function.
715 | */
716 |
717 | #undef Py_ADDRESS_IN_RANGE
718 |
719 | #if defined(__GNUC__4) && ((__GNUC__4 == 3) && (__GNUC_MINOR__2 >= 1) || \
720 | (__GNUC__4 >= 4))
721 | #define Py_NO_INLINE __attribute__((__noinline__))
722 | #else
723 | #define Py_NO_INLINE
724 | #endif
725 |
726 | /* Don't make static, to try to ensure this isn't inlined. */
727 | int Py_ADDRESS_IN_RANGE(void *P, poolp pool)((arenaindex_temp = (poolp pool)->arenaindex) < maxarenas && (Py_uintptr_t)(void *P) - arenas[arenaindex_temp] .address < (Py_uintptr_t)(256 << 10) && arenas [arenaindex_temp].address != 0) Py_NO_INLINE;
728 | #undef Py_NO_INLINE
729 | #endif
730 |
731 | /*==========================================================================*/
732 |
733 | /* malloc. Note that nbytes==0 tries to return a non-NULL pointer, distinct
734 | * from all other currently live pointers. This may not be possible.
735 | */
736 |
737 | /*
738 | * The basic blocks are ordered by decreasing execution frequency,
739 | * which minimizes the number of jumps in the most common cases,
740 | * improves branching prediction and instruction scheduling (small
741 | * block allocations typically result in a couple of instructions).
742 | * Unless the optimizer reorders everything, being too smart...
743 | */
744 |
745 | #undef PyObject_Malloc
746 | void *
747 | PyObject_Malloc(size_t nbytes)
748 | {
749 | block *bp;
750 | poolp pool;
751 | poolp next;
752 | uintunsigned int size;
753 |
754 | #ifdef WITH_VALGRIND
755 | if (UNLIKELY(running_on_valgrind == -1))
756 | running_on_valgrind = RUNNING_ON_VALGRIND;
757 | if (UNLIKELY(running_on_valgrind))
758 | goto redirect;
759 | #endif
760 |
761 | /*
762 | * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
763 | * Most python internals blindly use a signed Py_ssize_t to track
764 | * things without checking for overflows or negatives.
765 | * As size_t is unsigned, checking for nbytes < 0 is not required.
766 | */
767 | if (nbytes > PY_SSIZE_T_MAX((Py_ssize_t)(((size_t)-1)>>1)))
768 | return NULL((void *)0);
769 |
770 | /*
771 | * This implicitly redirects malloc(0).
772 | */
773 | if ((nbytes - 1) < SMALL_REQUEST_THRESHOLD256) {
774 | LOCK();
775 | /*
776 | * Most frequent paths first
777 | */
778 | size = (uintunsigned int)(nbytes - 1) >> ALIGNMENT_SHIFT3;
779 | pool = usedpools[size + size];
780 | if (pool != pool->nextpool) {
781 | /*
782 | * There is a used pool for this size class.
783 | * Pick up the head block of its free list.
784 | */
785 | ++pool->ref.count;
786 | bp = pool->freeblock;
787 | assert(bp != NULL)(__builtin_expect(!(bp != ((void *)0)), 0) ? __assert_rtn(__func__ , "Objects/obmalloc.c", 787, "bp != NULL") : (void)0);
788 | if ((pool->freeblock = *(block **)bp) != NULL((void *)0)) {
789 | UNLOCK();
790 | return (void *)bp;
791 | }
792 | /*
793 | * Reached the end of the free list, try to extend it.
794 | */
795 | if (pool->nextoffset <= pool->maxnextoffset) {
796 | /* There is room for another block. */
797 | pool->freeblock = (block*)pool +
798 | pool->nextoffset;
799 | pool->nextoffset += INDEX2SIZE(size)(((unsigned int)(size) + 1) << 3);
800 | *(block **)(pool->freeblock) = NULL((void *)0);
801 | UNLOCK();
802 | return (void *)bp;
803 | }
804 | /* Pool is full, unlink from used pools. */
805 | next = pool->nextpool;
806 | pool = pool->prevpool;
807 | next->prevpool = pool;
808 | pool->nextpool = next;
809 | UNLOCK();
810 | return (void *)bp;
811 | }
812 |
813 | /* There isn't a pool of the right size class immediately
814 | * available: use a free pool.
815 | */
816 | if (usable_arenas == NULL((void *)0)) {
817 | /* No arena has a free pool: allocate a new arena. */
818 | #ifdef WITH_MEMORY_LIMITS
819 | if (narenas_currently_allocated >= MAX_ARENAS) {
820 | UNLOCK();
821 | goto redirect;
822 | }
823 | #endif
824 | usable_arenas = new_arena();
825 | if (usable_arenas == NULL((void *)0)) {
826 | UNLOCK();
827 | goto redirect;
828 | }
829 | usable_arenas->nextarena =
830 | usable_arenas->prevarena = NULL((void *)0);
831 | }
832 | assert(usable_arenas->address != 0)(__builtin_expect(!(usable_arenas->address != 0), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 832, "usable_arenas->address != 0" ) : (void)0);
833 |
834 | /* Try to get a cached free pool. */
835 | pool = usable_arenas->freepools;
836 | if (pool != NULL((void *)0)) {
837 | /* Unlink from cached pools. */
838 | usable_arenas->freepools = pool->nextpool;
839 |
840 | /* This arena already had the smallest nfreepools
841 | * value, so decreasing nfreepools doesn't change
842 | * that, and we don't need to rearrange the
843 | * usable_arenas list. However, if the arena has
844 | * become wholly allocated, we need to remove its
845 | * arena_object from usable_arenas.
846 | */
847 | --usable_arenas->nfreepools;
848 | if (usable_arenas->nfreepools == 0) {
849 | /* Wholly allocated: remove. */
850 | assert(usable_arenas->freepools == NULL)(__builtin_expect(!(usable_arenas->freepools == ((void *)0 )), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 850, "usable_arenas->freepools == NULL" ) : (void)0);
851 | assert(usable_arenas->nextarena == NULL ||(__builtin_expect(!(usable_arenas->nextarena == ((void *)0 ) || usable_arenas->nextarena->prevarena == usable_arenas ), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 853, "usable_arenas->nextarena == NULL || usable_arenas->nextarena->prevarena == usable_arenas" ) : (void)0)
852 | usable_arenas->nextarena->prevarena ==(__builtin_expect(!(usable_arenas->nextarena == ((void *)0 ) || usable_arenas->nextarena->prevarena == usable_arenas ), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 853, "usable_arenas->nextarena == NULL || usable_arenas->nextarena->prevarena == usable_arenas" ) : (void)0)
853 | usable_arenas)(__builtin_expect(!(usable_arenas->nextarena == ((void *)0 ) || usable_arenas->nextarena->prevarena == usable_arenas ), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 853, "usable_arenas->nextarena == NULL || usable_arenas->nextarena->prevarena == usable_arenas" ) : (void)0);
854 |
855 | usable_arenas = usable_arenas->nextarena;
856 | if (usable_arenas != NULL((void *)0)) {
857 | usable_arenas->prevarena = NULL((void *)0);
858 | assert(usable_arenas->address != 0)(__builtin_expect(!(usable_arenas->address != 0), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 858, "usable_arenas->address != 0" ) : (void)0);
859 | }
860 | }
861 | else {
862 | /* nfreepools > 0: it must be that freepools
863 | * isn't NULL, or that we haven't yet carved
864 | * off all the arena's pools for the first
865 | * time.
866 | */
867 | assert(usable_arenas->freepools != NULL ||(__builtin_expect(!(usable_arenas->freepools != ((void *)0 ) || usable_arenas->pool_address <= (block*)usable_arenas ->address + (256 << 10) - (4 * 1024)), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 870, "usable_arenas->freepools != NULL || usable_arenas->pool_address <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE" ) : (void)0)
868 | usable_arenas->pool_address <=(__builtin_expect(!(usable_arenas->freepools != ((void *)0 ) || usable_arenas->pool_address <= (block*)usable_arenas ->address + (256 << 10) - (4 * 1024)), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 870, "usable_arenas->freepools != NULL || usable_arenas->pool_address <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE" ) : (void)0)
869 | (block*)usable_arenas->address +(__builtin_expect(!(usable_arenas->freepools != ((void *)0 ) || usable_arenas->pool_address <= (block*)usable_arenas ->address + (256 << 10) - (4 * 1024)), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 870, "usable_arenas->freepools != NULL || usable_arenas->pool_address <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE" ) : (void)0)
870 | ARENA_SIZE - POOL_SIZE)(__builtin_expect(!(usable_arenas->freepools != ((void *)0 ) || usable_arenas->pool_address <= (block*)usable_arenas ->address + (256 << 10) - (4 * 1024)), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 870, "usable_arenas->freepools != NULL || usable_arenas->pool_address <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE" ) : (void)0);
871 | }
872 | init_pool:
873 | /* Frontlink to used pools. */
874 | next = usedpools[size + size]; /* == prev */
875 | pool->nextpool = next;
876 | pool->prevpool = next;
877 | next->nextpool = pool;
878 | next->prevpool = pool;
879 | pool->ref.count = 1;
880 | if (pool->szidx == size) {
881 | /* Luckily, this pool last contained blocks
882 | * of the same size class, so its header
883 | * and free list are already initialized.
884 | */
885 | bp = pool->freeblock;
886 | pool->freeblock = *(block **)bp;
887 | UNLOCK();
888 | return (void *)bp;
889 | }
890 | /*
891 | * Initialize the pool header, set up the free list to
892 | * contain just the second block, and return the first
893 | * block.
894 | */
895 | pool->szidx = size;
896 | size = INDEX2SIZE(size)(((unsigned int)(size) + 1) << 3);
897 | bp = (block *)pool + POOL_OVERHEAD(((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1));
898 | pool->nextoffset = POOL_OVERHEAD(((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1)) + (size << 1);
899 | pool->maxnextoffset = POOL_SIZE(4 * 1024) - size;
900 | pool->freeblock = bp + size;
901 | *(block **)(pool->freeblock) = NULL((void *)0);
902 | UNLOCK();
903 | return (void *)bp;
904 | }
905 |
906 | /* Carve off a new pool. */
907 | assert(usable_arenas->nfreepools > 0)(__builtin_expect(!(usable_arenas->nfreepools > 0), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 907, "usable_arenas->nfreepools > 0" ) : (void)0);
908 | assert(usable_arenas->freepools == NULL)(__builtin_expect(!(usable_arenas->freepools == ((void *)0 )), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 908, "usable_arenas->freepools == NULL" ) : (void)0);
909 | pool = (poolp)usable_arenas->pool_address;
910 | assert((block*)pool <= (block*)usable_arenas->address +(__builtin_expect(!((block*)pool <= (block*)usable_arenas-> address + (256 << 10) - (4 * 1024)), 0) ? __assert_rtn( __func__, "Objects/obmalloc.c", 911, "(block*)pool <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE" ) : (void)0)
911 | ARENA_SIZE - POOL_SIZE)(__builtin_expect(!((block*)pool <= (block*)usable_arenas-> address + (256 << 10) - (4 * 1024)), 0) ? __assert_rtn( __func__, "Objects/obmalloc.c", 911, "(block*)pool <= (block*)usable_arenas->address + ARENA_SIZE - POOL_SIZE" ) : (void)0);
912 | pool->arenaindex = usable_arenas - arenas;
913 | assert(&arenas[pool->arenaindex] == usable_arenas)(__builtin_expect(!(&arenas[pool->arenaindex] == usable_arenas ), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 913, "&arenas[pool->arenaindex] == usable_arenas" ) : (void)0);
914 | pool->szidx = DUMMY_SIZE_IDX0xffff;
915 | usable_arenas->pool_address += POOL_SIZE(4 * 1024);
916 | --usable_arenas->nfreepools;
917 |
918 | if (usable_arenas->nfreepools == 0) {
919 | assert(usable_arenas->nextarena == NULL ||(__builtin_expect(!(usable_arenas->nextarena == ((void *)0 ) || usable_arenas->nextarena->prevarena == usable_arenas ), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 921, "usable_arenas->nextarena == NULL || usable_arenas->nextarena->prevarena == usable_arenas" ) : (void)0)
920 | usable_arenas->nextarena->prevarena ==(__builtin_expect(!(usable_arenas->nextarena == ((void *)0 ) || usable_arenas->nextarena->prevarena == usable_arenas ), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 921, "usable_arenas->nextarena == NULL || usable_arenas->nextarena->prevarena == usable_arenas" ) : (void)0)
921 | usable_arenas)(__builtin_expect(!(usable_arenas->nextarena == ((void *)0 ) || usable_arenas->nextarena->prevarena == usable_arenas ), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 921, "usable_arenas->nextarena == NULL || usable_arenas->nextarena->prevarena == usable_arenas" ) : (void)0);
922 | /* Unlink the arena: it is completely allocated. */
923 | usable_arenas = usable_arenas->nextarena;
924 | if (usable_arenas != NULL((void *)0)) {
925 | usable_arenas->prevarena = NULL((void *)0);
926 | assert(usable_arenas->address != 0)(__builtin_expect(!(usable_arenas->address != 0), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 926, "usable_arenas->address != 0" ) : (void)0);
927 | }
928 | }
929 |
930 | goto init_pool;
931 | }
932 |
933 | /* The small block allocator ends here. */
934 |
935 | redirect:
936 | /* Redirect the original request to the underlying (libc) allocator.
937 | * We jump here on bigger requests, on error in the code above (as a
938 | * last chance to serve the request) or when the max memory limit
939 | * has been reached.
940 | */
941 | if (nbytes == 0)
942 | nbytes = 1;
943 | return (void *)malloc(nbytes);
944 | }
945 |
946 | /* free */
947 |
948 | #undef PyObject_Free
949 | void
950 | PyObject_Free(void *p)
951 | {
952 | poolp pool;
953 | block *lastfree;
954 | poolp next, prev;
955 | uintunsigned int size;
956 | #ifndef Py_USING_MEMORY_DEBUGGER
957 | uintunsigned int arenaindex_temp;
958 | #endif
959 |
960 | if (p == NULL((void *)0)) /* free(NULL) has no effect */
1 | Taking false branch
961 | return;
962 |
963 | #ifdef WITH_VALGRIND
964 | if (UNLIKELY(running_on_valgrind > 0))
965 | goto redirect;
966 | #endif
967 |
968 | pool = POOL_ADDR(p)((poolp)((Py_uintptr_t)(p) & ~(Py_uintptr_t)((4 * 1024) - 1)));
969 | if (Py_ADDRESS_IN_RANGE(p, pool)((arenaindex_temp = (pool)->arenaindex) < maxarenas && (Py_uintptr_t)(p) - arenas[arenaindex_temp].address < (Py_uintptr_t )(256 << 10) && arenas[arenaindex_temp].address != 0)) {
2 | Taking true branch
970 | /* We allocated this address. */
971 | LOCK();
972 | /* Link p to the start of the pool's freeblock list. Since
973 | * the pool had at least the p block outstanding, the pool
974 | * wasn't empty (so it's already in a usedpools[] list, or
975 | * was full and is in no list -- it's not in the freeblocks
976 | * list in any case).
977 | */
978 | assert(pool->ref.count > 0)(__builtin_expect(!(pool->ref.count > 0), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 978, "pool->ref.count > 0" ) : (void)0); /* else it was empty */
979 | *(block **)p = lastfree = pool->freeblock;
980 | pool->freeblock = (block *)p;
981 | if (lastfree) {
3 | Taking true branch
982 | struct arena_object* ao;
983 | uintunsigned int nf; /* ao->nfreepools */
984 |
985 | /* freeblock wasn't NULL, so the pool wasn't full,
986 | * and the pool is in a usedpools[] list.
987 | */
988 | if (--pool->ref.count != 0) {
4 | Taking false branch
989 | /* pool isn't empty: leave it in usedpools */
990 | UNLOCK();
991 | return;
992 | }
993 | /* Pool is now empty: unlink from usedpools, and
994 | * link to the front of freepools. This ensures that
995 | * previously freed pools will be allocated later
996 | * (being not referenced, they are perhaps paged out).
997 | */
998 | next = pool->nextpool;
999 | prev = pool->prevpool;
1000 | next->prevpool = prev;
1001 | prev->nextpool = next;
1002 |
1003 | /* Link the pool to freepools. This is a singly-linked
1004 | * list, and pool->prevpool isn't used there.
1005 | */
1006 | ao = &arenas[pool->arenaindex];
1007 | pool->nextpool = ao->freepools;
1008 | ao->freepools = pool;
1009 | nf = ++ao->nfreepools;
1010 |
1011 | /* All the rest is arena management. We just freed
1012 | * a pool, and there are 4 cases for arena mgmt:
1013 | * 1. If all the pools are free, return the arena to
1014 | * the system free().
1015 | * 2. If this is the only free pool in the arena,
1016 | * add the arena back to the `usable_arenas` list.
1017 | * 3. If the "next" arena has a smaller count of free
1018 | * pools, we have to "slide this arena right" to
1019 | * restore that usable_arenas is sorted in order of
1020 | * nfreepools.
1021 | * 4. Else there's nothing more to do.
1022 | */
1023 | if (nf == ao->ntotalpools) {
5 | Taking false branch
1024 | /* Case 1. First unlink ao from usable_arenas.
1025 | */
1026 | assert(ao->prevarena == NULL ||(__builtin_expect(!(ao->prevarena == ((void *)0) || ao-> prevarena->address != 0), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 1027, "ao->prevarena == NULL || ao->prevarena->address != 0" ) : (void)0)
1027 | ao->prevarena->address != 0)(__builtin_expect(!(ao->prevarena == ((void *)0) || ao-> prevarena->address != 0), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 1027, "ao->prevarena == NULL || ao->prevarena->address != 0" ) : (void)0);
1028 | assert(ao ->nextarena == NULL ||(__builtin_expect(!(ao ->nextarena == ((void *)0) || ao-> nextarena->address != 0), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 1029, "ao ->nextarena == NULL || ao->nextarena->address != 0" ) : (void)0)
1029 | ao->nextarena->address != 0)(__builtin_expect(!(ao ->nextarena == ((void *)0) || ao-> nextarena->address != 0), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 1029, "ao ->nextarena == NULL || ao->nextarena->address != 0" ) : (void)0);
1030 |
1031 | /* Fix the pointer in the prevarena, or the
1032 | * usable_arenas pointer.
1033 | */
1034 | if (ao->prevarena == NULL((void *)0)) {
1035 | usable_arenas = ao->nextarena;
1036 | assert(usable_arenas == NULL ||(__builtin_expect(!(usable_arenas == ((void *)0) || usable_arenas ->address != 0), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 1037, "usable_arenas == NULL || usable_arenas->address != 0" ) : (void)0)
1037 | usable_arenas->address != 0)(__builtin_expect(!(usable_arenas == ((void *)0) || usable_arenas ->address != 0), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 1037, "usable_arenas == NULL || usable_arenas->address != 0" ) : (void)0);
1038 | }
1039 | else {
1040 | assert(ao->prevarena->nextarena == ao)(__builtin_expect(!(ao->prevarena->nextarena == ao), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1040, "ao->prevarena->nextarena == ao" ) : (void)0);
1041 | ao->prevarena->nextarena =
1042 | ao->nextarena;
1043 | }
1044 | /* Fix the pointer in the nextarena. */
1045 | if (ao->nextarena != NULL((void *)0)) {
1046 | assert(ao->nextarena->prevarena == ao)(__builtin_expect(!(ao->nextarena->prevarena == ao), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1046, "ao->nextarena->prevarena == ao" ) : (void)0);
1047 | ao->nextarena->prevarena =
1048 | ao->prevarena;
1049 | }
1050 | /* Record that this arena_object slot is
1051 | * available to be reused.
1052 | */
1053 | ao->nextarena = unused_arena_objects;
1054 | unused_arena_objects = ao;
1055 |
1056 | /* Free the entire arena. */
1057 | free((void *)ao->address);
1058 | ao->address = 0; /* mark unassociated */
1059 | --narenas_currently_allocated;
1060 |
1061 | UNLOCK();
1062 | return;
1063 | }
1064 | if (nf == 1) {
6 | Taking false branch
1065 | /* Case 2. Put ao at the head of
1066 | * usable_arenas. Note that because
1067 | * ao->nfreepools was 0 before, ao isn't
1068 | * currently on the usable_arenas list.
1069 | */
1070 | ao->nextarena = usable_arenas;
1071 | ao->prevarena = NULL((void *)0);
1072 | if (usable_arenas)
1073 | usable_arenas->prevarena = ao;
1074 | usable_arenas = ao;
1075 | assert(usable_arenas->address != 0)(__builtin_expect(!(usable_arenas->address != 0), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 1075, "usable_arenas->address != 0" ) : (void)0);
1076 |
1077 | UNLOCK();
1078 | return;
1079 | }
1080 | /* If this arena is now out of order, we need to keep
1081 | * the list sorted. The list is kept sorted so that
1082 | * the "most full" arenas are used first, which allows
1083 | * the nearly empty arenas to be completely freed. In
1084 | * a few un-scientific tests, it seems like this
1085 | * approach allowed a lot more memory to be freed.
1086 | */
1087 | if (ao->nextarena == NULL((void *)0) ||
7 | Taking false branch
1088 | nf <= ao->nextarena->nfreepools) {
1089 | /* Case 4. Nothing to do. */
1090 | UNLOCK();
1091 | return;
1092 | }
1093 | /* Case 3: We have to move the arena towards the end
1094 | * of the list, because it has more free pools than
1095 | * the arena to its right.
1096 | * First unlink ao from usable_arenas.
1097 | */
1098 | if (ao->prevarena != NULL((void *)0)) {
8 | Taking false branch
1099 | /* ao isn't at the head of the list */
1100 | assert(ao->prevarena->nextarena == ao)(__builtin_expect(!(ao->prevarena->nextarena == ao), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1100, "ao->prevarena->nextarena == ao" ) : (void)0);
1101 | ao->prevarena->nextarena = ao->nextarena;
1102 | }
1103 | else {
1104 | /* ao is at the head of the list */
1105 | assert(usable_arenas == ao)(__builtin_expect(!(usable_arenas == ao), 0) ? __assert_rtn(__func__ , "Objects/obmalloc.c", 1105, "usable_arenas == ao") : (void) 0);
1106 | usable_arenas = ao->nextarena;
1107 | }
1108 | ao->nextarena->prevarena = ao->prevarena;
1109 |
1110 | /* Locate the new insertion point by iterating over
1111 | * the list, using our nextarena pointer.
1112 | */
1113 | while (ao->nextarena != NULL((void *)0) &&
9 | Loop condition is false. Execution continues on line 1120
1114 | nf > ao->nextarena->nfreepools) {
1115 | ao->prevarena = ao->nextarena;
1116 | ao->nextarena = ao->nextarena->nextarena;
1117 | }
1118 |
1119 | /* Insert ao at this point. */
1120 | assert(ao->nextarena == NULL ||(__builtin_expect(!(ao->nextarena == ((void *)0) || ao-> prevarena == ao->nextarena->prevarena), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 1121, "ao->nextarena == NULL || ao->prevarena == ao->nextarena->prevarena" ) : (void)0)
1121 | ao->prevarena == ao->nextarena->prevarena)(__builtin_expect(!(ao->nextarena == ((void *)0) || ao-> prevarena == ao->nextarena->prevarena), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 1121, "ao->nextarena == NULL || ao->prevarena == ao->nextarena->prevarena" ) : (void)0);
1122 | assert(ao->prevarena->nextarena == ao->nextarena)(__builtin_expect(!(ao->prevarena->nextarena == ao-> nextarena), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1122, "ao->prevarena->nextarena == ao->nextarena") : (void)0);
10 | Within the expansion of the macro 'assert':
a | Access to field 'nextarena' results in a dereference of a null pointer (loaded from field 'prevarena')
1123 |
1124 | ao->prevarena->nextarena = ao;
1125 | if (ao->nextarena != NULL((void *)0))
1126 | ao->nextarena->prevarena = ao;
1127 |
1128 | /* Verify that the swaps worked. */
1129 | assert(ao->nextarena == NULL ||(__builtin_expect(!(ao->nextarena == ((void *)0) || nf <= ao->nextarena->nfreepools), 0) ? __assert_rtn(__func__ , "Objects/obmalloc.c", 1130, "ao->nextarena == NULL || nf <= ao->nextarena->nfreepools" ) : (void)0)
1130 | nf <= ao->nextarena->nfreepools)(__builtin_expect(!(ao->nextarena == ((void *)0) || nf <= ao->nextarena->nfreepools), 0) ? __assert_rtn(__func__ , "Objects/obmalloc.c", 1130, "ao->nextarena == NULL || nf <= ao->nextarena->nfreepools" ) : (void)0);
1131 | assert(ao->prevarena == NULL ||(__builtin_expect(!(ao->prevarena == ((void *)0) || nf > ao->prevarena->nfreepools), 0) ? __assert_rtn(__func__ , "Objects/obmalloc.c", 1132, "ao->prevarena == NULL || nf > ao->prevarena->nfreepools" ) : (void)0)
1132 | nf > ao->prevarena->nfreepools)(__builtin_expect(!(ao->prevarena == ((void *)0) || nf > ao->prevarena->nfreepools), 0) ? __assert_rtn(__func__ , "Objects/obmalloc.c", 1132, "ao->prevarena == NULL || nf > ao->prevarena->nfreepools" ) : (void)0);
1133 | assert(ao->nextarena == NULL ||(__builtin_expect(!(ao->nextarena == ((void *)0) || ao-> nextarena->prevarena == ao), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 1134, "ao->nextarena == NULL || ao->nextarena->prevarena == ao" ) : (void)0)
1134 | ao->nextarena->prevarena == ao)(__builtin_expect(!(ao->nextarena == ((void *)0) || ao-> nextarena->prevarena == ao), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c" , 1134, "ao->nextarena == NULL || ao->nextarena->prevarena == ao" ) : (void)0);
1135 | assert((usable_arenas == ao &&(__builtin_expect(!((usable_arenas == ao && ao->prevarena == ((void *)0)) || ao->prevarena->nextarena == ao), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1137, "(usable_arenas == ao && ao->prevarena == NULL) || ao->prevarena->nextarena == ao" ) : (void)0)
1136 | ao->prevarena == NULL) ||(__builtin_expect(!((usable_arenas == ao && ao->prevarena == ((void *)0)) || ao->prevarena->nextarena == ao), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1137, "(usable_arenas == ao && ao->prevarena == NULL) || ao->prevarena->nextarena == ao" ) : (void)0)
1137 | ao->prevarena->nextarena == ao)(__builtin_expect(!((usable_arenas == ao && ao->prevarena == ((void *)0)) || ao->prevarena->nextarena == ao), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1137, "(usable_arenas == ao && ao->prevarena == NULL) || ao->prevarena->nextarena == ao" ) : (void)0);
1138 |
1139 | UNLOCK();
1140 | return;
1141 | }
1142 | /* Pool was full, so doesn't currently live in any list:
1143 | * link it to the front of the appropriate usedpools[] list.
1144 | * This mimics LRU pool usage for new allocations and
1145 | * targets optimal filling when several pools contain
1146 | * blocks of the same size class.
1147 | */
1148 | --pool->ref.count;
1149 | assert(pool->ref.count > 0)(__builtin_expect(!(pool->ref.count > 0), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 1149, "pool->ref.count > 0" ) : (void)0); /* else the pool is empty */
1150 | size = pool->szidx;
1151 | next = usedpools[size + size];
1152 | prev = next->prevpool;
1153 | /* insert pool before next: prev <-> pool <-> next */
1154 | pool->nextpool = next;
1155 | pool->prevpool = prev;
1156 | next->prevpool = pool;
1157 | prev->nextpool = pool;
1158 | UNLOCK();
1159 | return;
1160 | }
1161 |
1162 | #ifdef WITH_VALGRIND
1163 | redirect:
1164 | #endif
1165 | /* We didn't allocate this address. */
1166 | free(p);
1167 | }
1168 |
1169 | /* realloc. If p is NULL, this acts like malloc(nbytes). Else if nbytes==0,
1170 | * then as the Python docs promise, we do not treat this like free(p), and
1171 | * return a non-NULL result.
1172 | */
1173 |
1174 | #undef PyObject_Realloc
1175 | void *
1176 | PyObject_Realloc(void *p, size_t nbytes)
1177 | {
1178 | void *bp;
1179 | poolp pool;
1180 | size_t size;
1181 | #ifndef Py_USING_MEMORY_DEBUGGER
1182 | uintunsigned int arenaindex_temp;
1183 | #endif
1184 |
1185 | if (p == NULL((void *)0))
1186 | return PyObject_Malloc(nbytes);
1187 |
1188 | /*
1189 | * Limit ourselves to PY_SSIZE_T_MAX bytes to prevent security holes.
1190 | * Most python internals blindly use a signed Py_ssize_t to track
1191 | * things without checking for overflows or negatives.
1192 | * As size_t is unsigned, checking for nbytes < 0 is not required.
1193 | */
1194 | if (nbytes > PY_SSIZE_T_MAX((Py_ssize_t)(((size_t)-1)>>1)))
1195 | return NULL((void *)0);
1196 |
1197 | #ifdef WITH_VALGRIND
1198 | /* Treat running_on_valgrind == -1 the same as 0 */
1199 | if (UNLIKELY(running_on_valgrind > 0))
1200 | goto redirect;
1201 | #endif
1202 |
1203 | pool = POOL_ADDR(p)((poolp)((Py_uintptr_t)(p) & ~(Py_uintptr_t)((4 * 1024) - 1)));
1204 | if (Py_ADDRESS_IN_RANGE(p, pool)((arenaindex_temp = (pool)->arenaindex) < maxarenas && (Py_uintptr_t)(p) - arenas[arenaindex_temp].address < (Py_uintptr_t )(256 << 10) && arenas[arenaindex_temp].address != 0)) {
1205 | /* We're in charge of this block */
1206 | size = INDEX2SIZE(pool->szidx)(((unsigned int)(pool->szidx) + 1) << 3);
1207 | if (nbytes <= size) {
1208 | /* The block is staying the same or shrinking. If
1209 | * it's shrinking, there's a tradeoff: it costs
1210 | * cycles to copy the block to a smaller size class,
1211 | * but it wastes memory not to copy it. The
1212 | * compromise here is to copy on shrink only if at
1213 | * least 25% of size can be shaved off.
1214 | */
1215 | if (4 * nbytes > 3 * size) {
1216 | /* It's the same,
1217 | * or shrinking and new/old > 3/4.
1218 | */
1219 | return p;
1220 | }
1221 | size = nbytes;
1222 | }
1223 | bp = PyObject_Malloc(nbytes);
1224 | if (bp != NULL((void *)0)) {
1225 | memcpy(bp, p, size)((__builtin_object_size (bp, 0) != (size_t) -1) ? __builtin___memcpy_chk (bp, p, size, __builtin_object_size (bp, 0)) : __inline_memcpy_chk (bp, p, size));
1226 | PyObject_Free(p);
1227 | }
1228 | return bp;
1229 | }
1230 | #ifdef WITH_VALGRIND
1231 | redirect:
1232 | #endif
1233 | /* We're not managing this block. If nbytes <=
1234 | * SMALL_REQUEST_THRESHOLD, it's tempting to try to take over this
1235 | * block. However, if we do, we need to copy the valid data from
1236 | * the C-managed block to one of our blocks, and there's no portable
1237 | * way to know how much of the memory space starting at p is valid.
1238 | * As bug 1185883 pointed out the hard way, it's possible that the
1239 | * C-managed block is "at the end" of allocated VM space, so that
1240 | * a memory fault can occur if we try to copy nbytes bytes starting
1241 | * at p. Instead we punt: let C continue to manage this block.
1242 | */
1243 | if (nbytes)
1244 | return realloc(p, nbytes);
1245 | /* C doesn't define the result of realloc(p, 0) (it may or may not
1246 | * return NULL then), but Python's docs promise that nbytes==0 never
1247 | * returns NULL. We don't pass 0 to realloc(), to avoid that endcase
1248 | * to begin with. Even then, we can't be sure that realloc() won't
1249 | * return NULL.
1250 | */
1251 | bp = realloc(p, 1);
1252 | return bp ? bp : p;
1253 | }
1254 |
1255 | #else /* ! WITH_PYMALLOC */
1256 |
1257 | /*==========================================================================*/
1258 | /* pymalloc not enabled: Redirect the entry points to malloc. These will
1259 | * only be used by extensions that are compiled with pymalloc enabled. */
1260 |
1261 | void *
1262 | PyObject_Malloc(size_t n)
1263 | {
1264 | return PyMem_MALLOC_PyMem_DebugMalloc(n);
1265 | }
1266 |
1267 | void *
1268 | PyObject_Realloc(void *p, size_t n)
1269 | {
1270 | return PyMem_REALLOC_PyMem_DebugRealloc(p, n);
1271 | }
1272 |
1273 | void
1274 | PyObject_Free(void *p)
1275 | {
1276 | PyMem_FREE_PyMem_DebugFree(p);
1277 | }
1278 | #endif /* WITH_PYMALLOC */
1279 |
1280 | #ifdef PYMALLOC_DEBUG
1281 | /*==========================================================================*/
1282 | /* A x-platform debugging allocator. This doesn't manage memory directly,
1283 | * it wraps a real allocator, adding extra debugging info to the memory blocks.
1284 | */
1285 |
1286 | /* Special bytes broadcast into debug memory blocks at appropriate times.
1287 | * Strings of these are unlikely to be valid addresses, floats, ints or
1288 | * 7-bit ASCII.
1289 | */
1290 | #undef CLEANBYTE0xCB
1291 | #undef DEADBYTE0xDB
1292 | #undef FORBIDDENBYTE0xFB
1293 | #define CLEANBYTE0xCB 0xCB /* clean (newly allocated) memory */
1294 | #define DEADBYTE0xDB 0xDB /* dead (newly freed) memory */
1295 | #define FORBIDDENBYTE0xFB 0xFB /* untouchable bytes at each end of a block */
1296 |
1297 | /* We tag each block with an API ID in order to tag API violations */
1298 | #define _PYMALLOC_MEM_ID'm' 'm' /* the PyMem_Malloc() API */
1299 | #define _PYMALLOC_OBJ_ID'o' 'o' /* The PyObject_Malloc() API */
1300 |
1301 | static size_t serialno = 0; /* incremented on each debug {m,re}alloc */
1302 |
1303 | /* serialno is always incremented via calling this routine. The point is
1304 | * to supply a single place to set a breakpoint.
1305 | */
1306 | static void
1307 | bumpserialno(void)
1308 | {
1309 | ++serialno;
1310 | }
1311 |
1312 | #define SST8 SIZEOF_SIZE_T8
1313 |
1314 | /* Read sizeof(size_t) bytes at p as a big-endian size_t. */
1315 | static size_t
1316 | read_size_t(const void *p)
1317 | {
1318 | const ucharunsigned char *q = (const ucharunsigned char *)p;
1319 | size_t result = *q++;
1320 | int i;
1321 |
1322 | for (i = SST8; --i > 0; ++q)
1323 | result = (result << 8) | *q;
1324 | return result;
1325 | }
1326 |
1327 | /* Write n as a big-endian size_t, MSB at address p, LSB at
1328 | * p + sizeof(size_t) - 1.
1329 | */
1330 | static void
1331 | write_size_t(void *p, size_t n)
1332 | {
1333 | ucharunsigned char *q = (ucharunsigned char *)p + SST8 - 1;
1334 | int i;
1335 |
1336 | for (i = SST8; --i >= 0; --q) {
1337 | *q = (ucharunsigned char)(n & 0xff);
1338 | n >>= 8;
1339 | }
1340 | }
1341 |
1342 | #ifdef Py_DEBUG1
1343 | /* Is target in the list? The list is traversed via the nextpool pointers.
1344 | * The list may be NULL-terminated, or circular. Return 1 if target is in
1345 | * list, else 0.
1346 | */
1347 | static int
1348 | pool_is_in_list(const poolp target, poolp list)
1349 | {
1350 | poolp origlist = list;
1351 | assert(target != NULL)(__builtin_expect(!(target != ((void *)0)), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 1351, "target != NULL") : (void )0);
1352 | if (list == NULL((void *)0))
1353 | return 0;
1354 | do {
1355 | if (target == list)
1356 | return 1;
1357 | list = list->nextpool;
1358 | } while (list != NULL((void *)0) && list != origlist);
1359 | return 0;
1360 | }
1361 |
1362 | #else
1363 | #define pool_is_in_list(X, Y) 1
1364 |
1365 | #endif /* Py_DEBUG */
1366 |
1367 | /* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and
1368 | fills them with useful stuff, here calling the underlying malloc's result p:
1369 |
1370 | p[0: S]
1371 | Number of bytes originally asked for. This is a size_t, big-endian (easier
1372 | to read in a memory dump).
1373 | p[S: 2*S]
1374 | Copies of FORBIDDENBYTE. Used to catch under- writes and reads.
1375 | p[2*S: 2*S+n]
1376 | The requested memory, filled with copies of CLEANBYTE.
1377 | Used to catch reference to uninitialized memory.
1378 | &p[2*S] is returned. Note that this is 8-byte aligned if pymalloc
1379 | handled the request itself.
1380 | p[2*S+n: 2*S+n+S]
1381 | Copies of FORBIDDENBYTE. Used to catch over- writes and reads.
1382 | p[2*S+n+S: 2*S+n+2*S]
1383 | A serial number, incremented by 1 on each call to _PyObject_DebugMalloc
1384 | and _PyObject_DebugRealloc.
1385 | This is a big-endian size_t.
1386 | If "bad memory" is detected later, the serial number gives an
1387 | excellent way to set a breakpoint on the next run, to capture the
1388 | instant at which this block was passed out.
1389 | */
1390 |
1391 | /* debug replacements for the PyMem_* memory API */
1392 | void *
1393 | _PyMem_DebugMalloc(size_t nbytes)
1394 | {
1395 | return _PyObject_DebugMallocApi(_PYMALLOC_MEM_ID'm', nbytes);
1396 | }
1397 | void *
1398 | _PyMem_DebugRealloc(void *p, size_t nbytes)
1399 | {
1400 | return _PyObject_DebugReallocApi(_PYMALLOC_MEM_ID'm', p, nbytes);
1401 | }
1402 | void
1403 | _PyMem_DebugFree(void *p)
1404 | {
1405 | _PyObject_DebugFreeApi(_PYMALLOC_MEM_ID'm', p);
1406 | }
1407 |
1408 | /* debug replacements for the PyObject_* memory API */
1409 | void *
1410 | _PyObject_DebugMalloc(size_t nbytes)
1411 | {
1412 | return _PyObject_DebugMallocApi(_PYMALLOC_OBJ_ID'o', nbytes);
1413 | }
1414 | void *
1415 | _PyObject_DebugRealloc(void *p, size_t nbytes)
1416 | {
1417 | return _PyObject_DebugReallocApi(_PYMALLOC_OBJ_ID'o', p, nbytes);
1418 | }
1419 | void
1420 | _PyObject_DebugFree(void *p)
1421 | {
1422 | _PyObject_DebugFreeApi(_PYMALLOC_OBJ_ID'o', p);
1423 | }
1424 | void
1425 | _PyObject_DebugCheckAddress(const void *p)
1426 | {
1427 | _PyObject_DebugCheckAddressApi(_PYMALLOC_OBJ_ID'o', p);
1428 | }
1429 |
1430 |
1431 | /* generic debug memory api, with an "id" to identify the API in use */
1432 | void *
1433 | _PyObject_DebugMallocApi(char id, size_t nbytes)
1434 | {
1435 | ucharunsigned char *p; /* base address of malloc'ed block */
1436 | ucharunsigned char *tail; /* p + 2*SST + nbytes == pointer to tail pad bytes */
1437 | size_t total; /* nbytes + 4*SST */
1438 |
1439 | bumpserialno();
1440 | total = nbytes + 4*SST8;
1441 | if (total < nbytes)
1442 | /* overflow: can't represent total as a size_t */
1443 | return NULL((void *)0);
1444 |
1445 | p = (ucharunsigned char *)PyObject_Malloc(total);
1446 | if (p == NULL((void *)0))
1447 | return NULL((void *)0);
1448 |
1449 | /* at p, write size (SST bytes), id (1 byte), pad (SST-1 bytes) */
1450 | write_size_t(p, nbytes);
1451 | p[SST8] = (ucharunsigned char)id;
1452 | memset(p + SST + 1 , FORBIDDENBYTE, SST-1)((__builtin_object_size (p + 8 + 1, 0) != (size_t) -1) ? __builtin___memset_chk (p + 8 + 1, 0xFB, 8 -1, __builtin_object_size (p + 8 + 1, 0) ) : __inline_memset_chk (p + 8 + 1, 0xFB, 8 -1));
1453 |
1454 | if (nbytes > 0)
1455 | memset(p + 2*SST, CLEANBYTE, nbytes)((__builtin_object_size (p + 2*8, 0) != (size_t) -1) ? __builtin___memset_chk (p + 2*8, 0xCB, nbytes, __builtin_object_size (p + 2*8, 0)) : __inline_memset_chk (p + 2*8, 0xCB, nbytes));
1456 |
1457 | /* at tail, write pad (SST bytes) and serialno (SST bytes) */
1458 | tail = p + 2*SST8 + nbytes;
1459 | memset(tail, FORBIDDENBYTE, SST)((__builtin_object_size (tail, 0) != (size_t) -1) ? __builtin___memset_chk (tail, 0xFB, 8, __builtin_object_size (tail, 0)) : __inline_memset_chk (tail, 0xFB, 8));
1460 | write_size_t(tail + SST8, serialno);
1461 |
1462 | return p + 2*SST8;
1463 | }
1464 |
1465 | /* The debug free first checks the 2*SST bytes on each end for sanity (in
1466 | particular, that the FORBIDDENBYTEs with the api ID are still intact).
1467 | Then fills the original bytes with DEADBYTE.
1468 | Then calls the underlying free.
1469 | */
1470 | void
1471 | _PyObject_DebugFreeApi(char api, void *p)
1472 | {
1473 | ucharunsigned char *q = (ucharunsigned char *)p - 2*SST8; /* address returned from malloc */
1474 | size_t nbytes;
1475 |
1476 | if (p == NULL((void *)0))
1477 | return;
1478 | _PyObject_DebugCheckAddressApi(api, p);
1479 | nbytes = read_size_t(q);
1480 | nbytes += 4*SST8;
1481 | if (nbytes > 0)
1482 | memset(q, DEADBYTE, nbytes)((__builtin_object_size (q, 0) != (size_t) -1) ? __builtin___memset_chk (q, 0xDB, nbytes, __builtin_object_size (q, 0)) : __inline_memset_chk (q, 0xDB, nbytes));
1483 | PyObject_Free(q);
1484 | }
1485 |
1486 | void *
1487 | _PyObject_DebugReallocApi(char api, void *p, size_t nbytes)
1488 | {
1489 | ucharunsigned char *q = (ucharunsigned char *)p;
1490 | ucharunsigned char *tail;
1491 | size_t total; /* nbytes + 4*SST */
1492 | size_t original_nbytes;
1493 | int i;
1494 |
1495 | if (p == NULL((void *)0))
1496 | return _PyObject_DebugMallocApi(api, nbytes);
1497 |
1498 | _PyObject_DebugCheckAddressApi(api, p);
1499 | bumpserialno();
1500 | original_nbytes = read_size_t(q - 2*SST8);
1501 | total = nbytes + 4*SST8;
1502 | if (total < nbytes)
1503 | /* overflow: can't represent total as a size_t */
1504 | return NULL((void *)0);
1505 |
1506 | if (nbytes < original_nbytes) {
1507 | /* shrinking: mark old extra memory dead */
1508 | memset(q + nbytes, DEADBYTE, original_nbytes - nbytes + 2*SST)((__builtin_object_size (q + nbytes, 0) != (size_t) -1) ? __builtin___memset_chk (q + nbytes, 0xDB, original_nbytes - nbytes + 2*8, __builtin_object_size (q + nbytes, 0)) : __inline_memset_chk (q + nbytes, 0xDB, original_nbytes - nbytes + 2*8));
1509 | }
1510 |
1511 | /* Resize and add decorations. We may get a new pointer here, in which
1512 | * case we didn't get the chance to mark the old memory with DEADBYTE,
1513 | * but we live with that.
1514 | */
1515 | q = (ucharunsigned char *)PyObject_Realloc(q - 2*SST8, total);
1516 | if (q == NULL((void *)0))
1517 | return NULL((void *)0);
1518 |
1519 | write_size_t(q, nbytes);
1520 | assert(q[SST] == (uchar)api)(__builtin_expect(!(q[8] == (unsigned char)api), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 1520, "q[SST] == (uchar)api" ) : (void)0);
1521 | for (i = 1; i < SST8; ++i)
1522 | assert(q[SST + i] == FORBIDDENBYTE)(__builtin_expect(!(q[8 + i] == 0xFB), 0) ? __assert_rtn(__func__ , "Objects/obmalloc.c", 1522, "q[SST + i] == FORBIDDENBYTE") : (void)0);
1523 | q += 2*SST8;
1524 | tail = q + nbytes;
1525 | memset(tail, FORBIDDENBYTE, SST)((__builtin_object_size (tail, 0) != (size_t) -1) ? __builtin___memset_chk (tail, 0xFB, 8, __builtin_object_size (tail, 0)) : __inline_memset_chk (tail, 0xFB, 8));
1526 | write_size_t(tail + SST8, serialno);
1527 |
1528 | if (nbytes > original_nbytes) {
1529 | /* growing: mark new extra memory clean */
1530 | memset(q + original_nbytes, CLEANBYTE,((__builtin_object_size (q + original_nbytes, 0) != (size_t) - 1) ? __builtin___memset_chk (q + original_nbytes, 0xCB, nbytes - original_nbytes, __builtin_object_size (q + original_nbytes , 0)) : __inline_memset_chk (q + original_nbytes, 0xCB, nbytes - original_nbytes))
1531 | nbytes - original_nbytes)((__builtin_object_size (q + original_nbytes, 0) != (size_t) - 1) ? __builtin___memset_chk (q + original_nbytes, 0xCB, nbytes - original_nbytes, __builtin_object_size (q + original_nbytes , 0)) : __inline_memset_chk (q + original_nbytes, 0xCB, nbytes - original_nbytes));
1532 | }
1533 |
1534 | return q;
1535 | }
1536 |
1537 | /* Check the forbidden bytes on both ends of the memory allocated for p.
1538 | * If anything is wrong, print info to stderr via _PyObject_DebugDumpAddress,
1539 | * and call Py_FatalError to kill the program.
1540 | * The API id, is also checked.
1541 | */
1542 | void
1543 | _PyObject_DebugCheckAddressApi(char api, const void *p)
1544 | {
1545 | const ucharunsigned char *q = (const ucharunsigned char *)p;
1546 | char msgbuf[64];
1547 | char *msg;
1548 | size_t nbytes;
1549 | const ucharunsigned char *tail;
1550 | int i;
1551 | char id;
1552 |
1553 | if (p == NULL((void *)0)) {
1554 | msg = "didn't expect a NULL pointer";
1555 | goto error;
1556 | }
1557 |
1558 | /* Check the API id */
1559 | id = (char)q[-SST8];
1560 | if (id != api) {
1561 | msg = msgbuf;
1562 | snprintf(msg, sizeof(msgbuf), "bad ID: Allocated using API '%c', verified using API '%c'", id, api)__builtin___snprintf_chk (msg, sizeof(msgbuf), 0, __builtin_object_size (msg, 2 > 1), "bad ID: Allocated using API '%c', verified using API '%c'" , id, api);
1563 | msgbuf[sizeof(msgbuf)-1] = 0;
1564 | goto error;
1565 | }
1566 |
1567 | /* Check the stuff at the start of p first: if there's underwrite
1568 | * corruption, the number-of-bytes field may be nuts, and checking
1569 | * the tail could lead to a segfault then.
1570 | */
1571 | for (i = SST8-1; i >= 1; --i) {
1572 | if (*(q-i) != FORBIDDENBYTE0xFB) {
1573 | msg = "bad leading pad byte";
1574 | goto error;
1575 | }
1576 | }
1577 |
1578 | nbytes = read_size_t(q - 2*SST8);
1579 | tail = q + nbytes;
1580 | for (i = 0; i < SST8; ++i) {
1581 | if (tail[i] != FORBIDDENBYTE0xFB) {
1582 | msg = "bad trailing pad byte";
1583 | goto error;
1584 | }
1585 | }
1586 |
1587 | return;
1588 |
1589 | error:
1590 | _PyObject_DebugDumpAddress(p);
1591 | Py_FatalError(msg);
1592 | }
1593 |
1594 | /* Display info to stderr about the memory block at p. */
1595 | void
1596 | _PyObject_DebugDumpAddress(const void *p)
1597 | {
1598 | const ucharunsigned char *q = (const ucharunsigned char *)p;
1599 | const ucharunsigned char *tail;
1600 | size_t nbytes, serial;
1601 | int i;
1602 | int ok;
1603 | char id;
1604 |
1605 | fprintf(stderr__stderrp, "Debug memory block at address p=%p:", p);
1606 | if (p == NULL((void *)0)) {
1607 | fprintf(stderr__stderrp, "\n");
1608 | return;
1609 | }
1610 | id = (char)q[-SST8];
1611 | fprintf(stderr__stderrp, " API '%c'\n", id);
1612 |
1613 | nbytes = read_size_t(q - 2*SST8);
1614 | fprintf(stderr__stderrp, " %" PY_FORMAT_SIZE_T"l" "u bytes originally "
1615 | "requested\n", nbytes);
1616 |
1617 | /* In case this is nuts, check the leading pad bytes first. */
1618 | fprintf(stderr__stderrp, " The %d pad bytes at p-%d are ", SST8-1, SST8-1);
1619 | ok = 1;
1620 | for (i = 1; i <= SST8-1; ++i) {
1621 | if (*(q-i) != FORBIDDENBYTE0xFB) {
1622 | ok = 0;
1623 | break;
1624 | }
1625 | }
1626 | if (ok)
1627 | fputs("FORBIDDENBYTE, as expected.\n", stderr__stderrp);
1628 | else {
1629 | fprintf(stderr__stderrp, "not all FORBIDDENBYTE (0x%02x):\n",
1630 | FORBIDDENBYTE0xFB);
1631 | for (i = SST8-1; i >= 1; --i) {
1632 | const ucharunsigned char byte = *(q-i);
1633 | fprintf(stderr__stderrp, " at p-%d: 0x%02x", i, byte);
1634 | if (byte != FORBIDDENBYTE0xFB)
1635 | fputs(" *** OUCH", stderr__stderrp);
1636 | fputc('\n', stderr__stderrp);
1637 | }
1638 |
1639 | fputs(" Because memory is corrupted at the start, the "
1640 | "count of bytes requested\n"
1641 | " may be bogus, and checking the trailing pad "
1642 | "bytes may segfault.\n", stderr__stderrp);
1643 | }
1644 |
1645 | tail = q + nbytes;
1646 | fprintf(stderr__stderrp, " The %d pad bytes at tail=%p are ", SST8, tail);
1647 | ok = 1;
1648 | for (i = 0; i < SST8; ++i) {
1649 | if (tail[i] != FORBIDDENBYTE0xFB) {
1650 | ok = 0;
1651 | break;
1652 | }
1653 | }
1654 | if (ok)
1655 | fputs("FORBIDDENBYTE, as expected.\n", stderr__stderrp);
1656 | else {
1657 | fprintf(stderr__stderrp, "not all FORBIDDENBYTE (0x%02x):\n",
1658 | FORBIDDENBYTE0xFB);
1659 | for (i = 0; i < SST8; ++i) {
1660 | const ucharunsigned char byte = tail[i];
1661 | fprintf(stderr__stderrp, " at tail+%d: 0x%02x",
1662 | i, byte);
1663 | if (byte != FORBIDDENBYTE0xFB)
1664 | fputs(" *** OUCH", stderr__stderrp);
1665 | fputc('\n', stderr__stderrp);
1666 | }
1667 | }
1668 |
1669 | serial = read_size_t(tail + SST8);
1670 | fprintf(stderr__stderrp, " The block was made by call #%" PY_FORMAT_SIZE_T"l"
1671 | "u to debug malloc/realloc.\n", serial);
1672 |
1673 | if (nbytes > 0) {
1674 | i = 0;
1675 | fputs(" Data at p:", stderr__stderrp);
1676 | /* print up to 8 bytes at the start */
1677 | while (q < tail && i < 8) {
1678 | fprintf(stderr__stderrp, " %02x", *q);
1679 | ++i;
1680 | ++q;
1681 | }
1682 | /* and up to 8 at the end */
1683 | if (q < tail) {
1684 | if (tail - q > 8) {
1685 | fputs(" ...", stderr__stderrp);
1686 | q = tail - 8;
1687 | }
1688 | while (q < tail) {
1689 | fprintf(stderr__stderrp, " %02x", *q);
1690 | ++q;
1691 | }
1692 | }
1693 | fputc('\n', stderr__stderrp);
1694 | }
1695 | }
1696 |
1697 | static size_t
1698 | printone(const char* msg, size_t value)
1699 | {
1700 | int i, k;
1701 | char buf[100];
1702 | size_t origvalue = value;
1703 |
1704 | fputs(msg, stderr__stderrp);
1705 | for (i = (int)strlen(msg); i < 35; ++i)
1706 | fputc(' ', stderr__stderrp);
1707 | fputc('=', stderr__stderrp);
1708 |
1709 | /* Write the value with commas. */
1710 | i = 22;
1711 | buf[i--] = '\0';
1712 | buf[i--] = '\n';
1713 | k = 3;
1714 | do {
1715 | size_t nextvalue = value / 10;
1716 | uintunsigned int digit = (uintunsigned int)(value - nextvalue * 10);
1717 | value = nextvalue;
1718 | buf[i--] = (char)(digit + '0');
1719 | --k;
1720 | if (k == 0 && value && i >= 0) {
1721 | k = 3;
1722 | buf[i--] = ',';
1723 | }
1724 | } while (value && i >= 0);
1725 |
1726 | while (i >= 0)
1727 | buf[i--] = ' ';
1728 | fputs(buf, stderr__stderrp);
1729 |
1730 | return origvalue;
1731 | }
1732 |
1733 | /* Print summary info to stderr about the state of pymalloc's structures.
1734 | * In Py_DEBUG mode, also perform some expensive internal consistency
1735 | * checks.
1736 | */
1737 | void
1738 | _PyObject_DebugMallocStats(void)
1739 | {
1740 | uintunsigned int i;
1741 | const uintunsigned int numclasses = SMALL_REQUEST_THRESHOLD256 >> ALIGNMENT_SHIFT3;
1742 | /* # of pools, allocated blocks, and free blocks per class index */
1743 | size_t numpools[SMALL_REQUEST_THRESHOLD256 >> ALIGNMENT_SHIFT3];
1744 | size_t numblocks[SMALL_REQUEST_THRESHOLD256 >> ALIGNMENT_SHIFT3];
1745 | size_t numfreeblocks[SMALL_REQUEST_THRESHOLD256 >> ALIGNMENT_SHIFT3];
1746 | /* total # of allocated bytes in used and full pools */
1747 | size_t allocated_bytes = 0;
1748 | /* total # of available bytes in used pools */
1749 | size_t available_bytes = 0;
1750 | /* # of free pools + pools not yet carved out of current arena */
1751 | uintunsigned int numfreepools = 0;
1752 | /* # of bytes for arena alignment padding */
1753 | size_t arena_alignment = 0;
1754 | /* # of bytes in used and full pools used for pool_headers */
1755 | size_t pool_header_bytes = 0;
1756 | /* # of bytes in used and full pools wasted due to quantization,
1757 | * i.e. the necessarily leftover space at the ends of used and
1758 | * full pools.
1759 | */
1760 | size_t quantization = 0;
1761 | /* # of arenas actually allocated. */
1762 | size_t narenas = 0;
1763 | /* running total -- should equal narenas * ARENA_SIZE */
1764 | size_t total;
1765 | char buf[128];
1766 |
1767 | fprintf(stderr__stderrp, "Small block threshold = %d, in %u size classes.\n",
1768 | SMALL_REQUEST_THRESHOLD256, numclasses);
1769 |
1770 | for (i = 0; i < numclasses; ++i)
1771 | numpools[i] = numblocks[i] = numfreeblocks[i] = 0;
1772 |
1773 | /* Because full pools aren't linked to from anything, it's easiest
1774 | * to march over all the arenas. If we're lucky, most of the memory
1775 | * will be living in full pools -- would be a shame to miss them.
1776 | */
1777 | for (i = 0; i < maxarenas; ++i) {
1778 | uintunsigned int j;
1779 | uptrPy_uintptr_t base = arenas[i].address;
1780 |
1781 | /* Skip arenas which are not allocated. */
1782 | if (arenas[i].address == (uptrPy_uintptr_t)NULL((void *)0))
1783 | continue;
1784 | narenas += 1;
1785 |
1786 | numfreepools += arenas[i].nfreepools;
1787 |
1788 | /* round up to pool alignment */
1789 | if (base & (uptrPy_uintptr_t)POOL_SIZE_MASK((4 * 1024) - 1)) {
1790 | arena_alignment += POOL_SIZE(4 * 1024);
1791 | base &= ~(uptrPy_uintptr_t)POOL_SIZE_MASK((4 * 1024) - 1);
1792 | base += POOL_SIZE(4 * 1024);
1793 | }
1794 |
1795 | /* visit every pool in the arena */
1796 | assert(base <= (uptr) arenas[i].pool_address)(__builtin_expect(!(base <= (Py_uintptr_t) arenas[i].pool_address ), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1796, "base <= (uptr) arenas[i].pool_address" ) : (void)0);
1797 | for (j = 0;
1798 | base < (uptrPy_uintptr_t) arenas[i].pool_address;
1799 | ++j, base += POOL_SIZE(4 * 1024)) {
1800 | poolp p = (poolp)base;
1801 | const uintunsigned int sz = p->szidx;
1802 | uintunsigned int freeblocks;
1803 |
1804 | if (p->ref.count == 0) {
1805 | /* currently unused */
1806 | assert(pool_is_in_list(p, arenas[i].freepools))(__builtin_expect(!(pool_is_in_list(p, arenas[i].freepools)), 0) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1806, "pool_is_in_list(p, arenas[i].freepools)" ) : (void)0);
1807 | continue;
1808 | }
1809 | ++numpools[sz];
1810 | numblocks[sz] += p->ref.count;
1811 | freeblocks = NUMBLOCKS(sz)((unsigned int)((4 * 1024) - (((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1))) / (((unsigned int)(sz) + 1) << 3)) - p->ref.count;
1812 | numfreeblocks[sz] += freeblocks;
1813 | #ifdef Py_DEBUG1
1814 | if (freeblocks > 0)
1815 | assert(pool_is_in_list(p, usedpools[sz + sz]))(__builtin_expect(!(pool_is_in_list(p, usedpools[sz + sz])), 0 ) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1815, "pool_is_in_list(p, usedpools[sz + sz])" ) : (void)0);
1816 | #endif
1817 | }
1818 | }
1819 | assert(narenas == narenas_currently_allocated)(__builtin_expect(!(narenas == narenas_currently_allocated), 0 ) ? __assert_rtn(__func__, "Objects/obmalloc.c", 1819, "narenas == narenas_currently_allocated" ) : (void)0);
1820 |
1821 | fputc('\n', stderr__stderrp);
1822 | fputs("class size num pools blocks in use avail blocks\n"
1823 | "----- ---- --------- ------------- ------------\n",
1824 | stderr__stderrp);
1825 |
1826 | for (i = 0; i < numclasses; ++i) {
1827 | size_t p = numpools[i];
1828 | size_t b = numblocks[i];
1829 | size_t f = numfreeblocks[i];
1830 | uintunsigned int size = INDEX2SIZE(i)(((unsigned int)(i) + 1) << 3);
1831 | if (p == 0) {
1832 | assert(b == 0 && f == 0)(__builtin_expect(!(b == 0 && f == 0), 0) ? __assert_rtn (__func__, "Objects/obmalloc.c", 1832, "b == 0 && f == 0" ) : (void)0);
1833 | continue;
1834 | }
1835 | fprintf(stderr__stderrp, "%5u %6u "
1836 | "%11" PY_FORMAT_SIZE_T"l" "u "
1837 | "%15" PY_FORMAT_SIZE_T"l" "u "
1838 | "%13" PY_FORMAT_SIZE_T"l" "u\n",
1839 | i, size, p, b, f);
1840 | allocated_bytes += b * size;
1841 | available_bytes += f * size;
1842 | pool_header_bytes += p * POOL_OVERHEAD(((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1));
1843 | quantization += p * ((POOL_SIZE(4 * 1024) - POOL_OVERHEAD(((sizeof(struct pool_header)) + (8 - 1)) & ~(8 - 1))) % size);
1844 | }
1845 | fputc('\n', stderr__stderrp);
1846 | (void)printone("# times object malloc called", serialno);
1847 |
1848 | (void)printone("# arenas allocated total", ntimes_arena_allocated);
1849 | (void)printone("# arenas reclaimed", ntimes_arena_allocated - narenas);
1850 | (void)printone("# arenas highwater mark", narenas_highwater);
1851 | (void)printone("# arenas allocated current", narenas);
1852 |
1853 | PyOS_snprintf(buf, sizeof(buf),
1854 | "%" PY_FORMAT_SIZE_T"l" "u arenas * %d bytes/arena",
1855 | narenas, ARENA_SIZE(256 << 10));
1856 | (void)printone(buf, narenas * ARENA_SIZE(256 << 10));
1857 |
1858 | fputc('\n', stderr__stderrp);
1859 |
1860 | total = printone("# bytes in allocated blocks", allocated_bytes);
1861 | total += printone("# bytes in available blocks", available_bytes);
1862 |
1863 | PyOS_snprintf(buf, sizeof(buf),
1864 | "%u unused pools * %d bytes", numfreepools, POOL_SIZE(4 * 1024));
1865 | total += printone(buf, (size_t)numfreepools * POOL_SIZE(4 * 1024));
1866 |
1867 | total += printone("# bytes lost to pool headers", pool_header_bytes);
1868 | total += printone("# bytes lost to quantization", quantization);
1869 | total += printone("# bytes lost to arena alignment", arena_alignment);
1870 | (void)printone("Total", total);
1871 | }
1872 |
1873 | #endif /* PYMALLOC_DEBUG */
1874 |
1875 | #ifdef Py_USING_MEMORY_DEBUGGER
1876 | /* Make this function last so gcc won't inline it since the definition is
1877 | * after the reference.
1878 | */
1879 | int
1880 | Py_ADDRESS_IN_RANGE(void *P, poolp pool)((arenaindex_temp = (poolp pool)->arenaindex) < maxarenas && (Py_uintptr_t)(void *P) - arenas[arenaindex_temp] .address < (Py_uintptr_t)(256 << 10) && arenas [arenaindex_temp].address != 0)
1881 | {
1882 | uintunsigned int arenaindex_temp = pool->arenaindex;
1883 |
1884 | return arenaindex_temp < maxarenas &&
1885 | (uptrPy_uintptr_t)P - arenas[arenaindex_temp].address < (uptrPy_uintptr_t)ARENA_SIZE(256 << 10) &&
1886 | arenas[arenaindex_temp].address != 0;
1887 | }
1888 | #endif