diff -r c5171d50e752 Doc/library/sys.rst --- a/Doc/library/sys.rst Mon May 14 16:51:35 2012 +0200 +++ b/Doc/library/sys.rst Mon May 14 14:38:00 2012 -0400 @@ -80,6 +80,22 @@ This function should be used for internal and specialized purposes only. +.. function:: _debugmallocstats() + + Print low-level information to stderr about the state of CPython's memory + allocator. + + If Python is configured --with-pydebug, it also performs some expensive + internal consistency checks. + + .. versionadded:: 3.3 + + .. impl-detail:: + + This function is specific to CPython. The exact output format is not + defined here, and may change. + + .. data:: dllhandle Integer specifying the handle of the Python DLL. Availability: Windows. diff -r c5171d50e752 Include/dictobject.h --- a/Include/dictobject.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/dictobject.h Mon May 14 14:38:00 2012 -0400 @@ -111,6 +111,7 @@ #ifndef Py_LIMITED_API int _PyObjectDict_SetItem(PyTypeObject *tp, PyObject **dictptr, PyObject *name, PyObject *value); PyObject *_PyDict_LoadGlobal(PyDictObject *, PyDictObject *, PyObject *); +PyAPI_FUNC(void) _PyDict_DebugMallocStats(FILE *out); #endif #ifdef __cplusplus diff -r c5171d50e752 Include/floatobject.h --- a/Include/floatobject.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/floatobject.h Mon May 14 14:38:00 2012 -0400 @@ -110,6 +110,8 @@ /* free list api */ PyAPI_FUNC(int) PyFloat_ClearFreeList(void); +PyAPI_FUNC(void) _PyFloat_DebugMallocStats(FILE* out); + /* Format the object based on the format_spec, as defined in PEP 3101 (Advanced String Formatting). */ PyAPI_FUNC(PyObject *) _PyFloat_FormatAdvanced(PyObject *obj, diff -r c5171d50e752 Include/frameobject.h --- a/Include/frameobject.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/frameobject.h Mon May 14 14:38:00 2012 -0400 @@ -79,6 +79,8 @@ PyAPI_FUNC(int) PyFrame_ClearFreeList(void); +PyAPI_FUNC(void) _PyFrame_DebugMallocStats(FILE *out); + /* Return the line of code the frame is currently executing. */ PyAPI_FUNC(int) PyFrame_GetLineNumber(PyFrameObject *); diff -r c5171d50e752 Include/listobject.h --- a/Include/listobject.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/listobject.h Mon May 14 14:38:00 2012 -0400 @@ -64,6 +64,7 @@ PyAPI_FUNC(PyObject *) _PyList_Extend(PyListObject *, PyObject *); PyAPI_FUNC(int) PyList_ClearFreeList(void); +PyAPI_FUNC(void) _PyList_DebugMallocStats(FILE *out); #endif /* Macro, trading safety for speed */ diff -r c5171d50e752 Include/methodobject.h --- a/Include/methodobject.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/methodobject.h Mon May 14 14:38:00 2012 -0400 @@ -82,6 +82,11 @@ PyAPI_FUNC(int) PyCFunction_ClearFreeList(void); +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyCFunction_DebugMallocStats(FILE *out); +PyAPI_FUNC(void) _PyMethod_DebugMallocStats(FILE *out); +#endif + #ifdef __cplusplus } #endif diff -r c5171d50e752 Include/object.h --- a/Include/object.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/object.h Mon May 14 14:38:00 2012 -0400 @@ -977,6 +977,14 @@ else \ _PyTrash_deposit_object((PyObject*)op); +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) +_PyDebugAllocatorStats(FILE *out, const char *block_name, int num_blocks, + size_t sizeof_block); +PyAPI_FUNC(void) +_PyObject_DebugTypeStats(FILE *out); +#endif /* ifndef Py_LIMITED_API */ + #ifdef __cplusplus } #endif diff -r c5171d50e752 Include/objimpl.h --- a/Include/objimpl.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/objimpl.h Mon May 14 14:38:00 2012 -0400 @@ -101,13 +101,15 @@ /* Macros */ #ifdef WITH_PYMALLOC +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyObject_DebugMallocStats(FILE *out); +#endif /* #ifndef Py_LIMITED_API */ #ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ PyAPI_FUNC(void *) _PyObject_DebugMalloc(size_t nbytes); PyAPI_FUNC(void *) _PyObject_DebugRealloc(void *p, size_t nbytes); PyAPI_FUNC(void) _PyObject_DebugFree(void *p); PyAPI_FUNC(void) _PyObject_DebugDumpAddress(const void *p); PyAPI_FUNC(void) _PyObject_DebugCheckAddress(const void *p); -PyAPI_FUNC(void) _PyObject_DebugMallocStats(void); PyAPI_FUNC(void *) _PyObject_DebugMallocApi(char api, size_t nbytes); PyAPI_FUNC(void *) _PyObject_DebugReallocApi(char api, void *p, size_t nbytes); PyAPI_FUNC(void) _PyObject_DebugFreeApi(char api, void *p); diff -r c5171d50e752 Include/setobject.h --- a/Include/setobject.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/setobject.h Mon May 14 14:38:00 2012 -0400 @@ -101,6 +101,7 @@ PyAPI_FUNC(int) _PySet_Update(PyObject *set, PyObject *iterable); PyAPI_FUNC(int) PySet_ClearFreeList(void); +PyAPI_FUNC(void) _PySet_DebugMallocStats(FILE *out); #endif #ifdef __cplusplus diff -r c5171d50e752 Include/tupleobject.h --- a/Include/tupleobject.h Mon May 14 16:51:35 2012 +0200 +++ b/Include/tupleobject.h Mon May 14 14:38:00 2012 -0400 @@ -63,6 +63,9 @@ #endif PyAPI_FUNC(int) PyTuple_ClearFreeList(void); +#ifndef Py_LIMITED_API +PyAPI_FUNC(void) _PyTuple_DebugMallocStats(FILE *out); +#endif /* Py_LIMITED_API */ #ifdef __cplusplus } diff -r c5171d50e752 Lib/test/test_sys.py --- a/Lib/test/test_sys.py Mon May 14 16:51:35 2012 +0200 +++ b/Lib/test/test_sys.py Mon May 14 14:38:00 2012 -0400 @@ -577,6 +577,12 @@ expected = None self.check_fsencoding(fs_encoding, expected) + def test_debugmallocstats(self): + # Test sys._debugmallocstats() + from test.script_helper import assert_python_ok + args = ['-c', 'import sys; sys._debugmallocstats()'] + ret, out, err = assert_python_ok(*args) + self.assertIn(b"free PyDictObjects", err) class SizeofTest(unittest.TestCase): diff -r c5171d50e752 Misc/NEWS --- a/Misc/NEWS Mon May 14 16:51:35 2012 +0200 +++ b/Misc/NEWS Mon May 14 14:38:00 2012 -0400 @@ -85,6 +85,9 @@ - Issue #14127 and #10148: shutil.copystat now preserves exact mtime and atime on filesystems providing nanosecond resolution. +- Issue #14785: Add sys._debugmallocstats() to help debug low-level memory + allocation issues + Tools/Demos ----------- diff -r c5171d50e752 Objects/classobject.c --- a/Objects/classobject.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/classobject.c Mon May 14 14:38:00 2012 -0400 @@ -400,6 +400,15 @@ (void)PyMethod_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyMethod_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyMethodObject", + numfree, sizeof(PyMethodObject)); +} + /* ------------------------------------------------------------------------ * instance method */ diff -r c5171d50e752 Objects/dictobject.c --- a/Objects/dictobject.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/dictobject.c Mon May 14 14:38:00 2012 -0400 @@ -255,6 +255,15 @@ return ret; } +/* Print summary info about the state of the optimized allocator */ +void +_PyDict_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyDictObject", numfree, sizeof(PyDictObject)); +} + + void PyDict_Fini(void) { diff -r c5171d50e752 Objects/floatobject.c --- a/Objects/floatobject.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/floatobject.c Mon May 14 14:38:00 2012 -0400 @@ -1920,6 +1920,16 @@ (void)PyFloat_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyFloat_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyFloatObject", + numfree, sizeof(PyFloatObject)); +} + + /*---------------------------------------------------------------------------- * _PyFloat_{Pack,Unpack}{4,8}. See floatobject.h. */ diff -r c5171d50e752 Objects/frameobject.c --- a/Objects/frameobject.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/frameobject.c Mon May 14 14:38:00 2012 -0400 @@ -953,3 +953,13 @@ Py_XDECREF(builtin_object); builtin_object = NULL; } + +/* Print summary info about the state of the optimized allocator */ +void +_PyFrame_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyFrameObject", + numfree, sizeof(PyFrameObject)); +} + diff -r c5171d50e752 Objects/listobject.c --- a/Objects/listobject.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/listobject.c Mon May 14 14:38:00 2012 -0400 @@ -117,6 +117,15 @@ PyList_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyList_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyListObject", + numfree, sizeof(PyListObject)); +} + PyObject * PyList_New(Py_ssize_t size) { diff -r c5171d50e752 Objects/methodobject.c --- a/Objects/methodobject.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/methodobject.c Mon May 14 14:38:00 2012 -0400 @@ -338,6 +338,15 @@ (void)PyCFunction_ClearFreeList(); } +/* Print summary info about the state of the optimized allocator */ +void +_PyCFunction_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PyCFunction", + numfree, sizeof(PyCFunction)); +} + /* PyCFunction_New() is now just a macro that calls PyCFunction_NewEx(), but it's part of the API so we need to keep a function around that existing C extensions can call. diff -r c5171d50e752 Objects/object.c --- a/Objects/object.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/object.c Mon May 14 14:38:00 2012 -0400 @@ -1849,6 +1849,18 @@ PyMem_FREE(p); } +void +_PyObject_DebugTypeStats(FILE *out) +{ + _PyCFunction_DebugMallocStats(out); + _PyDict_DebugMallocStats(out); + _PyFloat_DebugMallocStats(out); + _PyFrame_DebugMallocStats(out); + _PyList_DebugMallocStats(out); + _PyMethod_DebugMallocStats(out); + _PySet_DebugMallocStats(out); + _PyTuple_DebugMallocStats(out); +} /* These methods are used to control infinite recursion in repr, str, print, etc. Container objects that may recursively contain themselves, diff -r c5171d50e752 Objects/obmalloc.c --- a/Objects/obmalloc.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/obmalloc.c Mon May 14 14:38:00 2012 -0400 @@ -523,12 +523,10 @@ /* Number of arenas allocated that haven't been free()'d. */ static size_t narenas_currently_allocated = 0; -#ifdef PYMALLOC_DEBUG /* Total number of times malloc() called to allocate an arena. */ static size_t ntimes_arena_allocated = 0; /* High water mark (max value ever seen) for narenas_currently_allocated. */ static size_t narenas_highwater = 0; -#endif /* Allocate a new arena. If we run out of memory, return NULL. Else * allocate a new arena, and return the address of an arena_object @@ -545,7 +543,7 @@ #ifdef PYMALLOC_DEBUG if (Py_GETENV("PYTHONMALLOCSTATS")) - _PyObject_DebugMallocStats(); + _PyObject_DebugMallocStats(stderr); #endif if (unused_arena_objects == NULL) { uint i; @@ -613,11 +611,9 @@ arenaobj->address = (uptr)address; ++narenas_currently_allocated; -#ifdef PYMALLOC_DEBUG ++ntimes_arena_allocated; if (narenas_currently_allocated > narenas_highwater) narenas_highwater = narenas_currently_allocated; -#endif arenaobj->freepools = NULL; /* pool_address <- first pool-aligned address in the arena nfreepools <- number of whole pools that fit after alignment */ @@ -1723,17 +1719,19 @@ } } +#endif /* PYMALLOC_DEBUG */ + static size_t -printone(const char* msg, size_t value) +printone(FILE *out, const char* msg, size_t value) { int i, k; char buf[100]; size_t origvalue = value; - fputs(msg, stderr); + fputs(msg, out); for (i = (int)strlen(msg); i < 35; ++i) - fputc(' ', stderr); - fputc('=', stderr); + fputc(' ', out); + fputc('=', out); /* Write the value with commas. */ i = 22; @@ -1754,17 +1752,33 @@ while (i >= 0) buf[i--] = ' '; - fputs(buf, stderr); + fputs(buf, out); return origvalue; } -/* Print summary info to stderr about the state of pymalloc's structures. +void +_PyDebugAllocatorStats(FILE *out, + const char *block_name, int num_blocks, size_t sizeof_block) +{ + char buf1[128]; + char buf2[128]; + PyOS_snprintf(buf1, sizeof(buf1), + "%d %ss * %zd bytes each", + num_blocks, block_name, sizeof_block); + PyOS_snprintf(buf2, sizeof(buf2), + "%48s ", buf1); + (void)printone(out, buf2, num_blocks * sizeof_block); +} + +#ifdef WITH_PYMALLOC + +/* Print summary info to "out" about the state of pymalloc's structures. * In Py_DEBUG mode, also perform some expensive internal consistency * checks. */ void -_PyObject_DebugMallocStats(void) +_PyObject_DebugMallocStats(FILE *out) { uint i; const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT; @@ -1793,7 +1807,7 @@ size_t total; char buf[128]; - fprintf(stderr, "Small block threshold = %d, in %u size classes.\n", + fprintf(out, "Small block threshold = %d, in %u size classes.\n", SMALL_REQUEST_THRESHOLD, numclasses); for (i = 0; i < numclasses; ++i) @@ -1847,10 +1861,10 @@ } assert(narenas == narenas_currently_allocated); - fputc('\n', stderr); + fputc('\n', out); fputs("class size num pools blocks in use avail blocks\n" "----- ---- --------- ------------- ------------\n", - stderr); + out); for (i = 0; i < numclasses; ++i) { size_t p = numpools[i]; @@ -1861,7 +1875,7 @@ assert(b == 0 && f == 0); continue; } - fprintf(stderr, "%5u %6u " + fprintf(out, "%5u %6u " "%11" PY_FORMAT_SIZE_T "u " "%15" PY_FORMAT_SIZE_T "u " "%13" PY_FORMAT_SIZE_T "u\n", @@ -1871,35 +1885,36 @@ pool_header_bytes += p * POOL_OVERHEAD; quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } - fputc('\n', stderr); - (void)printone("# times object malloc called", serialno); - - (void)printone("# arenas allocated total", ntimes_arena_allocated); - (void)printone("# arenas reclaimed", ntimes_arena_allocated - narenas); - (void)printone("# arenas highwater mark", narenas_highwater); - (void)printone("# arenas allocated current", narenas); + fputc('\n', out); +#ifdef PYMALLOC_DEBUG + (void)printone(out, "# times object malloc called", serialno); +#endif + (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); + (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); + (void)printone(out, "# arenas highwater mark", narenas_highwater); + (void)printone(out, "# arenas allocated current", narenas); PyOS_snprintf(buf, sizeof(buf), "%" PY_FORMAT_SIZE_T "u arenas * %d bytes/arena", narenas, ARENA_SIZE); - (void)printone(buf, narenas * ARENA_SIZE); + (void)printone(out, buf, narenas * ARENA_SIZE); - fputc('\n', stderr); + fputc('\n', out); - total = printone("# bytes in allocated blocks", allocated_bytes); - total += printone("# bytes in available blocks", available_bytes); + total = printone(out, "# bytes in allocated blocks", allocated_bytes); + total += printone(out, "# bytes in available blocks", available_bytes); PyOS_snprintf(buf, sizeof(buf), "%u unused pools * %d bytes", numfreepools, POOL_SIZE); - total += printone(buf, (size_t)numfreepools * POOL_SIZE); + total += printone(out, buf, (size_t)numfreepools * POOL_SIZE); - total += printone("# bytes lost to pool headers", pool_header_bytes); - total += printone("# bytes lost to quantization", quantization); - total += printone("# bytes lost to arena alignment", arena_alignment); - (void)printone("Total", total); + total += printone(out, "# bytes lost to pool headers", pool_header_bytes); + total += printone(out, "# bytes lost to quantization", quantization); + total += printone(out, "# bytes lost to arena alignment", arena_alignment); + (void)printone(out, "Total", total); } -#endif /* PYMALLOC_DEBUG */ +#endif /* #ifdef WITH_PYMALLOC */ #ifdef Py_USING_MEMORY_DEBUGGER /* Make this function last so gcc won't inline it since the definition is diff -r c5171d50e752 Objects/setobject.c --- a/Objects/setobject.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/setobject.c Mon May 14 14:38:00 2012 -0400 @@ -1133,6 +1133,16 @@ Py_CLEAR(emptyfrozenset); } +/* Print summary info about the state of the optimized allocator */ +void +_PySet_DebugMallocStats(FILE *out) +{ + _PyDebugAllocatorStats(out, + "free PySetObject", + numfree, sizeof(PySetObject)); +} + + static PyObject * set_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { diff -r c5171d50e752 Objects/tupleobject.c --- a/Objects/tupleobject.c Mon May 14 16:51:35 2012 +0200 +++ b/Objects/tupleobject.c Mon May 14 14:38:00 2012 -0400 @@ -45,6 +45,22 @@ } #endif +/* Print summary info about the state of the optimized allocator */ +void +_PyTuple_DebugMallocStats(FILE *out) +{ +#if PyTuple_MAXSAVESIZE > 0 + int i; + char buf[128]; + for (i = 1; i < PyTuple_MAXSAVESIZE; i++) { + PyOS_snprintf(buf, sizeof(buf), + "free %d-sized PyTupleObject", i); + _PyDebugAllocatorStats(out, + buf, + numfree[i], _PyObject_VAR_SIZE(&PyTuple_Type, i)); + } +#endif +} PyObject * PyTuple_New(register Py_ssize_t size) diff -r c5171d50e752 Python/pythonrun.c --- a/Python/pythonrun.c Mon May 14 16:51:35 2012 +0200 +++ b/Python/pythonrun.c Mon May 14 14:38:00 2012 -0400 @@ -633,7 +633,7 @@ #endif /* Py_TRACE_REFS */ #ifdef PYMALLOC_DEBUG if (Py_GETENV("PYTHONMALLOCSTATS")) - _PyObject_DebugMallocStats(); + _PyObject_DebugMallocStats(stderr); #endif call_ll_exitfuncs(); diff -r c5171d50e752 Python/sysmodule.c --- a/Python/sysmodule.c Mon May 14 16:51:35 2012 +0200 +++ b/Python/sysmodule.c Mon May 14 14:38:00 2012 -0400 @@ -997,6 +997,27 @@ extern "C" { #endif +static PyObject * +sys_debugmallocstats(PyObject *self, PyObject *args) +{ +#ifdef WITH_PYMALLOC + _PyObject_DebugMallocStats(stderr); + fputc('\n', stderr); +#endif + _PyObject_DebugTypeStats(stderr); + + Py_RETURN_NONE; +} +PyDoc_STRVAR(debugmallocstats_doc, +"_debugmallocstats()\n\ +\n\ +Print summary info to stderr about the state of\n\ +pymalloc's structures.\n\ +\n\ +In Py_DEBUG mode, also perform some expensive internal consistency\n\ +checks.\n\ +"); + #ifdef Py_TRACE_REFS /* Defined in objects.c because it uses static globals if that file */ extern PyObject *_Py_GetObjects(PyObject *, PyObject *); @@ -1093,6 +1114,8 @@ {"settrace", sys_settrace, METH_O, settrace_doc}, {"gettrace", sys_gettrace, METH_NOARGS, gettrace_doc}, {"call_tracing", sys_call_tracing, METH_VARARGS, call_tracing_doc}, + {"_debugmallocstats", sys_debugmallocstats, METH_VARARGS, + debugmallocstats_doc}, {NULL, NULL} /* sentinel */ };