diff -r da020e408c7f Include/pymem.h --- a/Include/pymem.h Tue Mar 08 16:12:46 2016 +0200 +++ b/Include/pymem.h Wed Mar 09 12:09:38 2016 +0100 @@ -16,6 +16,8 @@ PyAPI_FUNC(void *) PyMem_RawMalloc(size_ PyAPI_FUNC(void *) PyMem_RawCalloc(size_t nelem, size_t elsize); PyAPI_FUNC(void *) PyMem_RawRealloc(void *ptr, size_t new_size); PyAPI_FUNC(void) PyMem_RawFree(void *ptr); + +PyAPI_FUNC(int) _PyMem_SetupAllocators(const char *opt); #endif diff -r da020e408c7f Lib/test/test_capi.py --- a/Lib/test/test_capi.py Tue Mar 08 16:12:46 2016 +0200 +++ b/Lib/test/test_capi.py Wed Mar 09 12:09:38 2016 +0100 @@ -6,6 +6,7 @@ import pickle import random import subprocess import sys +import sysconfig import textwrap import time import unittest @@ -521,6 +522,7 @@ class SkipitemTest(unittest.TestCase): self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, (), {}, b'', [42]) + @unittest.skipUnless(threading, 'Threading required for this test.') class TestThreadState(unittest.TestCase): @@ -545,6 +547,7 @@ class TestThreadState(unittest.TestCase) t.start() t.join() + class Test_testcapi(unittest.TestCase): def test__testcapi(self): for name in dir(_testcapi): @@ -553,5 +556,53 @@ class Test_testcapi(unittest.TestCase): test = getattr(_testcapi, name) test() + +class MallocTests(unittest.TestCase): + ENV = 'malloc_debug' + + def check(self, code): + with support.SuppressCrashReport(): + out = assert_python_failure('-c', code, PYTHONMALLOC=self.ENV) + stderr = out.err + return stderr.decode('ascii', 'replace') + + def test_buffer_overflow(self): + out = self.check('import _testcapi; _testcapi.pymem_buffer_overflow()') + regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'm'\n" + r" 16 bytes originally requested\n" + r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" + r" The 8 pad bytes at tail=0x[0-9a-f]+ are not all FORBIDDENBYTE \(0x[0-9a-f]{2}\):\n" + r" at tail\+0: 0x78 \*\*\* OUCH\n" + r" at tail\+1: 0xfb\n" + r" at tail\+2: 0xfb\n" + r" at tail\+3: 0xfb\n" + r" at tail\+4: 0xfb\n" + r" at tail\+5: 0xfb\n" + r" at tail\+6: 0xfb\n" + r" at tail\+7: 0xfb\n" + r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r" Data at p: cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb cb\n" + r"Fatal Python error: bad trailing pad byte") + self.assertRegex(out, regex) + + def test_api_misuse(self): + out = self.check('import _testcapi; _testcapi.pymem_api_misuse()') + regex = (r"Debug memory block at address p=0x[0-9a-f]+: API 'o'\n" + r" 16 bytes originally requested\n" + r" The 7 pad bytes at p-7 are FORBIDDENBYTE, as expected.\n" + r" The 8 pad bytes at tail=0x[0-9a-f]+ are FORBIDDENBYTE, as expected.\n" + r" The block was made by call #[0-9]+ to debug malloc/realloc.\n" + r" Data at p: .*\n" + r"Fatal Python error: bad ID: Allocated using API 'o', verified using API 'm'\n") + self.assertRegex(out, regex) + + + +@unittest.skipUnless(sysconfig.get_config_var('WITH_PYMALLOC') == 1, + 'need pymalloc') +class PymallocTests(MallocTests): + ENV = 'pymalloc_debug' + + if __name__ == "__main__": unittest.main() diff -r da020e408c7f Modules/_testcapimodule.c --- a/Modules/_testcapimodule.c Tue Mar 08 16:12:46 2016 +0200 +++ b/Modules/_testcapimodule.c Wed Mar 09 12:09:38 2016 +0100 @@ -3616,6 +3616,29 @@ get_recursion_depth(PyObject *self, PyOb return PyLong_FromLong(tstate->recursion_depth - 1); } +static PyObject* +pymem_buffer_overflow(PyObject *self, PyObject *args) +{ + char *buffer; + + buffer = PyMem_Malloc(16); + buffer[16] = 'x'; + PyMem_Free(buffer); + + Py_RETURN_NONE; +} + +static PyObject* +pymem_api_misuse(PyObject *self, PyObject *args) +{ + char *buffer; + + buffer = PyObject_Malloc(16); + PyMem_Free(buffer); + + Py_RETURN_NONE; +} + static PyMethodDef TestMethods[] = { {"raise_exception", raise_exception, METH_VARARGS}, @@ -3798,6 +3821,8 @@ static PyMethodDef TestMethods[] = { {"PyTime_AsMilliseconds", test_PyTime_AsMilliseconds, METH_VARARGS}, {"PyTime_AsMicroseconds", test_PyTime_AsMicroseconds, METH_VARARGS}, {"get_recursion_depth", get_recursion_depth, METH_NOARGS}, + {"pymem_buffer_overflow", pymem_buffer_overflow, METH_NOARGS}, + {"pymem_api_misuse", pymem_api_misuse, METH_NOARGS}, {NULL, NULL} /* sentinel */ }; diff -r da020e408c7f Objects/obmalloc.c --- a/Objects/obmalloc.c Tue Mar 08 16:12:46 2016 +0200 +++ b/Objects/obmalloc.c Wed Mar 09 12:09:38 2016 +0100 @@ -2,7 +2,19 @@ /* Python's malloc wrappers (see pymem.h) */ -#ifdef PYMALLOC_DEBUG /* WITH_PYMALLOC && PYMALLOC_DEBUG */ +/* + * Basic types + * I don't care if these are defined in or elsewhere. Axiom. + */ +#undef uchar +#define uchar unsigned char /* assuming == 8 bits */ + +#undef uint +#define uint unsigned int /* assuming >= 16 bits */ + +#undef uptr +#define uptr Py_uintptr_t + /* Forward declaration */ static void* _PyMem_DebugMalloc(void *ctx, size_t size); static void* _PyMem_DebugCalloc(void *ctx, size_t nelem, size_t elsize); @@ -11,7 +23,6 @@ static void* _PyMem_DebugRealloc(void *c static void _PyObject_DebugDumpAddress(const void *p); static void _PyMem_DebugCheckAddress(char api_id, const void *p); -#endif #if defined(__has_feature) /* Clang */ #if __has_feature(address_sanitizer) /* is ASAN enabled? */ @@ -147,7 +158,6 @@ static void #endif #define PYMEM_FUNCS PYRAW_FUNCS -#ifdef PYMALLOC_DEBUG typedef struct { /* We tag each block with an API ID in order to tag API violations */ char api_id; @@ -164,7 +174,6 @@ static struct { }; #define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree -#endif static PyMemAllocatorEx _PyMem_Raw = { #ifdef PYMALLOC_DEBUG @@ -190,6 +199,39 @@ static PyMemAllocatorEx _PyObject = { #endif }; +int +_PyMem_SetupAllocators(const char *opt) +{ + if (strcmp(opt, "malloc") == 0 + || strcmp(opt, "malloc_debug") == 0) { + PyMemAllocatorEx alloc = {NULL, PYRAW_FUNCS}; + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + + if (strcmp(opt, "malloc_debug") == 0) + PyMem_SetupDebugHooks(); + } +#ifdef WITH_PYMALLOC + else if (strcmp(opt, "pymalloc") == 0 + || strcmp(opt, "pymalloc_debug") == 0) { + PyMemAllocatorEx mem_alloc = {NULL, PYRAW_FUNCS}; + PyMemAllocatorEx obj_alloc = {NULL, PYOBJ_FUNCS}; + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &mem_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &mem_alloc); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &obj_alloc); + + if (strcmp(opt, "pymalloc_debug") == 0) + PyMem_SetupDebugHooks(); + } +#endif + else { + /* unknown allocator */ + return -1; + } + return 0; +} + #undef PYRAW_FUNCS #undef PYMEM_FUNCS #undef PYOBJ_FUNCS @@ -205,12 +247,21 @@ static PyObjectArenaAllocator _PyObject_ #endif }; +static int +_PyMem_DebugEnabled(void) +{ + return (_PyObject.malloc == _PyMem_DebugMalloc); +} + void PyMem_SetupDebugHooks(void) { -#ifdef PYMALLOC_DEBUG PyMemAllocatorEx alloc; + /* hooks already installed */ + if (_PyMem_DebugEnabled()) + return; + alloc.malloc = _PyMem_DebugMalloc; alloc.calloc = _PyMem_DebugCalloc; alloc.realloc = _PyMem_DebugRealloc; @@ -233,7 +284,6 @@ PyMem_SetupDebugHooks(void) PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); } -#endif } void @@ -264,7 +314,6 @@ PyMem_SetAllocator(PyMemAllocatorDomain case PYMEM_DOMAIN_OBJ: _PyObject = *allocator; break; /* ignore unknown domain */ } - } void @@ -642,22 +691,6 @@ static int running_on_valgrind = -1; #define SIMPLELOCK_LOCK(lock) /* acquire released lock */ #define SIMPLELOCK_UNLOCK(lock) /* release acquired lock */ -/* - * Basic types - * I don't care if these are defined in or elsewhere. Axiom. - */ -#undef uchar -#define uchar unsigned char /* assuming == 8 bits */ - -#undef uint -#define uint unsigned int /* assuming >= 16 bits */ - -#undef ulong -#define ulong unsigned long /* assuming >= 32 bits */ - -#undef uptr -#define uptr Py_uintptr_t - /* When you say memory, my mind reasons in terms of (pointers to) blocks */ typedef uchar block; @@ -950,10 +983,8 @@ new_arena(void) uint excess; /* number of bytes above pool alignment */ void *address; -#ifdef PYMALLOC_DEBUG - if (Py_GETENV("PYTHONMALLOCSTATS")) + if (_PyMem_DebugEnabled() && Py_GETENV("PYTHONMALLOCSTATS")) _PyObject_DebugMallocStats(stderr); -#endif if (unused_arena_objects == NULL) { uint i; uint numarenas; @@ -1709,7 +1740,7 @@ Py_ssize_t #endif /* WITH_PYMALLOC */ -#ifdef PYMALLOC_DEBUG + /*==========================================================================*/ /* A x-platform debugging allocator. This doesn't manage memory directly, * it wraps a real allocator, adding extra debugging info to the memory blocks. @@ -1767,31 +1798,6 @@ write_size_t(void *p, size_t n) } } -#ifdef Py_DEBUG -/* Is target in the list? The list is traversed via the nextpool pointers. - * The list may be NULL-terminated, or circular. Return 1 if target is in - * list, else 0. - */ -static int -pool_is_in_list(const poolp target, poolp list) -{ - poolp origlist = list; - assert(target != NULL); - if (list == NULL) - return 0; - do { - if (target == list) - return 1; - list = list->nextpool; - } while (list != NULL && list != origlist); - return 0; -} - -#else -#define pool_is_in_list(X, Y) 1 - -#endif /* Py_DEBUG */ - /* Let S = sizeof(size_t). The debug malloc asks for 4*S extra bytes and fills them with useful stuff, here calling the underlying malloc's result p: @@ -2106,7 +2112,6 @@ static void } } -#endif /* PYMALLOC_DEBUG */ static size_t printone(FILE *out, const char* msg, size_t value) @@ -2158,8 +2163,28 @@ void (void)printone(out, buf2, num_blocks * sizeof_block); } + #ifdef WITH_PYMALLOC +/* Is target in the list? The list is traversed via the nextpool pointers. + * The list may be NULL-terminated, or circular. Return 1 if target is in + * list, else 0. + */ +static int +pool_is_in_list(const poolp target, poolp list) +{ + poolp origlist = list; + assert(target != NULL); + if (list == NULL) + return 0; + do { + if (target == list) + return 1; + list = list->nextpool; + } while (list != NULL && list != origlist); + return 0; +} + /* Print summary info to "out" about the state of pymalloc's structures. * In Py_DEBUG mode, also perform some expensive internal consistency * checks. @@ -2273,9 +2298,8 @@ void quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } fputc('\n', out); -#ifdef PYMALLOC_DEBUG - (void)printone(out, "# times object malloc called", serialno); -#endif + if (_PyMem_DebugEnabled()) + (void)printone(out, "# times object malloc called", serialno); (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); (void)printone(out, "# arenas highwater mark", narenas_highwater); diff -r da020e408c7f Programs/python.c --- a/Programs/python.c Tue Mar 08 16:12:46 2016 +0200 +++ b/Programs/python.c Wed Mar 09 12:09:38 2016 +0100 @@ -7,6 +7,19 @@ #include #endif +static void +setup_malloc(void) +{ + char *opt = getenv("PYTHONMALLOC"); + if (!opt) + return; + + if (_PyMem_SetupAllocators(opt) < 0) { + fprintf(stderr, "Error in PYTHONMALLOC: unknown allocator \"%s\"!\n", opt); + exit(1); + } +} + #ifdef MS_WINDOWS int wmain(int argc, wchar_t **argv) @@ -24,6 +37,8 @@ main(int argc, char **argv) int i, res; char *oldloc; + setup_malloc(); + argv_copy = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); argv_copy2 = (wchar_t **)PyMem_RawMalloc(sizeof(wchar_t*) * (argc+1)); if (!argv_copy || !argv_copy2) {