diff -r 2ed8d500e113 -r 65e72bf01246 Doc/library/debug.rst --- a/Doc/library/debug.rst Sun Nov 03 13:53:12 2013 +0100 +++ b/Doc/library/debug.rst Sun Nov 03 14:25:07 2013 +0100 @@ -15,3 +15,4 @@ allowing you to identify bottlenecks in profile.rst timeit.rst trace.rst + tracemalloc.rst diff -r 2ed8d500e113 -r 65e72bf01246 Doc/library/tracemalloc.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/tracemalloc.rst Sun Nov 03 14:25:07 2013 +0100 @@ -0,0 +1,590 @@ +:mod:`tracemalloc` --- Trace memory allocations +=============================================== + +.. module:: tracemalloc + :synopsis: Trace memory allocations. + +The tracemalloc module is a debug tool to trace memory blocks allocated by +Python. It provides the following information: + +* Traceback where an object was allocated +* Statistics on allocated memory blocks per filename and per line number: + total size, number and average size of allocated memory blocks +* Compute the differences between two snapshots to detect memory leaks + +To trace most memory blocks allocated by Python, the module should be started +as early as possible by setting the :envvar:`PYTHONTRACEMALLOC` environment +variable to ``1``, or by using :option:`-X` ``tracemalloc`` command line +option. The :func:`tracemalloc.start` function can be called at runtime to +start tracing Python memory allocations. + +By default, a trace of an allocated memory block only stores the most recent +frame (1 frame). To store 25 frames at startup: set the +:envvar:`PYTHONTRACEMALLOC` environment variable to ``25``, or use the +:option:`-X` ``tracemalloc=25`` command line option. The +:func:`set_traceback_limit` function can be used at runtime to set the limit. + +By default, Python memory blocks allocated in the :mod:`tracemalloc` module are +ignored using a filter. Use :func:`clear_filters` to trace also these memory +allocations. + +.. versionadded:: 3.4 + + +Examples +======== + +Display the top 10 +------------------ + +Display the 10 files allocating the most memory:: + + import tracemalloc + + tracemalloc.start() + + # ... run your application ... + + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics('filename') + + print("[ Top 10 ]") + for stat in top_stats[:10]: + filename = stat.key + print("%s: %.1f kB" % (stat.filename, stat.size / 1024)) + + +Example of output of the Python test suite:: + + [ Top 10 ] + : 20901.6 kB + /usr/lib/python3.4/linecache.py: 2123.0 kB + /usr/lib/python3.4/test/test_argparse.py: 1937.8 kB + /usr/lib/python3.4/unittest/case.py: 997.1 kB + /usr/lib/python3.4/unittest/mock.py: 851.6 kB + /usr/lib/python3.4/collections/__init__.py: 676.2 kB + /usr/lib/python3.4/sre_parse.py: 377.3 kB + /usr/lib/python3.4/mimetypes.py: 319.5 kB + : 315.5 kB + /usr/lib/python3.4/abc.py: 311.5 kB + +See :meth:`Snapshot.statistics` for more options. + + +Compute differences +------------------- + +Take two snapshots and display the differences:: + + import tracemalloc + tracemalloc.start() + # ... start your application ... + + snapshot1 = tracemalloc.take_snapshot() + # ... call the function leaking memory ... + snapshot2 = tracemalloc.take_snapshot() + + top_stats = snapshot2.statistics('lineno', compare_to=snapshot1) + + limit = 10 + print("[ Top %s differences ]" % limit) + for stat in top_stats[:limit]: + filename, lineno = stat.key + print("%s:%s: %.1f kB (%+.1f kB)" + % (filename, lineno, stat.size / 1024, stat.size_diff / 1024)) + +Example of output of a short script:: + + [ Top 10 differences ] + test.py:4: 0.0 kB (-5.0 kB) + test.py:8: 0.6 kB (+0.6 kB) + +If the system has few free memory, snapshots can be written on disk using the +:meth:`Snapshot.dump` method. The snapshot can then be loaded using the +:meth:`Snapshot.load` method to analyze the snapshot after the application +exited, or on another computer. Using files allow also deeper analysis using +filters or different views: see :meth:`Snapshot.apply_filters` and +:meth:`Snapshot.statistics` methods. + + +Get the traceback of a memory block +----------------------------------- + +Code to display the traceback of the biggest memory block:: + + import linecache + import tracemalloc + + tracemalloc.start() + + # ... run your application ... + + snapshot = tracemalloc.take_snapshot() + top_stats = snapshot.statistics('traceback') + + # pick the biggest memory block + stat = top_stats[0] + traceback = stat.key + print("Memory blocks: size=%.1f kB, count=%s" % (stat.size / 1024, stat.count)) + for frame in traceback: + filename, lineno = frame + print(' File "%s", line %s' % (filename, lineno)) + + line = linecache.getline(filename, lineno) + line = line.strip() + if line: + print(' ' + line) + print() + +Example of output of the Python test suite (traceback limited to 25 frames):: + + Memory blocks: 768.0 kB, count=1 + File "", line 704 + File "", line 1024 + File "", line 922 + File "", line 1056 + File "", line 607 + File "", line 1566 + File "", line 1599 + File "Lib/test/support/__init__.py", line 142 + __import__(name) + File "Lib/test/support/__init__.py", line 206 + _save_and_remove_module(name, orig_modules) + File "Lib/test/test_decimal.py", line 48 + C = import_fresh_module('decimal', fresh=['_decimal']) + File "", line 274 + File "", line 926 + File "", line 1056 + File "", line 607 + File "", line 1566 + File "", line 1599 + File "", line 1618 + File "Lib/importlib/__init__.py", line 95 + return _bootstrap._gcd_import(name[level:], package, level) + File "Lib/test/regrtest.py", line 1269 + the_module = importlib.import_module(abstest) + File "Lib/test/regrtest.py", line 976 + display_failure=not verbose) + +.. note:: + + This memory block of 768 kB is the dictionary of Unicode interned strings. + + +Pretty top +---------- + +Display the 10 lines allocating the most memory with a pretty output and all +information, ignoring ```` and ```` +files:: + + import os + import tracemalloc + + def display_top(snapshot, limit=10): + top_stats = snapshot.statistics('lineno') + + print("%s: Top %s lines" % (snapshot.timestamp, limit)) + for index, stat in enumerate(top_stats[:limit], 1): + filename, lineno = stat.key + # replace "/path/to/module/file.py" with "module/file.py" + filename = os.sep.join(filename.split(os.sep)[-2:]) + print("#%s: %s:%s: size=%.1f kB, count=%s, average=%.0f B" + % (index, filename, lineno, + stat.size / 1024, stat.count, stat.size / stat.count)) + other = top_stats[limit:] + if other: + size = sum(stat.size for stat in other) + print("%s other: %.1f kB" % (len(other), size / 1024)) + total = sum(stat.size for stat in top_stats) + print("Total allocated size: %.1f KB" % (total / 1024)) + + tracemalloc.start() + + # ... run your application ... + + snapshot = tracemalloc.take_snapshot() + snapshot.apply_filters(( + tracemalloc.Filter(False, ""), + tracemalloc.Filter(False, ""), + )) + display_top(snapshot, 10) + +Example of output of the Python test suite:: + + 2013-10-25 15:52:53.298949: Top 10 lines + #1: Lib/linecache.py:127: size=225.5 kB, count=2848, average=81 B + #2: collections/__init__.py:368: size=200.6 kB, count=2911, average=71 B + #3: unittest/case.py:571: size=108.6 kB, count=285, average=390 B + #4: Lib/abc.py:133: size=87.3 kB, count=482, average=185 B + #5: Lib/shutil.py:401: size=61.9 kB, count=110, average=576 B + #6: Lib/sre_compile.py:508: size=58.6 kB, count=187, average=321 B + #7: collections/__init__.py:362: size=53.9 kB, count=25, average=2209 B + #8: Lib/_weakrefset.py:37: size=40.5 kB, count=510, average=81 B + #9: urllib/parse.py:476: size=37.1 kB, count=969, average=39 B + #10: support/__init__.py:1529: size=35.0 kB, count=95, average=377 B + 10347 other: 3279.3 kB + Total allocated size: 4188.3 KB + +See :meth:`Snapshot.statistics` for more options. + + +API +=== + +Main functions +-------------- + +.. function:: clear_traces() + + Clear traces of memory blocks allocated by Python. + + See also :func:`stop`. + + +.. function:: get_traced_memory() + + Get the current size and maximum size of memory blocks traced by the + :mod:`tracemalloc` module as a tuple: ``(size: int, max_size: int)``. + + +.. function:: get_tracemalloc_memory() + + Get the memory usage in bytes of the :mod:`tracemalloc` module used to store + traces of memory blocks. + Return an :class:`int`. + + +.. function:: is_tracing() + + ``True`` if the :mod:`tracemalloc` module is tracing Python memory + allocations, ``False`` otherwise. + + See also :func:`start` and :func:`stop` functions. + + +.. function:: stop() + + Stop tracing Python memory allocations and clear traces of memory blocks + allocated by Python. + + The function uninstalls hooks on Python memory allocators, so the overhead + of the module becomes null. + + Call :func:`get_traces` or :func:`take_snapshot` function to get traces + before clearing them. + + See also :func:`start` and :func:`is_tracing` functions. + + +.. function:: start() + + Start tracing Python memory allocations. + + The function installs hooks on Python memory allocators. These hooks have + important overhead in term of performances and memory usage: see `Filter + functions`_ to limit the overhead. + + See also :func:`stop` and :func:`is_tracing` functions. + + +.. function:: take_snapshot() + + Take a snapshot of traces of memory blocks allocated by Python using the + :func:`get_traces` function. Return a new :class:`Snapshot` instance. + + The :mod:`tracemalloc` module must be tracing memory allocations to take a + snapshot, see the the :func:`start` function. + + See also :func:`get_traces` and :func:`get_object_traceback` functions. + + +Trace functions +--------------- + +When Python allocates a memory block, :mod:`tracemalloc` attachs a "trace" to +the memory block to store its size in bytes and the traceback where the +allocation occured. + +The following functions give access to these traces. A trace is a ``(size: int, +traceback)`` tuple. *size* is the size of the memory block in bytes. +*traceback* is a tuple of frames sorted from the most recent to the oldest +frame, limited to :func:`get_traceback_limit` frames. A frame is +a ``(filename: str, lineno: int)`` tuple. + +A traceback contains at least ``1`` frame. If the :mod:`tracemalloc` module +failed to get a frame, the ``""`` filename and the line number ``0`` +are used. If it failed to get the traceback or if the traceback limit is ``0``, +the traceback is ``(('', 0),)``. + +Example of a trace: ``(32, (('x.py', 7), ('x.py', 11)))``. The memory block +has a size of 32 bytes and was allocated at ``x.py:7``, line called from line +``x.py:11``. + + +.. function:: get_object_traceback(obj) + + Get the traceback where the Python object *obj* was allocated. + Return a tuple of ``(filename: str, lineno: int)`` tuples. + + Return ``None`` if the :mod:`tracemalloc` module is not tracing memory + allocations or did not trace the allocation of the object. + + See also :func:`gc.get_referrers` and :func:`sys.getsizeof` functions. + + +.. function:: get_traceback_limit() + + Get the maximum number of frames stored in the traceback of a trace. + + By default, a trace of an allocated memory block only stores the most recent + frame: the limit is ``1``. + + Use the :func:`set_traceback_limit` function to change the limit. + + +.. function:: get_traces() + + Get traces of memory blocks allocated by Python. Return a list of ``(size: + int, traceback: tuple)`` tuples. *traceback* is a tuple of ``(filename: str, + lineno: int)`` tuples. + + The list of traces do not include memory blocks allocated before the + :mod:`tracemalloc` module started to trace memory allocations nor memory + blocks ignored by filters (see :func:`get_filters`). + + The list is not sorted. Take a snapshot using :func:`take_snapshot` and use + the :meth:`Snapshot.statistics` method to get a sorted list of statistics. + + Tracebacks of traces are limited to :attr:`traceback_limit` frames. + Use :func:`set_traceback_limit` to store more frames. + + Return an empty list if the :mod:`tracemalloc` module is not tracing memory + allocations. + + See also :func:`take_snapshot` and :func:`get_object_traceback` functions. + + +.. function:: set_traceback_limit(nframe: int) + + Set the maximum number of frames stored in the traceback of a trace. + + Storing the traceback of each memory allocation has an important overhead on + the memory usage. Use the :func:`get_tracemalloc_memory` function to measure + the overhead and the :func:`add_filter` function to select which memory + allocations are traced. + + If the limit is set to ``0`` frame, the traceback ``(('', 0),)`` + will be used for all traces. + + Use the :func:`get_traceback_limit` function to get the current limit. + + The :envvar:`PYTHONTRACEMALLOC` environment variable and the :option:`-X` + ``tracemalloc=NFRAME`` command line option can be used to set a limit at + startup. + + +Filter functions +---------------- + +Tracing all Python memroy allocations has an important overhead on performances +and on the memory usage. + +To limit the overhead, some files can be excluded or tracing can be restricted +to a set of files using filters. Examples: ``add_filter(Filter(True, +subprocess.__file__))`` only traces memory allocations in the :mod:`subprocess` +module, and ``add_filter(Filter(False, tracemalloc.__file__))`` do not trace +memory allocations in the :mod:`tracemalloc` module + +By default, there is one exclusive filter to ignore Python memory blocks +allocated by the :mod:`tracemalloc` module. + +Use the :func:`get_tracemalloc_memory` function to measure the memory usage. +See also the :func:`set_traceback_limit` function to configure how many +frames are stored. + +.. function:: add_filter(filter) + + Add a new filter on Python memory allocations, *filter* is a :class:`Filter` + instance. + + All inclusive filters are applied at once, a memory allocation is ignored if + no inclusive filters match its trace. A memory allocation is ignored if at + least one exclusive filter matchs its trace. + + The new filter is not applied on already collected traces. Use the + :func:`clear_traces` function to ensure that all traces match the new + filter. + + +.. function:: clear_filters() + + Clear the filter list. + + See also the :func:`get_filters` function. + + +.. function:: get_filters() + + Get the filters on Python memory allocations. + Return a list of :class:`Filter` instances. + + See also the :func:`clear_filters` function. + + +Filter +------ + +.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False) + + Filter to select which memory allocations are traced. Filters can be used to + reduce the memory usage of the :mod:`tracemalloc` module, which can be read + using the :func:`get_tracemalloc_memory` function. + + The ``'*'`` joker character can be used in *filename_pattern* to match any + substring, including empty string. The ``'.pyc'`` and ``'.pyo'`` file + extensions are replaced with ``'.py'``. On Windows, the comparison is case + insensitive and the alternative separator ``'/'`` is replaced with the + standard separator ``'\'``. + + Use ``Filter(False, "")`` to exclude empty tracebacks. + + .. attribute:: inclusive + + If *inclusive* is ``True`` (include), only trace memory blocks allocated + in a file with a name matching :attr:`filename_pattern` at line number + :attr:`lineno`. + + If *inclusive* is ``False`` (exclude), ignore memory blocks allocated in + a file with a name matching :attr:`filename_pattern` at line number + :attr:`lineno`. + + .. attribute:: lineno + + Line number (``int``) of the filter. If *lineno* is ``None``, the filter + matches any line number. + + .. attribute:: filename_pattern + + Filename pattern (``str``) of the filter. + + .. attribute:: all_frames + + If *all_frames* is ``True``, all frames of the traceback are checked. If + *all_frames* is ``False``, only the most recent frame is checked. + + This attribute is ignored if the traceback limit is less than ``2``. + See the :func:`get_traceback_limit` function. + + +Snapshot +-------- + +.. class:: Snapshot(timestamp: datetime.datetime, traceback_limit: int, traces: dict=None) + + Snapshot of traces of memory blocks allocated by Python. + + The :func:`take_snapshot` function create a snapshot instance. + + .. method:: apply_filters(filters) + + Apply filters on the :attr:`traces` dictionary, *filters* is a list of + :class:`Filter` instances. Return a new :class:`Snapshot` instance with + the filtered traces. + + If *filters* is an empty list, return a new :class:`Snapshot` instance + with a copy of the traces. + + + .. method:: dump(filename) + + Write the snapshot into a file. + + Use :meth:`load` to reload the snapshot. + + + .. classmethod:: load(filename) + + Load a snapshot from a file. + + See also :meth:`dump`. + + + .. method:: statistics(key_type: str, cumulative: bool=False, compare_to=None) + + Get statistics as a sorted list of :class:`Statistic` instances, grouped + by *key_type*: + + ===================== ======================== ================================================ + key_type description type + ===================== ======================== ================================================ + ``'filename'`` filename ``str`` + ``'lineno'`` filename and line number ``(filename: str, lineno: int)`` + ``'traceback'`` traceback tuple of ``(filename: str, lineno: int)`` tuples + ===================== ======================== ================================================ + + If *cumulative* is ``True``, cumulate size and count of memory blocks of + all frames of the traceback of a trace, not only the most recent frame. + The cumulative mode can only be used with key types ``'filename'`` and + ``'lineno'`` with :attr:`traceback_limit` greater than ``1``. + + If *compare_to* is set to a previous :class:`Snapshot` instance, compute + the differences betwen the two snapshots. Otherwise, + :attr:`Statistic.size_diff` and :attr:`Statistic.count_diff` attributes + are set to zero. + + The result is sorted from the biggest to the smallest by: absolute value + of :attr:`Statistic.size_diff`, :attr:`Statistic.size`, absolute value of + :attr:`Statistic.count_diff`, :attr:`Statistic.count` and then by + :attr:`Statistic.key`. + + + .. attribute:: traceback_limit + + Maximum number of frames stored in the traceback of :attr:`traces`: + see the :func:`get_traceback_limit` function. + + .. attribute:: traces + + Traces of all memory blocks allocated by Python: see the + :func:`get_traces` function. + + .. attribute:: timestamp + + Creation date and time of the snapshot, :class:`datetime.datetime` + instance. + + +Statistic +--------- + +.. class:: Statistic(key, size, size_diff, count, count_diff) + + Statistic on memory allocations. + + :attr:`size_diff` and :attr:`count_diff` attributes are the difference + between two :class:`Snapshot` instance. + + :func:`Snapshot.statistics` returns a list of :class:`Statistic` instances. + + .. attribute:: key + + Key identifying the statistic. The key type depends on the *key_type* + parameter of the :meth:`Snapshot.statistics` method. + + .. attribute:: count + + Number of memory blocks (``int``). + + .. attribute:: count_diff + + Difference of number of memory blocks (``int``). + + .. attribute:: size + + Total size of memory blocks in bytes (``int``). + + .. attribute:: size_diff + + Difference of total size of memory blocks in bytes (``int``). + diff -r 2ed8d500e113 -r 65e72bf01246 Doc/license.rst --- a/Doc/license.rst Sun Nov 03 13:53:12 2013 +0100 +++ b/Doc/license.rst Sun Nov 03 14:25:07 2013 +0100 @@ -864,3 +864,44 @@ used for the build:: Jean-loup Gailly Mark Adler jloup@gzip.org madler@alumni.caltech.edu + +cfuhash +------- + +The implementtation of the hash table used by the :mod:`tracemalloc` is based +on the cfuhash project:: + + Copyright (c) 2005 Don Owens + All rights reserved. + + This code is released under the BSD license: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + diff -r 2ed8d500e113 -r 65e72bf01246 Doc/using/cmdline.rst --- a/Doc/using/cmdline.rst Sun Nov 03 13:53:12 2013 +0100 +++ b/Doc/using/cmdline.rst Sun Nov 03 14:25:07 2013 +0100 @@ -376,11 +376,15 @@ Miscellaneous options .. cmdoption:: -X Reserved for various implementation-specific options. CPython currently - defines two possible values: + defines the following possible values: * ``-X faulthandler`` to enable :mod:`faulthandler`; * ``-X showrefcount`` to enable the output of the total reference count and memory blocks (only works on debug builds); + * ``-X tracemalloc`` to enable :mod:`tracemalloc`. + * ``-X tracemalloc=NFRAME`` to enable :mod:`tracemalloc`, *NFRAME* is the + maximum number of frames stored in a trace: see the + :func:`tracemalloc.set_traceback_limit` function. It also allows to pass arbitrary values and retrieve them through the :data:`sys._xoptions` dictionary. @@ -392,7 +396,7 @@ Miscellaneous options The ``-X faulthandler`` option. .. versionadded:: 3.4 - The ``-X showrefcount`` option. + The ``-X showrefcount`` and ``-X tracemalloc`` options. Options you shouldn't use @@ -594,6 +598,16 @@ conflict. .. versionadded:: 3.3 +.. envvar:: PYTHONTRACEMALLOC + + If this environment variable is set to a non-empty string, all memory + allocations made by Python are traced by the :mod:`tracemalloc` module. + The value of the variable is the maximum number of frames stored in a trace: + see the :func:`tracemalloc.set_traceback_limit` function. + + .. versionadded:: 3.4 + + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff -r 2ed8d500e113 -r 65e72bf01246 Include/hashtable.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Include/hashtable.h Sun Nov 03 14:25:07 2013 +0100 @@ -0,0 +1,127 @@ +#ifndef Py_HASHTABLE_H +#define Py_HASHTABLE_H + +/* The whole API is private */ +#ifndef Py_LIMITED_API + +typedef struct _Py_slist_item_s { + struct _Py_slist_item_s *next; +} _Py_slist_item_t; + +typedef struct { + _Py_slist_item_t *head; +} _Py_slist_t; + +#define SLIST_ITEM_NEXT(ITEM) (((_Py_slist_item_t *)ITEM)->next) + +#define SLIST_HEAD(SLIST) (((_Py_slist_t *)SLIST)->head) + +typedef struct { + /* used by _Py_hashtable_t.buckets to link entries */ + _Py_slist_item_t _Py_slist_item; + + const void *key; + Py_uhash_t key_hash; + + /* data folllows */ +} hashtable_entry_t; + +#define _PY_HASHTABLE_ENTRY_DATA(ENTRY) \ + ((char *)(ENTRY) + sizeof(hashtable_entry_t)) + +#define _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(ENTRY) \ + (*(void **)_PY_HASHTABLE_ENTRY_DATA(ENTRY)) + +#define _Py_HASHTABLE_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + memcpy(DATA, _PY_HASHTABLE_ENTRY_DATA(ENTRY), DATA_SIZE); \ + } while (0) + +typedef Py_uhash_t (*hashtable_hash_func) (const void *key); +typedef int (*hashtable_compare_func) (const void *key, hashtable_entry_t *he); +typedef void* (*hashtable_copy_data_func)(void *data); +typedef void (*hashtable_free_data_func)(void *data); +typedef size_t (*hashtable_get_data_size_func)(void *data); + +typedef struct { + /* allocate a memory block */ + void* (*malloc) (size_t size); + + /* release a memory block */ + void (*free) (void *ptr); +} _Py_hashtable_allocator_t; + +typedef struct { + size_t num_buckets; + size_t entries; /* Total number of entries in the table. */ + _Py_slist_t *buckets; + size_t data_size; + + hashtable_hash_func hash_func; + hashtable_compare_func compare_func; + hashtable_copy_data_func copy_data_func; + hashtable_free_data_func free_data_func; + hashtable_get_data_size_func get_data_size_func; + _Py_hashtable_allocator_t alloc; +} _Py_hashtable_t; + +/* hash and compare functions for integers and pointers */ +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_ptr(const void *key); +PyAPI_FUNC(Py_uhash_t) _Py_hashtable_hash_int(const void *key); +PyAPI_FUNC(int) _Py_hashtable_compare_direct(const void *key, hashtable_entry_t *entry); + +PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new( + size_t data_size, + hashtable_hash_func hash_func, + hashtable_compare_func compare_func); +PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_new_full( + size_t data_size, + size_t init_size, + hashtable_hash_func hash_func, + hashtable_compare_func compare_func, + hashtable_copy_data_func copy_data_func, + hashtable_free_data_func free_data_func, + hashtable_get_data_size_func get_data_size_func, + _Py_hashtable_allocator_t *allocator); +PyAPI_FUNC(_Py_hashtable_t *) _Py_hashtable_copy(_Py_hashtable_t *src); +PyAPI_FUNC(void) _Py_hashtable_clear(_Py_hashtable_t *ht); +PyAPI_FUNC(void) _Py_hashtable_destroy(_Py_hashtable_t *ht); + +PyAPI_FUNC(int) _Py_hashtable_foreach( + _Py_hashtable_t *ht, + int (*fe_fn) (hashtable_entry_t *entry, void *arg), + void *arg); +PyAPI_FUNC(size_t) _Py_hashtable_size(_Py_hashtable_t *ht); + +PyAPI_FUNC(hashtable_entry_t*) _Py_hashtable_get_entry( + _Py_hashtable_t *ht, + const void *key); +PyAPI_FUNC(int) _Py_hashtable_set( + _Py_hashtable_t *ht, + const void *key, + void *data, + size_t data_size); +PyAPI_FUNC(int) _Py_hashtable_get( + _Py_hashtable_t *ht, + const void *key, + void *data, + size_t data_size); +PyAPI_FUNC(int) _Py_hashtable_pop( + _Py_hashtable_t *ht, + const void *key, + void *data, + size_t data_size); +PyAPI_FUNC(void) _Py_hashtable_delete( + _Py_hashtable_t *ht, + const void *key); + +#define HASHTABLE_SET(TABLE, KEY, DATA) \ + _Py_hashtable_set(TABLE, KEY, &(DATA), sizeof(DATA)) + +#define HASHTABLE_GET(TABLE, KEY, DATA) \ + _Py_hashtable_get(TABLE, KEY, &(DATA), sizeof(DATA)) + +#endif /* Py_LIMITED_API */ + +#endif diff -r 2ed8d500e113 -r 65e72bf01246 Include/pymacro.h --- a/Include/pymacro.h Sun Nov 03 13:53:12 2013 +0100 +++ b/Include/pymacro.h Sun Nov 03 14:25:07 2013 +0100 @@ -4,6 +4,9 @@ #define Py_MIN(x, y) (((x) > (y)) ? (y) : (x)) #define Py_MAX(x, y) (((x) > (y)) ? (x) : (y)) +#define Py_INT_TO_POINTER(value) ((void*)(Py_uintptr_t)(int)(value)) +#define Py_POINTER_TO_INT(ptr) ((int)(Py_uintptr_t)(ptr)) + /* Argument must be a char or an int in [-128, 127] or [0, 255]. */ #define Py_CHARMASK(c) ((unsigned char)((c) & 0xff)) diff -r 2ed8d500e113 -r 65e72bf01246 Lib/test/support/__init__.py --- a/Lib/test/support/__init__.py Sun Nov 03 13:53:12 2013 +0100 +++ b/Lib/test/support/__init__.py Sun Nov 03 14:25:07 2013 +0100 @@ -2116,3 +2116,22 @@ def patch(test_instance, object_to_patch # actually override the attribute setattr(object_to_patch, attr_name, new_value) + + +def run_in_subinterp(code): + """ + Run code in a subinterpreter. Raise unittest.SkipTest if the tracemalloc + module is enabled. + """ + # Issue #10915, #15751: PyGILState_*() functions don't work with + # sub-interpreters, the tracemalloc module uses these functions internally + try: + import tracemalloc + except ImportError: + pass + else: + if tracemalloc.is_enabled(): + raise unittest.SkipTest("run_in_subinterp() cannot be used " + "if tracemalloc module is enabled") + return _testcapi.run_in_subinterp(code) + diff -r 2ed8d500e113 -r 65e72bf01246 Lib/test/test_atexit.py --- a/Lib/test/test_atexit.py Sun Nov 03 13:53:12 2013 +0100 +++ b/Lib/test/test_atexit.py Sun Nov 03 14:25:07 2013 +0100 @@ -158,7 +158,7 @@ class SubinterpreterTest(unittest.TestCa atexit.register(f) del atexit """ - ret = _testcapi.run_in_subinterp(code) + ret = support.run_in_subinterp(code) self.assertEqual(ret, 0) self.assertEqual(atexit._ncallbacks(), n) @@ -173,7 +173,7 @@ class SubinterpreterTest(unittest.TestCa atexit.register(f) atexit.__atexit = atexit """ - ret = _testcapi.run_in_subinterp(code) + ret = support.run_in_subinterp(code) self.assertEqual(ret, 0) self.assertEqual(atexit._ncallbacks(), n) diff -r 2ed8d500e113 -r 65e72bf01246 Lib/test/test_capi.py --- a/Lib/test/test_capi.py Sun Nov 03 13:53:12 2013 +0100 +++ b/Lib/test/test_capi.py Sun Nov 03 14:25:07 2013 +0100 @@ -206,7 +206,7 @@ class SubinterpreterTest(unittest.TestCa pickle.dump(id(builtins), f) """.format(w) with open(r, "rb") as f: - ret = _testcapi.run_in_subinterp(code) + ret = support.run_in_subinterp(code) self.assertEqual(ret, 0) self.assertNotEqual(pickle.load(f), id(sys.modules)) self.assertNotEqual(pickle.load(f), id(builtins)) diff -r 2ed8d500e113 -r 65e72bf01246 Lib/test/test_threading.py --- a/Lib/test/test_threading.py Sun Nov 03 13:53:12 2013 +0100 +++ b/Lib/test/test_threading.py Sun Nov 03 14:25:07 2013 +0100 @@ -803,7 +803,7 @@ class SubinterpThreadingTests(BaseTestCa os.write(%d, b"x") threading.Thread(target=f).start() """ % (w,) - ret = _testcapi.run_in_subinterp(code) + ret = test.support.run_in_subinterp(code) self.assertEqual(ret, 0) # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") @@ -835,7 +835,7 @@ class SubinterpThreadingTests(BaseTestCa os.write(%d, b"x") threading.Thread(target=f).start() """ % (w,) - ret = _testcapi.run_in_subinterp(code) + ret = test.support.run_in_subinterp(code) self.assertEqual(ret, 0) # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") diff -r 2ed8d500e113 -r 65e72bf01246 Lib/test/test_tracemalloc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_tracemalloc.py Sun Nov 03 14:25:07 2013 +0100 @@ -0,0 +1,840 @@ +import _tracemalloc +import contextlib +import datetime +import os +import sys +import tracemalloc +import unittest +from unittest.mock import patch +from test.script_helper import assert_python_ok +from test import support +try: + import threading +except ImportError: + threading = None + +EMPTY_STRING_SIZE = sys.getsizeof(b'') + +def get_frames(nframe, lineno_delta): + frames = [] + frame = sys._getframe(1) + for index in range(nframe): + code = frame.f_code + lineno = frame.f_lineno + lineno_delta + frames.append((code.co_filename, lineno)) + lineno_delta = 0 + frame = frame.f_back + if frame is None: + break + return tuple(frames) + +def allocate_bytes(size): + nframe = tracemalloc.get_traceback_limit() + bytes_len = (size - EMPTY_STRING_SIZE) + frames = get_frames(nframe, 1) + data = b'x' * bytes_len + return data, frames + +def create_snapshots(): + traceback_limit = 2 + + timestamp = datetime.datetime(2013, 9, 12, 15, 16, 17) + traces = [ + (10, (('a.py', 2), ('b.py', 4))), + (10, (('a.py', 2), ('b.py', 4))), + (10, (('a.py', 2), ('b.py', 4))), + + (2, (('a.py', 5), ('b.py', 4))), + + (66, (('b.py', 1),)), + + (7, (('', 0),)), + ] + snapshot = tracemalloc.Snapshot(timestamp, traceback_limit, traces) + + timestamp2 = datetime.datetime(2013, 9, 12, 15, 16, 50) + traces2 = [ + (10, (('a.py', 2), ('b.py', 4))), + (10, (('a.py', 2), ('b.py', 4))), + (10, (('a.py', 2), ('b.py', 4))), + + (2, (('a.py', 5), ('b.py', 4))), + (5000, (('a.py', 5), ('b.py', 4))), + + (400, (('c.py', 578),)), + ] + snapshot2 = tracemalloc.Snapshot(timestamp2, traceback_limit, traces2) + + return (snapshot, snapshot2) + + +class TestTracemallocEnabled(unittest.TestCase): + def setUp(self): + if tracemalloc.is_tracing(): + self.skipTest("tracemalloc must be stopped before the test") + + tracemalloc.clear_filters() + tracemalloc.add_filter(tracemalloc.Filter(False, tracemalloc.__file__)) + tracemalloc.set_traceback_limit(1) + tracemalloc.start() + + def tearDown(self): + tracemalloc.stop() + tracemalloc.clear_filters() + + def test_get_tracemalloc_memory(self): + data = [allocate_bytes(123) for count in range(1000)] + size = tracemalloc.get_tracemalloc_memory() + self.assertGreaterEqual(size, 0) + + tracemalloc.clear_traces() + size2 = tracemalloc.get_tracemalloc_memory() + self.assertGreaterEqual(size2, 0) + self.assertLessEqual(size2, size) + + def test_get_object_traceback(self): + tracemalloc.clear_traces() + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) + traceback = tracemalloc.get_object_traceback(obj) + self.assertEqual(traceback, obj_frames) + + def test_set_traceback_limit(self): + obj_size = 10 + + nframe = tracemalloc.get_traceback_limit() + self.addCleanup(tracemalloc.set_traceback_limit, nframe) + + self.assertRaises(ValueError, tracemalloc.set_traceback_limit, -1) + + tracemalloc.clear_traces() + tracemalloc.set_traceback_limit(0) + obj, obj_frames = allocate_bytes(obj_size) + traceback = tracemalloc.get_object_traceback(obj) + self.assertEqual(len(traceback), 1) + self.assertEqual(traceback, (("", 0),)) + + tracemalloc.clear_traces() + tracemalloc.set_traceback_limit(1) + obj, obj_frames = allocate_bytes(obj_size) + traceback = tracemalloc.get_object_traceback(obj) + self.assertEqual(len(traceback), 1) + self.assertEqual(traceback, obj_frames) + + tracemalloc.clear_traces() + tracemalloc.set_traceback_limit(10) + obj2, obj2_frames = allocate_bytes(obj_size) + traceback = tracemalloc.get_object_traceback(obj2) + self.assertEqual(len(traceback), 10) + self.assertEqual(traceback, obj2_frames) + + def find_trace(self, traces, traceback): + for trace in traces: + if trace[1] == traceback: + return trace + + self.fail("trace not found") + + def test_get_traces(self): + tracemalloc.clear_traces() + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) + + traces = tracemalloc.get_traces() + trace = self.find_trace(traces, obj_frames) + + self.assertIsInstance(trace, tuple) + size, traceback = trace + self.assertEqual(size, obj_size) + self.assertEqual(traceback, obj_frames) + + def test_get_traces_intern_traceback(self): + # dummy wrappers to get more useful and identical frames in the traceback + def allocate_bytes2(size): + return allocate_bytes(size) + def allocate_bytes3(size): + return allocate_bytes2(size) + def allocate_bytes4(size): + return allocate_bytes3(size) + + # Ensure that two identical tracebacks are not duplicated + tracemalloc.clear_traces() + tracemalloc.set_traceback_limit(4) + obj_size = 123 + obj1, obj1_frames = allocate_bytes4(obj_size) + obj2, obj2_frames = allocate_bytes4(obj_size) + + traces = tracemalloc.get_traces() + + trace1 = self.find_trace(traces, obj1_frames) + trace2 = self.find_trace(traces, obj2_frames) + size1, traceback1 = trace1 + size2, traceback2 = trace2 + self.assertEqual(traceback2, traceback1) + self.assertIs(traceback2, traceback1) + + def test_get_traced_memory(self): + # get the allocation location to filter allocations + size = 12345 + obj, frames = allocate_bytes(size) + filename, lineno = frames[0] + tracemalloc.add_filter(tracemalloc.Filter(True, filename, lineno)) + + # allocate one object + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + self.assertEqual(tracemalloc.get_traced_memory(), (size, size)) + + # destroy the object + obj = None + self.assertEqual(tracemalloc.get_traced_memory(), (0, size)) + + # clear_traces() must reset traced memory counters + tracemalloc.clear_traces() + self.assertEqual(tracemalloc.get_traced_memory(), (0, 0)) + + # allocate another object + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + self.assertEqual(tracemalloc.get_traced_memory(), (size, size)) + + # stop() rests also traced memory counters + tracemalloc.stop() + self.assertEqual(tracemalloc.get_traced_memory(), (0, 0)) + + def test_clear_traces(self): + obj, obj_frames = allocate_bytes(123) + traceback = tracemalloc.get_object_traceback(obj) + self.assertIsNotNone(traceback) + + tracemalloc.clear_traces() + traceback2 = tracemalloc.get_object_traceback(obj) + self.assertIsNone(traceback2) + + def test_is_tracing(self): + tracemalloc.stop() + self.assertFalse(tracemalloc.is_tracing()) + + tracemalloc.start() + self.assertTrue(tracemalloc.is_tracing()) + + def test_snapshot(self): + obj, source = allocate_bytes(123) + + # take a snapshot + snapshot = tracemalloc.take_snapshot() + + # write on disk + snapshot.dump(support.TESTFN) + self.addCleanup(support.unlink, support.TESTFN) + + # load from disk + snapshot2 = tracemalloc.Snapshot.load(support.TESTFN) + self.assertEqual(snapshot2.timestamp, snapshot.timestamp) + self.assertEqual(snapshot2.traces, snapshot.traces) + + # tracemalloc must be tracing memory allocations to take a snapshot + tracemalloc.stop() + with self.assertRaises(RuntimeError) as cm: + tracemalloc.take_snapshot() + self.assertEqual(str(cm.exception), + "the tracemalloc module must be tracing memory " + "allocations to take a snapshot") + + def test_filters(self): + tracemalloc.clear_filters() + tracemalloc.add_filter(tracemalloc.Filter(False, tracemalloc.__file__)) + # test multiple inclusive filters + tracemalloc.add_filter(tracemalloc.Filter(True, 'should never match 1')) + tracemalloc.add_filter(tracemalloc.Filter(True, 'should never match 2')) + tracemalloc.add_filter(tracemalloc.Filter(True, __file__)) + + tracemalloc.clear_traces() + size = 1000 + obj, obj_frames = allocate_bytes(size) + traceback = tracemalloc.get_object_traceback(obj) + self.assertIsNotNone(traceback) + + # test exclusive filter, based on previous filters + filename, lineno = obj_frames[0] + tracemalloc.add_filter(tracemalloc.Filter(False, filename, lineno)) + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + traceback = tracemalloc.get_object_traceback(obj) + self.assertIsNone(traceback) + + def fork_child(self): + if not tracemalloc.is_tracing(): + return 2 + + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) + traceback = tracemalloc.get_object_traceback(obj) + if traceback is None: + return 3 + + # everything is fine + return 0 + + @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork()') + def test_fork(self): + # check that tracemalloc is still working after fork + pid = os.fork() + if not pid: + # child + exitcode = 1 + try: + exitcode = self.fork_child() + finally: + os._exit(exitcode) + else: + pid2, status = os.waitpid(pid, 0) + self.assertTrue(os.WIFEXITED(status)) + exitcode = os.WEXITSTATUS(status) + self.assertEqual(exitcode, 0) + + +class TestSnapshot(unittest.TestCase): + maxDiff = 1000 + + def test_create_snapshot(self): + stats = {'a.py': {1: (5, 1)}} + traces = {0x123: (5, ('a.py', 1))} + + with contextlib.ExitStack() as stack: + stack.enter_context(patch.object(tracemalloc, 'is_tracing', return_value=True)) + stack.enter_context(patch.object(tracemalloc, 'get_traceback_limit', return_value=5)) + stack.enter_context(patch.object(tracemalloc, 'get_traces', return_value=traces)) + + snapshot = tracemalloc.take_snapshot() + self.assertIsInstance(snapshot.timestamp, datetime.datetime) + self.assertEqual(snapshot.traceback_limit, 5) + self.assertEqual(snapshot.traces, traces) + + def test_apply_filters(self): + snapshot, snapshot2 = create_snapshots() + filter1 = tracemalloc.Filter(False, "b.py") + filter2 = tracemalloc.Filter(True, "a.py", 2) + filter3 = tracemalloc.Filter(True, "a.py", 5) + + original_traces = snapshot.traces.copy() + + # exclude b.py + snapshot3 = snapshot.apply_filters((filter1,)) + self.assertEqual(snapshot3.traces, [ + (10, (('a.py', 2), ('b.py', 4))), + (10, (('a.py', 2), ('b.py', 4))), + (10, (('a.py', 2), ('b.py', 4))), + (2, (('a.py', 5), ('b.py', 4))), + (7, (('', 0),)), + ]) + + # apply_filters() must not touch the original snapshot + self.assertEqual(snapshot.traces, original_traces) + + # only include two lines of a.py + snapshot4 = snapshot3.apply_filters((filter2, filter3)) + self.assertEqual(snapshot4.traces, [ + (10, (('a.py', 2), ('b.py', 4))), + (10, (('a.py', 2), ('b.py', 4))), + (10, (('a.py', 2), ('b.py', 4))), + (2, (('a.py', 5), ('b.py', 4))), + ]) + + # No filter: just duplicate the snapshot + snapshot5 = snapshot.apply_filters(()) + self.assertIsNot(snapshot5, snapshot) + self.assertIsNot(snapshot5.traces, snapshot.traces) + self.assertEqual(snapshot5.traces, snapshot.traces) + + def test_snapshot_group_by_line(self): + snapshot, snapshot2 = create_snapshots() + + # stats per file and line + unknown = ('', 0) + stats1 = snapshot.statistics('lineno') + self.assertEqual(stats1, [ + tracemalloc.Statistic(('b.py', 1), 66, 0, 1, 0), + tracemalloc.Statistic(('a.py', 2), 30, 0, 3, 0), + tracemalloc.Statistic(unknown, 7, 0, 1, 0), + tracemalloc.Statistic(('a.py', 5), 2, 0, 1, 0), + ]) + + # stats per file and line (2) + stats2 = snapshot2.statistics('lineno') + self.assertEqual(stats2, [ + tracemalloc.Statistic(('a.py', 5), 5002, 0, 2, 0), + tracemalloc.Statistic(('c.py', 578), 400, 0, 1, 0), + tracemalloc.Statistic(('a.py', 2), 30, 0, 3, 0), + ]) + + # stats diff per file and line + statistics = snapshot2.statistics('lineno', compare_to=snapshot) + self.assertEqual(statistics, [ + tracemalloc.Statistic(('a.py', 5), 5002, 5000, 2, 1), + tracemalloc.Statistic(('c.py', 578), 400, 400, 1, 1), + tracemalloc.Statistic(('b.py', 1), 0, -66, 0, -1), + tracemalloc.Statistic(('', 0), 0, -7, 0, -1), + tracemalloc.Statistic(('a.py', 2), 30, 0, 3, 0), + ]) + + def test_snapshot_group_by_file(self): + snapshot, snapshot2 = create_snapshots() + + # stats per file + stats1 = snapshot.statistics('filename') + self.assertEqual(stats1, [ + tracemalloc.Statistic('b.py', 66, 0, 1, 0), + tracemalloc.Statistic('a.py', 32, 0, 4, 0), + tracemalloc.Statistic('', 7, 0, 1, 0), + ]) + + # stats per file (2) + stats2 = snapshot2.statistics('filename') + self.assertEqual(stats2, [ + tracemalloc.Statistic('a.py', 5032, 0, 5, 0), + tracemalloc.Statistic('c.py', 400, 0, 1, 0), + ]) + + # stats diff per file + diff = snapshot2.statistics('filename', compare_to=snapshot) + self.assertEqual(diff, [ + tracemalloc.Statistic('a.py', 5032, 5000, 5, 1), + tracemalloc.Statistic('c.py', 400, 400, 1, 1), + tracemalloc.Statistic('b.py', 0, -66, 0, -1), + tracemalloc.Statistic('', 0, -7, 0, -1), + ]) + + def test_snapshot_group_by_traceback(self): + snapshot, snapshot2 = create_snapshots() + + # stats per file + tb1 = (('a.py', 2), ('b.py', 4)) + tb2 = (('a.py', 5), ('b.py', 4)) + tb3 = (('b.py', 1),) + tb4 = (('', 0),) + stats1 = snapshot.statistics('traceback') + self.assertEqual(stats1, [ + tracemalloc.Statistic(tb3, 66, 0, 1, 0), + tracemalloc.Statistic(tb1, 30, 0, 3, 0), + tracemalloc.Statistic(tb4, 7, 0, 1, 0), + tracemalloc.Statistic(tb2, 2, 0, 1, 0), + ]) + + # stats per file (2) + tb5 = (('c.py', 578),) + stats2 = snapshot2.statistics('traceback') + self.assertEqual(stats2, [ + tracemalloc.Statistic(tb2, 5002, 0, 2, 0), + tracemalloc.Statistic(tb5, 400, 0, 1, 0), + tracemalloc.Statistic(tb1, 30, 0, 3, 0), + ]) + + # stats diff per file + diff = snapshot2.statistics('traceback', compare_to=snapshot) + self.assertEqual(diff, [ + tracemalloc.Statistic((('a.py', 5), ('b.py', 4)), 5002, 5000, 2, 1), + tracemalloc.Statistic((('c.py', 578),), 400, 400, 1, 1), + tracemalloc.Statistic((('b.py', 1),), 0, -66, 0, -1), + tracemalloc.Statistic((('', 0),), 0, -7, 0, -1), + tracemalloc.Statistic((('a.py', 2), ('b.py', 4)), 30, 0, 3, 0), + ]) + + self.assertRaises(ValueError, + snapshot.statistics, 'traceback', cumulative=True) + + def test_snapshot_group_by_cumulative(self): + snapshot, snapshot2 = create_snapshots() + + # per file + stats = snapshot.statistics('filename', True) + self.assertEqual(stats, [ + tracemalloc.Statistic('b.py', 98, 0, 5, 0), + tracemalloc.Statistic('a.py', 32, 0, 4, 0), + tracemalloc.Statistic('', 7, 0, 1, 0), + ]) + + # per line + stats = snapshot.statistics('lineno', True) + self.assertEqual(stats, [ + tracemalloc.Statistic(('b.py', 1), 66, 0, 1, 0), + tracemalloc.Statistic(('b.py', 4), 32, 0, 4, 0), + tracemalloc.Statistic(('a.py', 2), 30, 0, 3, 0), + tracemalloc.Statistic(('', 0), 7, 0, 1, 0), + tracemalloc.Statistic(('a.py', 5), 2, 0, 1, 0), + ]) + + +class TestFilters(unittest.TestCase): + maxDiff = 2048 + def test_add_clear_filter(self): + old_filters = tracemalloc.get_filters() + try: + # test add_filter() + tracemalloc.clear_filters() + tracemalloc.add_filter(tracemalloc.Filter(True, "abc", 3)) + tracemalloc.add_filter(tracemalloc.Filter(False, "12345", 0)) + tracemalloc.add_filter(tracemalloc.Filter(False, "6789", None)) + tracemalloc.add_filter(tracemalloc.Filter(False, "def#", 55)) + tracemalloc.add_filter(tracemalloc.Filter(False, "trace", 123, True)) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abc', 3, False), + tracemalloc.Filter(False, '12345', 0, False), + tracemalloc.Filter(False, '6789', None, False), + tracemalloc.Filter(False, "def#", 55, False), + tracemalloc.Filter(False, "trace", 123, True)]) + + # test filename normalization (.pyc/.pyo) + tracemalloc.clear_filters() + tracemalloc.add_filter(tracemalloc.Filter(True, "abc.pyc")) + tracemalloc.add_filter(tracemalloc.Filter(True, "name.pyo")) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abc.py', None, False), + tracemalloc.Filter(True, 'name.py', None, False) ]) + + # test filename normalization ('*' joker character) + tracemalloc.clear_filters() + tracemalloc.add_filter(tracemalloc.Filter(True, 'a****b')) + tracemalloc.add_filter(tracemalloc.Filter(True, '***x****')) + tracemalloc.add_filter(tracemalloc.Filter(True, '1*2**3***4')) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'a*b', None, False), + tracemalloc.Filter(True, '*x*', None, False), + tracemalloc.Filter(True, '1*2*3*4', None, False)]) + + # ignore duplicated filters + tracemalloc.clear_filters() + tracemalloc.add_filter(tracemalloc.Filter(True, 'a.py')) + tracemalloc.add_filter(tracemalloc.Filter(True, 'a.py', 5)) + tracemalloc.add_filter(tracemalloc.Filter(True, 'a.py')) + tracemalloc.add_filter(tracemalloc.Filter(True, 'a.py', 5)) + tracemalloc.add_filter(tracemalloc.Filter(False, 'b.py')) + tracemalloc.add_filter(tracemalloc.Filter(False, 'b.py', 10)) + tracemalloc.add_filter(tracemalloc.Filter(False, 'b.py')) + tracemalloc.add_filter(tracemalloc.Filter(False, 'b.py', 10, True)) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'a.py', None, False), + tracemalloc.Filter(True, 'a.py', 5, False), + tracemalloc.Filter(False, 'b.py', None, False), + tracemalloc.Filter(False, 'b.py', 10, False), + tracemalloc.Filter(False, 'b.py', 10, True)]) + + # Windows: test filename normalization (lower case, slash) + if os.name == "nt": + tracemalloc.clear_filters() + tracemalloc.add_filter(tracemalloc.Filter(True, "aBcD\xC9")) + tracemalloc.add_filter(tracemalloc.Filter(True, "MODule.PYc")) + tracemalloc.add_filter(tracemalloc.Filter(True, r"path/to\file")) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abcd\xe9', None, False), + tracemalloc.Filter(True, 'module.py', None, False), + tracemalloc.Filter(True, r'path\to\file', None, False)]) + + # test clear_filters() + tracemalloc.clear_filters() + self.assertEqual(tracemalloc.get_filters(), []) + finally: + tracemalloc.clear_filters() + for trace_filter in old_filters: + tracemalloc.add_filter(trace_filter) + + def test_filter_attributes(self): + # test default values + f = tracemalloc.Filter(True, "abc") + self.assertEqual(f.inclusive, True) + self.assertEqual(f.filename_pattern, "abc") + self.assertIsNone(f.lineno) + self.assertEqual(f.all_frames, False) + + # test custom values + f = tracemalloc.Filter(False, "test.py", 123, True) + self.assertEqual(f.inclusive, False) + self.assertEqual(f.filename_pattern, "test.py") + self.assertEqual(f.lineno, 123) + self.assertEqual(f.all_frames, True) + + # parameters passed by keyword + f = tracemalloc.Filter(inclusive=False, filename_pattern="test.py", lineno=123, all_frames=True) + self.assertEqual(f.inclusive, False) + self.assertEqual(f.filename_pattern, "test.py") + self.assertEqual(f.lineno, 123) + self.assertEqual(f.all_frames, True) + + # attributes are read-only + self.assertRaises(AttributeError, setattr, f, "inclusive", True) + self.assertRaises(AttributeError, setattr, f, "filename_pattern", "abc") + self.assertRaises(AttributeError, setattr, f, "lineno", 5) + self.assertRaises(AttributeError, setattr, f, "all_frames", ()) + + def test_filter_match(self): + # filter without line number + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f._match("abc", 0)) + self.assertTrue(f._match("abc", 5)) + self.assertTrue(f._match("abc", 10)) + self.assertFalse(f._match("12356", 0)) + self.assertFalse(f._match("12356", 5)) + self.assertFalse(f._match("12356", 10)) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f._match("abc", 0)) + self.assertFalse(f._match("abc", 5)) + self.assertFalse(f._match("abc", 10)) + self.assertTrue(f._match("12356", 0)) + self.assertTrue(f._match("12356", 5)) + self.assertTrue(f._match("12356", 10)) + + # filter with line number > 0 + f = tracemalloc.Filter(True, "abc", 5) + self.assertFalse(f._match("abc", 0)) + self.assertTrue(f._match("abc", 5)) + self.assertFalse(f._match("abc", 10)) + self.assertFalse(f._match("12356", 0)) + self.assertFalse(f._match("12356", 5)) + self.assertFalse(f._match("12356", 10)) + + f = tracemalloc.Filter(False, "abc", 5) + self.assertTrue(f._match("abc", 0)) + self.assertFalse(f._match("abc", 5)) + self.assertTrue(f._match("abc", 10)) + self.assertTrue(f._match("12356", 0)) + self.assertTrue(f._match("12356", 5)) + self.assertTrue(f._match("12356", 10)) + + # filter with line number 0 + f = tracemalloc.Filter(True, "abc", 0) + self.assertTrue(f._match("abc", 0)) + self.assertFalse(f._match("abc", 5)) + self.assertFalse(f._match("abc", 10)) + self.assertFalse(f._match("12356", 0)) + self.assertFalse(f._match("12356", 5)) + self.assertFalse(f._match("12356", 10)) + + f = tracemalloc.Filter(False, "abc", 0) + self.assertFalse(f._match("abc", 0)) + self.assertTrue(f._match("abc", 5)) + self.assertTrue(f._match("abc", 10)) + self.assertTrue(f._match("12356", 0)) + self.assertTrue(f._match("12356", 5)) + self.assertTrue(f._match("12356", 10)) + + def test_filter_match_filename(self): + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f._match_filename("abc")) + self.assertFalse(f._match_filename("12356")) + self.assertFalse(f._match_filename("")) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f._match_filename("abc")) + self.assertTrue(f._match_filename("12356")) + self.assertTrue(f._match_filename("")) + + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f._match_filename("abc")) + self.assertFalse(f._match_filename("12356")) + self.assertFalse(f._match_filename("")) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f._match_filename("abc")) + self.assertTrue(f._match_filename("12356")) + self.assertTrue(f._match_filename("")) + + def test_filter_match_filename_joker(self): + def fnmatch(filename, pattern): + filter = tracemalloc.Filter(True, pattern) + return filter._match_filename(filename) + + # empty string + self.assertFalse(fnmatch('abc', '')) + self.assertFalse(fnmatch('', 'abc')) + self.assertTrue(fnmatch('', '')) + self.assertTrue(fnmatch('', '*')) + + # no * + self.assertTrue(fnmatch('abc', 'abc')) + self.assertFalse(fnmatch('abc', 'abcd')) + self.assertFalse(fnmatch('abc', 'def')) + + # a* + self.assertTrue(fnmatch('abc', 'a*')) + self.assertTrue(fnmatch('abc', 'abc*')) + self.assertFalse(fnmatch('abc', 'b*')) + self.assertFalse(fnmatch('abc', 'abcd*')) + + # a*b + self.assertTrue(fnmatch('abc', 'a*c')) + self.assertTrue(fnmatch('abcdcx', 'a*cx')) + self.assertFalse(fnmatch('abb', 'a*c')) + self.assertFalse(fnmatch('abcdce', 'a*cx')) + + # a*b*c + self.assertTrue(fnmatch('abcde', 'a*c*e')) + self.assertTrue(fnmatch('abcbdefeg', 'a*bd*eg')) + self.assertFalse(fnmatch('abcdd', 'a*c*e')) + self.assertFalse(fnmatch('abcbdefef', 'a*bd*eg')) + + # replace .pyc and .pyo suffix with .py + self.assertTrue(fnmatch('a.pyc', 'a.py')) + self.assertTrue(fnmatch('a.pyo', 'a.py')) + self.assertTrue(fnmatch('a.py', 'a.pyc')) + self.assertTrue(fnmatch('a.py', 'a.pyo')) + + if os.name == 'nt': + # case insensitive + self.assertTrue(fnmatch('aBC', 'ABc')) + self.assertTrue(fnmatch('aBcDe', 'Ab*dE')) + + self.assertTrue(fnmatch('a.pyc', 'a.PY')) + self.assertTrue(fnmatch('a.PYO', 'a.py')) + self.assertTrue(fnmatch('a.py', 'a.PYC')) + self.assertTrue(fnmatch('a.PY', 'a.pyo')) + else: + # case sensitive + self.assertFalse(fnmatch('aBC', 'ABc')) + self.assertFalse(fnmatch('aBcDe', 'Ab*dE')) + + self.assertFalse(fnmatch('a.pyc', 'a.PY')) + self.assertFalse(fnmatch('a.PYO', 'a.py')) + self.assertFalse(fnmatch('a.py', 'a.PYC')) + self.assertFalse(fnmatch('a.PY', 'a.pyo')) + + if os.name == 'nt': + # normalize alternate separator "/" to the standard separator "\" + self.assertTrue(fnmatch(r'a/b', r'a\b')) + self.assertTrue(fnmatch(r'a\b', r'a/b')) + self.assertTrue(fnmatch(r'a/b\c', r'a\b/c')) + self.assertTrue(fnmatch(r'a/b/c', r'a\b\c')) + else: + # there is no alternate separator + self.assertFalse(fnmatch(r'a/b', r'a\b')) + self.assertFalse(fnmatch(r'a\b', r'a/b')) + self.assertFalse(fnmatch(r'a/b\c', r'a\b/c')) + self.assertFalse(fnmatch(r'a/b/c', r'a\b\c')) + + # a******b + N = 10 ** 6 + self.assertTrue (fnmatch('a' * N, '*' * N)) + self.assertTrue (fnmatch('a' * N + 'c', '*' * N)) + self.assertTrue (fnmatch('a' * N, 'a' + '*' * N + 'a')) + self.assertTrue (fnmatch('a' * N + 'b', 'a' + '*' * N + 'b')) + self.assertFalse(fnmatch('a' * N + 'b', 'a' + '*' * N + 'c')) + + # a*a*a*a* + self.assertTrue(fnmatch('a' * 10, 'a*' * 10)) + self.assertFalse(fnmatch('a' * 10, 'a*' * 10 + 'b')) + with self.assertRaises(ValueError) as cm: + fnmatch('abc', 'a*' * 101) + self.assertEqual(str(cm.exception), + "too many joker characters in the filename pattern") + + def test_filter_match_trace(self): + t1 = (("a.py", 2), ("b.py", 3)) + t2 = (("b.py", 4), ("b.py", 5)) + t3 = (("c.py", 5), ('', 0)) + empty = () + + f = tracemalloc.Filter(True, "b.py", all_frames=True) + self.assertTrue(f._match_traceback(t1)) + self.assertTrue(f._match_traceback(t2)) + self.assertFalse(f._match_traceback(t3)) + self.assertFalse(f._match_traceback(empty)) + + f = tracemalloc.Filter(True, "b.py", all_frames=False) + self.assertFalse(f._match_traceback(t1)) + self.assertTrue(f._match_traceback(t2)) + self.assertFalse(f._match_traceback(t3)) + self.assertFalse(f._match_traceback(empty)) + + f = tracemalloc.Filter(False, "b.py", all_frames=True) + self.assertFalse(f._match_traceback(t1)) + self.assertFalse(f._match_traceback(t2)) + self.assertTrue(f._match_traceback(t3)) + self.assertTrue(f._match_traceback(empty)) + + f = tracemalloc.Filter(False, "b.py", all_frames=False) + self.assertTrue(f._match_traceback(t1)) + self.assertFalse(f._match_traceback(t2)) + self.assertTrue(f._match_traceback(t3)) + self.assertTrue(f._match_traceback(empty)) + + f = tracemalloc.Filter(False, "", all_frames=False) + self.assertTrue(f._match_traceback(t1)) + self.assertTrue(f._match_traceback(t2)) + self.assertTrue(f._match_traceback(t3)) + self.assertFalse(f._match_traceback(empty)) + + f = tracemalloc.Filter(True, "", all_frames=True) + self.assertFalse(f._match_traceback(t1)) + self.assertFalse(f._match_traceback(t2)) + self.assertTrue(f._match_traceback(t3)) + self.assertTrue(f._match_traceback(empty)) + + f = tracemalloc.Filter(False, "", all_frames=True) + self.assertTrue(f._match_traceback(t1)) + self.assertTrue(f._match_traceback(t2)) + self.assertFalse(f._match_traceback(t3)) + self.assertFalse(f._match_traceback(empty)) + + def test_invalid_parameters(self): + self.assertRaises(TypeError, tracemalloc.Filter, False, None, 123) + self.assertRaises(TypeError, tracemalloc.Filter, False, "abc", 1.2) + + f = tracemalloc.Filter(False, "abc.py") + self.assertRaises(TypeError, f._match, None, 123) + self.assertRaises(TypeError, f._match, "x.py", None) + self.assertRaises(TypeError, f._match_filename, None) + self.assertRaises(TypeError, f._match_traceback, ((None, 5),)) + self.assertRaises(TypeError, f._match_traceback, (("x.py", None),)) + + +class TestCommandLine(unittest.TestCase): + def test_env_var(self): + # not tracing by default + code = 'import tracemalloc; print(tracemalloc.is_tracing())' + ok, stdout, stderr = assert_python_ok('-c', code) + stdout = stdout.rstrip() + self.assertEqual(stdout, b'False') + + # PYTHON* environment varibles must be ignored when -E option is + # present + code = 'import tracemalloc; print(tracemalloc.is_tracing())' + ok, stdout, stderr = assert_python_ok('-E', '-c', code, PYTHONTRACEMALLOC='1') + stdout = stdout.rstrip() + self.assertEqual(stdout, b'False') + + # tracing at startup + code = 'import tracemalloc; print(tracemalloc.is_tracing())' + ok, stdout, stderr = assert_python_ok('-c', code, PYTHONTRACEMALLOC='1') + stdout = stdout.rstrip() + self.assertEqual(stdout, b'True') + + def test_env_var_nframe(self): + code = 'import tracemalloc; print(tracemalloc.get_traceback_limit())' + ok, stdout, stderr = assert_python_ok('-c', code, PYTHONTRACEMALLOC='10') + stdout = stdout.rstrip() + self.assertEqual(stdout, b'10') + + def test_sys_xoptions_nframe(self): + for xoptions, nframe in ( + ('tracemalloc', 1), + ('tracemalloc=0', 0), + ('tracemalloc=1', 1), + ('tracemalloc=15', 15), + ): + with self.subTest(xoptions=xoptions, nframe=nframe): + code = 'import tracemalloc; print(tracemalloc.get_traceback_limit())' + ok, stdout, stderr = assert_python_ok('-X', xoptions, '-c', code) + stdout = stdout.rstrip() + self.assertEqual(stdout, str(nframe).encode('ascii')) + + +def test_main(): + support.run_unittest( + TestTracemallocEnabled, + TestSnapshot, + TestFilters, + TestCommandLine, + ) + +if __name__ == "__main__": + test_main() diff -r 2ed8d500e113 -r 65e72bf01246 Lib/tracemalloc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/tracemalloc.py Sun Nov 03 14:25:07 2013 +0100 @@ -0,0 +1,225 @@ +from __future__ import with_statement +import _tracemalloc +import datetime +import gc +import os +import pickle +import sys + +# Import types and functions implemented in C +from _tracemalloc import * + +class Statistic: + """ + Statistic on memory allocations. + """ + + __slots__ = ('key', 'size', 'size_diff', 'count', 'count_diff') + + def __init__(self, key, size, size_diff, count, count_diff): + self.key = key + self.size = size + self.size_diff = size_diff + self.count = count + self.count_diff = count_diff + + def _sort_diff_key(self): + return (abs(self.size_diff), self.size, + abs(self.count_diff), self.count, + self.key) + + def _sort_key(self): + return (self.size, self.count, self.key) + + def __eq__(self, other): + return (self.key == other.key + and self.size == other.size + and self.size_diff == other.size_diff + and self.count == other.count + and self.count_diff == other.count_diff) + + def __repr__(self): + return ('' + % (self.key, self.size, self.size_diff, + self.count, self.count_diff)) + + +def _compare_grouped_stats(old_group, new_group): + statistics = [] + for key, stat in new_group.items(): + previous = old_group.pop(key, None) + if previous is not None: + stat = Statistic(key, + stat.size, stat.size - previous.size, + stat.count, stat.count - previous.count) + else: + stat = Statistic(key, + stat.size, stat.size, + stat.count, stat.count) + statistics.append(stat) + + for key, stat in old_group.items(): + stat = Statistic(key, 0, -stat.size, 0, -stat.count) + statistics.append(stat) + return statistics + +class Snapshot: + """ + Snapshot of traces of memory blocks allocated by Python. + """ + + __slots__ = ('timestamp', 'traceback_limit', 'traces') + + def __init__(self, timestamp, traceback_limit, traces): + self.timestamp = timestamp + self.traceback_limit = traceback_limit + self.traces = traces + + def dump(self, filename): + """ + Write the snapshot into a file. + """ + with open(filename, "wb") as fp: + pickle.dump(self, fp, pickle.HIGHEST_PROTOCOL) + + @staticmethod + def load(filename): + """ + Load a snapshot from a file. + """ + with open(filename, "rb") as fp: + return pickle.load(fp) + + def _filter_trace(self, include_filters, exclude_filters, trace): + traceback = trace[1] + if include_filters: + if not any(trace_filter._match_traceback(traceback) + for trace_filter in include_filters): + return False + if exclude_filters: + if any(not trace_filter._match_traceback(traceback) + for trace_filter in exclude_filters): + return False + return True + + def apply_filters(self, filters): + """ + Apply filters on the traces dictionary, filters is a list of Filter + instances. + """ + if filters: + include_filters = [] + exclude_filters = [] + for trace_filter in filters: + if trace_filter.inclusive: + include_filters.append(trace_filter) + else: + exclude_filters.append(trace_filter) + new_traces = [trace for trace in self.traces + if self._filter_trace(include_filters, exclude_filters, trace)] + else: + new_traces = self.traces.copy() + return Snapshot(self.timestamp, self.traceback_limit, new_traces) + + def _group_by(self, key_type, cumulative): + if key_type not in ('traceback', 'filename', 'lineno'): + raise ValueError("unknown key_type: %r" % (key_type,)) + if cumulative and key_type not in ('lineno', 'filename'): + raise ValueError("cumulative mode cannot by used " + "with key type %r" % key_type) + if cumulative and self.traceback_limit < 2: + raise ValueError("cumulative mode needs tracebacks with at least " + "2 frames, traceback limit is %s" + % self.traceback_limit) + + stats = {} + if key_type == 'traceback': + # statistics by traceback + for trace in self.traces: + size, traceback = trace + try: + stat = stats[traceback] + stat.size += size + stat.count += 1 + except KeyError: + stats[traceback] = Statistic(traceback, size, 0, 1, 0) + + elif not cumulative and key_type == 'lineno': + # statistics by line number + for trace in self.traces: + size, traceback = trace + frame = traceback[0] + try: + stat = stats[frame] + stat.size += size + stat.count += 1 + except KeyError: + stats[frame] = Statistic(frame, size, 0, 1, 0) + + elif not cumulative and key_type == 'filename': + # statistics by filename + for trace in self.traces: + size, traceback = trace + filename = traceback[0][0] + try: + stat = stats[filename] + stat.size += size + stat.count += 1 + except KeyError: + stats[filename] = Statistic(filename, size, 0, 1, 0) + + elif key_type == 'lineno': + # cumulative statistics by line number + for trace in self.traces: + size, traceback = trace + for frame in traceback: + try: + stat = stats[frame] + stat.size += size + stat.count += 1 + except KeyError: + stats[frame] = Statistic(frame, size, 0, 1, 0) + + else: + # cumulative statistics by filename + for trace in self.traces: + size, traceback = trace + for frame in traceback: + filename = frame[0] + try: + stat = stats[filename] + stat.size += size + stat.count += 1 + except KeyError: + stats[filename] = Statistic(filename, size, 0, 1, 0) + + return stats + + def statistics(self, key_type, cumulative=False, compare_to=None): + """ + Group statistics by key_type. Return a sorted list of Statistic + instances. + """ + grouped = self._group_by(key_type, cumulative) + if compare_to is not None: + previous = compare_to._group_by(key_type, cumulative) + statistics = _compare_grouped_stats(previous, grouped) + statistics.sort(reverse=True, key=Statistic._sort_diff_key) + else: + statistics = list(grouped.values()) + statistics.sort(reverse=True, key=Statistic._sort_key) + return statistics + + +def take_snapshot(): + """ + Take a snapshot of traces of memory blocks allocated by Python. + """ + if not is_tracing(): + raise RuntimeError("the tracemalloc module must be tracing memory " + "allocations to take a snapshot") + timestamp = datetime.datetime.now() + traceback_limit = get_traceback_limit() + traces = get_traces() + return Snapshot(timestamp, traceback_limit, traces) + diff -r 2ed8d500e113 -r 65e72bf01246 Makefile.pre.in --- a/Makefile.pre.in Sun Nov 03 13:53:12 2013 +0100 +++ b/Makefile.pre.in Sun Nov 03 14:25:07 2013 +0100 @@ -356,6 +356,7 @@ PYTHON_OBJS= \ Python/getplatform.o \ Python/getversion.o \ Python/graminit.o \ + Python/hashtable.o \ Python/import.o \ Python/importdl.o \ Python/marshal.o \ diff -r 2ed8d500e113 -r 65e72bf01246 Modules/Setup.dist --- a/Modules/Setup.dist Sun Nov 03 13:53:12 2013 +0100 +++ b/Modules/Setup.dist Sun Nov 03 14:25:07 2013 +0100 @@ -132,6 +132,9 @@ zipimport zipimport.c # faulthandler module faulthandler faulthandler.c +# debug tool to trace memory blocks allocated by Python +_tracemalloc _tracemalloc.c + # The rest of the modules listed in this file are all commented out by # default. Usually they can be detected and built as dynamically # loaded modules by the new setup.py script added in Python 2.1. If diff -r 2ed8d500e113 -r 65e72bf01246 Modules/_tracemalloc.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Modules/_tracemalloc.c Sun Nov 03 14:25:07 2013 +0100 @@ -0,0 +1,2437 @@ +#include "Python.h" +#include "hashtable.h" +#include "frameobject.h" +#include "pythread.h" +#include "osdefs.h" + +/* Trace memory blocks allocated by PyMem_RawMalloc() */ +#define TRACE_RAW_MALLOC + +/* Forward declaration */ +static void tracemalloc_stop(void); +static int tracemalloc_atexit_register(void); +static void* raw_malloc(size_t size); +static void* raw_realloc(void *ptr, size_t size); +static void raw_free(void *ptr); + +#ifdef MS_WINDOWS +# define TRACE_CASE_INSENSITIVE +#endif + +#if defined(ALTSEP) || defined(TRACE_CASE_INSENSITIVE) +# define TRACE_NORMALIZE_FILENAME +#endif + +#ifdef Py_DEBUG +# define TRACE_DEBUG +#endif + +#define STR(VAL) #VAL + +/* Protected by the GIL */ +static struct { + PyMemAllocator mem; + PyMemAllocator raw; + PyMemAllocator obj; +} allocators; + +/* Arbitrary limit of the number of frames in a traceback. The value was chosen + to not allocate too much memory on the stack (see TRACEBACK_STACK_SIZE + below). */ +#define MAX_NFRAME 100 + +/* arbitrary limit of the depth of the recursive + match_filename_joker() function to avoid a stack overflow */ +#define MAX_NJOKER 100 + +/* Protected by the GIL */ +static PyObjectArenaAllocator arena_allocator; + +static struct { + /* Module initialized? + + 0: not initialized yet + 1: tracemalloc_init() has been called + 2: tracemalloc_deinit() has been called + + Variable protected by the GIL */ + int init; + + /* are the hooks installed in memory allocators? + Variable protected by the GIL */ + int installed; + + /* limit of the number of frames in a traceback, 1 by default. + Variable protected by the GIL. */ + int max_nframe; +} tracemalloc_config = {0, 0, 0, 1}; + +#ifdef TRACE_DEBUG +static void +tracemalloc_error(const char *format, ...) +{ + va_list ap; + fprintf(stderr, "tracemalloc: "); + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + fprintf(stderr, "\n"); + fflush(stderr); +} +#endif + +#if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) +#define REENTRANT_THREADLOCAL + +/* If your OS does not provide native thread local storage, you can implement + it manually using a lock. Functions of thread.c cannot be used because + they use PyMem_RawMalloc() which leads to a reentrant call. */ +#if !(defined(_POSIX_THREADS) || defined(NT_THREADS)) +# error "need native thread local storage (TLS)" +#endif + +static int tracemalloc_reentrant_key; + +static int +get_reentrant(void) +{ + void *ptr = PyThread_get_key_value(tracemalloc_reentrant_key); + if (ptr != NULL) { + assert(Py_POINTER_TO_INT(ptr) == 1); + return 1; + } + else + return 0; +} + +static void +set_reentrant(int reentrant) +{ + /* FIXME: PyThread_set_key_value(tracemalloc_reentrant_key, ptr) cannot + be used directly because PyThread_set_key_value() does nothing + if the thread has already a value. */ + if (reentrant) { + assert(PyThread_get_key_value(tracemalloc_reentrant_key) == NULL); + PyThread_set_key_value(tracemalloc_reentrant_key, + Py_INT_TO_POINTER(1)); + } + else + PyThread_delete_key_value(tracemalloc_reentrant_key); +} + +#else + +/* WITH_THREAD not defined: Python compiled without threads, + or TRACE_RAW_MALLOC not defined: variable protected by the GIL */ +static int tracemalloc_reentrant = 0; + +#define get_reentrant() tracemalloc_reentrant +#define set_reentrant(reentrant) \ + do { \ + assert(!reentrant || !get_reentrant()); \ + tracemalloc_reentrant = reentrant; \ + } while (0) + +#endif + +static int +hashtable_compare_unicode(const void *key, hashtable_entry_t *entry) +{ + if (key != NULL && entry->key != NULL) + return (PyUnicode_Compare((PyObject *)key, (PyObject *)entry->key) == 0); + else + return key == entry->key; +} + +static _Py_hashtable_allocator_t hashtable_alloc = {malloc, free}; + +_Py_hashtable_t * +hashtable_new(size_t data_size, + hashtable_hash_func hash_func, + hashtable_compare_func compare_func) +{ + return _Py_hashtable_new_full(data_size, 0, + hash_func, compare_func, + NULL, NULL, NULL, &hashtable_alloc); +} + +typedef struct { + /* include (1) or exclude (0) matching frame? */ + int include; + Py_hash_t pattern_hash; + PyObject *filename_pattern; +#ifndef TRACE_NORMALIZE_FILENAME + int use_joker; +#endif + /* ignore any line number if lineno < 1 */ + int lineno; + /* check all frames or only the most recent frame? */ + int all_frames; +} filter_t; + +typedef struct { + size_t nfilter; + filter_t *filters; +} filter_list_t; + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) +/* This lock is needed because tracemalloc_free() is called without + the GIL held from PyMem_RawFree(). It cannot acquire the lock because it + would introduce a deadlock in PyThreadState_DeleteCurrent(). */ +static PyThread_type_lock tables_lock; +# define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1) +# define TABLES_UNLOCK() PyThread_release_lock(tables_lock) +#else + /* variables are protected by the GIL */ +# define TABLES_LOCK() +# define TABLES_UNLOCK() +#endif + +#ifdef WITH_THREAD +#endif + +/* Protected by the GIL */ +static filter_list_t tracemalloc_include_filters; +static filter_list_t tracemalloc_exclude_filters; + +#pragma pack(4) +typedef struct +#ifdef __GNUC__ +__attribute__((packed)) +#endif +{ + PyObject *filename; + int lineno; +} frame_t; + +typedef struct { + Py_uhash_t hash; + int nframe; + frame_t frames[1]; +} traceback_t; + +static PyObject *unknown_filename = NULL; +static traceback_t tracemalloc_empty_traceback; + +typedef struct { + size_t size; + traceback_t *traceback; +} trace_t; + +#define TRACEBACK_SIZE(NFRAME) \ + (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1)) +#define TRACEBACK_STACK_SIZE TRACEBACK_SIZE(MAX_NFRAME) + +typedef struct { + size_t size; + size_t count; +} trace_stats_t; + +/* Size of Currently traced memory. + Protected by TABLES_LOCK(). */ +static size_t tracemalloc_traced_memory = 0; + +/* Maximum size of traced memory. + Protected by TABLES_LOCK(). */ +static size_t tracemalloc_max_traced_memory = 0; + +/* Hash table used to intern filenames. + Protected by the GIL */ +static _Py_hashtable_t *tracemalloc_filenames = NULL; + +/* Hash table used to intern tracebacks. + Protected by the GIL */ +static _Py_hashtable_t *tracemalloc_tracebacks = NULL; + +/* pointer (void*) => trace (trace_t). + Protected by TABLES_LOCK(). */ +static _Py_hashtable_t *tracemalloc_traces = NULL; + +/* Forward declaration */ +static int tracemalloc_add_filter(filter_t *filter); +static int traceback_match_filters(traceback_t *traceback); + +static void* +raw_malloc(size_t size) +{ + return allocators.raw.malloc(allocators.raw.ctx, size); +} + +static void* +raw_realloc(void *ptr, size_t size) +{ + return allocators.raw.realloc(allocators.raw.ctx, ptr, size); +} + +static void +raw_free(void *ptr) +{ + allocators.raw.free(allocators.raw.ctx, ptr); +} + +static Py_uhash_t +hashtable_hash_traceback(const void *key) +{ + const traceback_t *traceback = key; + return traceback->hash; +} + +static int +hashtable_compare_traceback(const traceback_t *traceback1, hashtable_entry_t *he) +{ + const traceback_t *traceback2 = he->key; + const frame_t *frame1, *frame2; + Py_hash_t hash1, hash2; + int i; + + if (traceback1->nframe != traceback2->nframe) + return 0; + + for (i=0; i < traceback1->nframe; i++) { + frame1 = &traceback1->frames[i]; + frame2 = &traceback2->frames[i]; + + if (frame1->lineno != frame2->lineno) + return 0; + + if (frame1->filename != frame2->filename) { + hash1 = PyObject_Hash(frame1->filename); + hash2 = PyObject_Hash(frame2->filename); + if (hash1 != hash2) + return 0; + + if (PyUnicode_Compare(frame1->filename, frame2->filename) != 0) + return 0; + } + } + return 1; +} + +static void +tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) +{ + PyCodeObject *code; + PyObject *filename; + hashtable_entry_t *entry; + + frame->filename = unknown_filename; + frame->lineno = PyFrame_GetLineNumber(pyframe); + assert(frame->lineno >= 0); + if (frame->lineno < 0) + frame->lineno = 0; + + code = pyframe->f_code; + if (code == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to get the code object of the a frame"); +#endif + return; + } + + if (code->co_filename == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to get the filename of the code object"); +#endif + return; + } + + filename = code->co_filename; + assert(filename != NULL); + if (filename == NULL) + return; + + if (!PyUnicode_CheckExact(filename)) { +#ifdef TRACE_DEBUG + tracemalloc_error("filename is not an unicode string"); +#endif + return; + } + if (!PyUnicode_IS_READY(filename)) { + /* Don't make a Unicode string ready to avoid reentrant calls + to tracemalloc_malloc() or tracemalloc_realloc() */ +#ifdef TRACE_DEBUG + tracemalloc_error("filename is not a ready unicode string"); +#endif + return; + } + + /* intern the filename */ + entry = _Py_hashtable_get_entry(tracemalloc_filenames, filename); + if (entry != NULL) { + filename = (PyObject *)entry->key; + } + else { + /* tracemalloc_filenames is responsible to keep a reference + to the filename */ + Py_INCREF(filename); + if (_Py_hashtable_set(tracemalloc_filenames, filename, NULL, 0) < 0) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to intern the filename"); +#endif + return; + } + } + + /* the tracemalloc_filenames table keeps a reference to the filename */ + frame->filename = filename; +} + +static Py_uhash_t +traceback_hash(traceback_t *traceback) +{ + /* code based on tuplehash() of Objects/tupleobject.c */ + Py_uhash_t x; /* Unsigned for defined overflow behavior. */ + Py_hash_t y; + int len = traceback->nframe; + Py_uhash_t mult = _PyHASH_MULTIPLIER; + frame_t *frame; + + x = 0x345678UL; + frame = traceback->frames; + while (--len >= 0) { + y = PyObject_Hash(frame->filename); + y ^= frame->lineno; + frame++; + + x = (x ^ y) * mult; + /* the cast might truncate len; that doesn't change hash stability */ + mult += (Py_hash_t)(82520UL + len + len); + } + x += 97531UL; + return x; +} + +static void +traceback_get_frames(traceback_t *traceback) +{ + PyThreadState *tstate; + PyFrameObject *pyframe; + +#ifdef WITH_THREAD + tstate = PyGILState_GetThisThreadState(); +#else + tstate = PyThreadState_Get(); +#endif + if (tstate == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to get the current thread state"); +#endif + return; + } + + for (pyframe = tstate->frame; pyframe != NULL; pyframe = pyframe->f_back) { + tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]); + assert(traceback->frames[traceback->nframe].filename != NULL); + assert(traceback->frames[traceback->nframe].lineno >= 0); + traceback->nframe++; + if (traceback->nframe == tracemalloc_config.max_nframe) + break; + } +} + +static traceback_t * +traceback_new(void) +{ + char stack_buffer[TRACEBACK_STACK_SIZE]; + traceback_t *traceback = (traceback_t *)stack_buffer; + hashtable_entry_t *entry; + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + + traceback->nframe = 0; + if (tracemalloc_config.max_nframe > 0) + traceback_get_frames(traceback); + + if (traceback->nframe == 0) { + traceback = &tracemalloc_empty_traceback; + if (!traceback_match_filters(traceback)) + return NULL; + return traceback; + } + + traceback->hash = traceback_hash(traceback); + + if (!traceback_match_filters(traceback)) + return NULL; + + /* intern the traceback */ + entry = _Py_hashtable_get_entry(tracemalloc_tracebacks, traceback); + if (entry != NULL) { + traceback = (traceback_t *)entry->key; + } + else { + traceback_t *copy; + size_t traceback_size; + + traceback_size = TRACEBACK_SIZE(traceback->nframe); + + copy = raw_malloc(traceback_size); + if (copy == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to intern the traceback: malloc failed"); +#endif + return NULL; + } + memcpy(copy, traceback, traceback_size); + + /* tracemalloc_tracebacks is responsible to keep a reference + to the traceback */ + if (_Py_hashtable_set(tracemalloc_tracebacks, copy, NULL, 0) < 0) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to intern the traceback: putdata failed"); +#endif + return NULL; + } + traceback = copy; + } + return traceback; +} + +#ifdef TRACE_NORMALIZE_FILENAME +static Py_UCS4 +tracemalloc_normalize_filename(Py_UCS4 ch) +{ +#ifdef ALTSEP + if (ch == ALTSEP) + return SEP; +#endif +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + return ch; +} +#endif + +typedef struct { + PyObject *filename, *filename_pattern; + int file_kind, pat_kind; + void *file_data, *pat_data; + Py_ssize_t file_len, pat_len; +} match_t; + +static int +match_filename_joker(match_t *match, Py_ssize_t file_pos, Py_ssize_t pat_pos) +{ + Py_UCS4 ch1, ch2; + + assert(pat_pos >= 0); + + while (file_pos >= 0 && pat_pos >= 0) { + ch1 = PyUnicode_READ(match->file_kind, match->file_data, file_pos); + ch2 = PyUnicode_READ(match->pat_kind, match->pat_data, pat_pos); + if (ch2 == '*') { + int found; + + pat_pos--; + if (pat_pos < 0) { + /* 'abc' always match '*' */ + return 1; + } + + do { + found = match_filename_joker(match, file_pos, pat_pos); + if (found) + break; + file_pos--; + } while (file_pos >= 0); + + return found; + } + +#ifdef TRACE_NORMALIZE_FILENAME + ch1 = tracemalloc_normalize_filename(ch1); +#endif + if (ch1 != ch2) + return 0; + + file_pos--; + pat_pos--; + } + + if (pat_pos == 0) { + ch2 = PyUnicode_READ(match->pat_kind, match->pat_data, pat_pos); + if (ch2 == '*') { + /* 'abc' matchs '*abc' */ + return 1; + } + } + else if (pat_pos > 0) { + return 0; + } + + if (file_pos >= 0) + return 0; + + return 1; +} + +static int +filename_endswith_pyc_pyo(PyObject *filename) +{ + void* data; + int kind; + Py_UCS4 ch; + Py_ssize_t len; + + len = PyUnicode_GetLength(filename); + if (len < 4) + return 0; + + data = PyUnicode_DATA(filename); + kind = PyUnicode_KIND(filename); + + ch = PyUnicode_READ(kind, data, len-1); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if ((ch != 'c' && ch != 'o')) + return 0; + + ch = PyUnicode_READ(kind, data, len-2); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if (ch != 'y') + return 0; + + ch = PyUnicode_READ(kind, data, len-3); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if (ch != 'p') + return 0; + + if (PyUnicode_READ(kind, data, len-4) != '.') + return 0; + return 1; +} + +static int +match_filename(filter_t *filter, PyObject *filename) +{ + Py_ssize_t len; + match_t match; + Py_hash_t hash; + + assert(filename != NULL); + + assert(PyUnicode_IS_READY(filename)); + + if (filename == filter->filename_pattern) + return 1; + + hash = PyObject_Hash(filename); + if (hash == filter->pattern_hash) { + if (PyUnicode_Compare(filename, filter->filename_pattern) == 0) + return 1; + } + +#ifndef TRACE_NORMALIZE_FILENAME + if (!filter->use_joker) { + if (!filename_endswith_pyc_pyo(filename)) { + /* hash is different: strings are different */ + return 0; + } + else { + len = PyUnicode_GetLength(filename); + + /* don't compare last character */ + return PyUnicode_Tailmatch(filename, filter->filename_pattern, + 0, len - 1, 1); + } + } +#endif + + len = PyUnicode_GetLength(filename); + + /* replace "a.pyc" and "a.pyo" with "a.py" */ + if (filename_endswith_pyc_pyo(filename)) + len--; + + match.filename = filename; + match.file_kind = PyUnicode_KIND(match.filename); + match.file_data = PyUnicode_DATA(match.filename); + match.file_len = len; + + match.filename_pattern = filter->filename_pattern; + match.pat_kind = PyUnicode_KIND(match.filename_pattern); + match.pat_data = PyUnicode_DATA(match.filename_pattern); + match.pat_len = PyUnicode_GET_LENGTH(match.filename_pattern); + + if (match.pat_len != 0) + return match_filename_joker(&match, + match.file_len - 1, match.pat_len - 1); + else + return (match.file_len == 0); +} + +static int +filter_match_filename(filter_t *filter, PyObject *filename) +{ + int match; + + if (filename == NULL) + return !filter->include; + + match = match_filename(filter, filename); + return match ^ !filter->include; +} + +static int +filter_match(filter_t *filter, PyObject *filename, int lineno) +{ + int match; + + match = match_filename(filter, filename); + if (filter->include) { + if (!match) + return 0; + } + else { + /* exclude */ + if (!match) + return 1; + else if (filter->lineno < 0) + return 0; + } + + if (filter->lineno < 0) + return 1; + + match = (lineno == filter->lineno); + return match ^ !filter->include; +} + +static int +filter_match_traceback(filter_t *filter, traceback_t *traceback) +{ + int i; + PyObject *filename; + int lineno; + int nframe; + int match; + + nframe = traceback->nframe; + if (nframe == 0) { + return filter_match(filter, unknown_filename, 0); + } + else if (!filter->all_frames) { + filename = traceback->frames[0].filename; + lineno = traceback->frames[0].lineno; + return filter_match(filter, filename, lineno); + } + else { + for (i = 0; i < nframe; i++) { + filename = traceback->frames[i].filename; + lineno = traceback->frames[i].lineno; + + match = filter_match(filter, filename, lineno); + if (match) { + if (filter->include) + return 1; + } + else { + if (!filter->include) + return 0; + } + } + return !filter->include; + } +} + +static int +filter_list_match(filter_list_t *filters, int include, traceback_t *traceback) +{ + size_t i; + filter_t *filter; + int match; + + if (filters->nfilter == 0) + return 1; + + for (i = 0; i < filters->nfilter; i++) { + filter = &filters->filters[i]; + + match = filter_match_traceback(filter, traceback); + if (include) { + if (match) + return 1; + } + else { + if (!match) + return 0; + } + } + return !include; +} + +static int +traceback_match_filters(traceback_t *traceback) +{ + if (!filter_list_match(&tracemalloc_include_filters, 1, traceback)) + return 0; + if (!filter_list_match(&tracemalloc_exclude_filters, 0, traceback)) + return 0; + return 1; +} + +static void +tracemalloc_log_alloc(void *ptr, size_t size) +{ + traceback_t *traceback; + trace_t trace; + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + + traceback = traceback_new(); + if (traceback == NULL) + return; + + trace.size = size; + trace.traceback = traceback; + + TABLES_LOCK(); + assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); + tracemalloc_traced_memory += size; + if (tracemalloc_traced_memory > tracemalloc_max_traced_memory) + tracemalloc_max_traced_memory = tracemalloc_traced_memory; + + HASHTABLE_SET(tracemalloc_traces, ptr, trace); + TABLES_UNLOCK(); +} + +static void +tracemalloc_log_free(void *ptr) +{ + trace_t trace; + + TABLES_LOCK(); + if (_Py_hashtable_pop(tracemalloc_traces, ptr, &trace, sizeof(trace))) { + assert(tracemalloc_traced_memory >= trace.size); + tracemalloc_traced_memory -= trace.size; + } + TABLES_UNLOCK(); +} + +static void* +tracemalloc_malloc(void *ctx, size_t size, int gil_held) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) + PyGILState_STATE gil_state; +#endif + void *ptr; + + if (get_reentrant()) { + return alloc->malloc(alloc->ctx, size); + } + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) + if (!gil_held) { + /* PyGILState_Ensure() may call PyMem_RawMalloc() indirectly which + would call PyGILState_Ensure() if reentrant are not disabled. */ + set_reentrant(1); + gil_state = PyGILState_Ensure(); + + ptr = alloc->malloc(alloc->ctx, size); + set_reentrant(0); + } + else +#endif + { +#ifdef WITH_THREAD + assert(gil_held); +#endif + + /* Ignore reentrant call: PyObjet_Malloc() calls PyMem_Malloc() for + allocations larger than 512 bytes */ + set_reentrant(1); + ptr = alloc->malloc(alloc->ctx, size); + set_reentrant(0); + } + + if (ptr != NULL) + tracemalloc_log_alloc(ptr, size); + + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) + if (!gil_held) + PyGILState_Release(gil_state); +#endif + + return ptr; +} + +static void* +tracemalloc_realloc(void *ctx, void *ptr, size_t new_size, int gil_held) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) + PyGILState_STATE gil_state; +#endif + void *ptr2; + + if (get_reentrant()) { + /* Reentrant call to PyMem_Realloc() and PyMem_RawRealloc(). + Example: PyMem_RawRealloc() is called internally by pymalloc + (_PyObject_Malloc() and _PyObject_Realloc()) to allocate a new + arena (new_arena()). */ + ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); + + if (ptr2 != NULL && ptr != NULL) + tracemalloc_log_free(ptr); + + return ptr2; + } + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) + if (!gil_held) { + /* reentrant: PyGILState_Ensure() may call PyMem_RawMalloc() + indirectly which would call PyGILState_Ensure() if reentrant are not + disabled. */ + set_reentrant(1); + gil_state = PyGILState_Ensure(); + + ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); + set_reentrant(0); + } + else +#endif + { +#ifdef WITH_THREAD + assert(gil_held); +#endif + + /* reentrant: PyObjet_Realloc() calls PyMem_Realloc() for allocations + larger than 512 bytes */ + set_reentrant(1); + ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); + set_reentrant(0); + } + + if (ptr2 != NULL) { + if (ptr != NULL) + tracemalloc_log_free(ptr); + + tracemalloc_log_alloc(ptr2, new_size); + } + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) + if (!gil_held) + PyGILState_Release(gil_state); +#endif + + return ptr2; +} + +static void +tracemalloc_free(void *ctx, void *ptr) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; + + /* Cannot lock the GIL in PyMem_RawFree() because it would introduce + a deadlock in PyThreadState_DeleteCurrent(). */ + + if (ptr != NULL) { + alloc->free(alloc->ctx, ptr); + tracemalloc_log_free(ptr); + } +} + +static void* +tracemalloc_malloc_gil(void *ctx, size_t size) +{ + return tracemalloc_malloc(ctx, size, 1); +} + +static void* +tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) +{ + return tracemalloc_realloc(ctx, ptr, new_size, 1); +} + +#ifdef TRACE_RAW_MALLOC +static void* +tracemalloc_raw_malloc(void *ctx, size_t size) +{ + return tracemalloc_malloc(ctx, size, 0); +} + +static void* +tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) +{ + return tracemalloc_realloc(ctx, ptr, new_size, 0); +} +#endif + +static int +filter_init(filter_t *filter, + int include, PyObject *filename_pattern, int lineno, + int all_frames) +{ + Py_ssize_t len, len2; + Py_ssize_t i, j; + PyObject *new_filename_pattern; + Py_UCS4 maxchar, ch; + int kind, kind2; + void *data, *data2; + int previous_joker; + size_t njoker; + + if (!PyUnicode_Check(filename_pattern)) { + PyErr_Format(PyExc_TypeError, "filename_pattern must be a str, not %s", + Py_TYPE(filename_pattern)->tp_name); + return -1; + } + + if (PyUnicode_READY(filename_pattern) < 0) + return -1; + + len = PyUnicode_GetLength(filename_pattern); + kind = PyUnicode_KIND(filename_pattern); + data = PyUnicode_DATA(filename_pattern); + + if (filename_endswith_pyc_pyo(filename_pattern)) + len--; + + maxchar = 0; + len2 = 0; + njoker = 0; + previous_joker = 0; + for (i=0; i < len; i++) { + ch = PyUnicode_READ(kind, data, i); +#ifdef TRACE_NORMALIZE_FILENAME + ch = tracemalloc_normalize_filename(ch); +#endif + if (!previous_joker || ch != '*') { + previous_joker = (ch == '*'); + if (previous_joker) + njoker++; + maxchar = Py_MAX(maxchar, ch); + len2++; + } + else { + /* skip consecutive joker character */ + } + } + + if (njoker > MAX_NJOKER) { + PyErr_SetString(PyExc_ValueError, + "too many joker characters in the filename pattern"); + return -1; + } + + new_filename_pattern = PyUnicode_New(len2, maxchar); + if (new_filename_pattern == NULL) + return -1; + kind2 = PyUnicode_KIND(new_filename_pattern); + data2 = PyUnicode_DATA(new_filename_pattern); + + j = 0; + previous_joker = 0; + for (i=0; i < len; i++) { + ch = PyUnicode_READ(kind, data, i); +#ifdef TRACE_NORMALIZE_FILENAME + ch = tracemalloc_normalize_filename(ch); +#endif + if (!previous_joker || ch != '*') { + previous_joker = (ch == '*'); + PyUnicode_WRITE(kind2, data2, j, ch); + j++; + } + else { + /* skip consecutive joker character */ + } + } + assert(j == len2); + + assert(_PyUnicode_CheckConsistency(new_filename_pattern, 1)); + + filter->filename_pattern = new_filename_pattern; +#ifndef TRACE_NORMALIZE_FILENAME + filter->use_joker = (njoker != 0); +#endif + + filter->include = include; + filter->pattern_hash = PyObject_Hash(filter->filename_pattern); + if (lineno >= 0) + filter->lineno = lineno; + else + filter->lineno = -1; + filter->all_frames = all_frames; + return 0; +} + +static void +filter_deinit(filter_t *filter) +{ + Py_CLEAR(filter->filename_pattern); +} + +static void +filter_list_init(filter_list_t *filters) +{ + filters->nfilter = 0; + filters->filters = NULL; +} + +static void +filter_list_clear(filter_list_t *filters) +{ + size_t i; + + if (filters->nfilter == 0) { + assert(filters->filters == NULL); + return; + } + + for (i=0; infilter; i++) + filter_deinit(&filters->filters[i]); + + filters->nfilter = 0; + raw_free(filters->filters); + filters->filters = NULL; +} + +static void +tracemalloc_clear_filters(void) +{ + filter_list_clear(&tracemalloc_include_filters); + filter_list_clear(&tracemalloc_exclude_filters); +} + +static int +tracemalloc_clear_filename(hashtable_entry_t *entry, void *user_data) +{ + PyObject *filename = (PyObject *)entry->key; + Py_DECREF(filename); + return 0; +} + +static int +traceback_free_cb(hashtable_entry_t *entry, void *user_data) +{ + traceback_t *traceback = (traceback_t *)entry->key; + raw_free(traceback); + return 0; +} + +/* reentrant flag must be set to call this function and GIL must be held */ +static void +tracemalloc_clear_traces(void) +{ +#ifdef WITH_THREAD + /* The GIL protects variables againt concurrent access */ + assert(PyGILState_Check()); +#endif + + /* Disable also reentrant calls to tracemalloc_malloc() to not add a new + trace while we are clearing traces */ + assert(get_reentrant()); + + TABLES_LOCK(); + _Py_hashtable_clear(tracemalloc_traces); + tracemalloc_traced_memory = 0; + tracemalloc_max_traced_memory = 0; + TABLES_UNLOCK(); + + _Py_hashtable_foreach(tracemalloc_tracebacks, traceback_free_cb, NULL); + _Py_hashtable_clear(tracemalloc_tracebacks); + + _Py_hashtable_foreach(tracemalloc_filenames, tracemalloc_clear_filename, NULL); + _Py_hashtable_clear(tracemalloc_filenames); +} + +static int +tracemalloc_init(void) +{ + PyObject *filename; + filter_t filter; + + if (tracemalloc_config.init == 2) { + PyErr_SetString(PyExc_RuntimeError, + "the tracemalloc module has been unloaded"); + return -1; + } + + if (tracemalloc_config.init != 0) + return 0; + + /* ensure that the frame_t structure is packed */ + assert(sizeof(frame_t) == (sizeof(PyObject*) + sizeof(int))); + + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); + PyObject_GetArenaAllocator(&arena_allocator); + +#ifdef REENTRANT_THREADLOCAL + tracemalloc_reentrant_key = PyThread_create_key(); + if (tracemalloc_reentrant_key == -1) { +#ifdef MS_WINDOWS + PyErr_SetFromWindowsErr(0); +#else + PyErr_SetFromErrno(PyExc_OSError); +#endif + return -1; + } +#endif + + filter_list_init(&tracemalloc_include_filters); + filter_list_init(&tracemalloc_exclude_filters); + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) + if (tables_lock == NULL) { + tables_lock = PyThread_allocate_lock(); + if (tables_lock == NULL) { + PyErr_SetString(PyExc_RuntimeError, "cannot allocate lock"); + return -1; + } + } +#endif + + tracemalloc_filenames = hashtable_new(0, + (hashtable_hash_func)PyObject_Hash, + hashtable_compare_unicode); + + tracemalloc_tracebacks = hashtable_new(0, + (hashtable_hash_func)hashtable_hash_traceback, + (hashtable_compare_func)hashtable_compare_traceback); + + tracemalloc_traces = hashtable_new(sizeof(trace_t), + _Py_hashtable_hash_ptr, _Py_hashtable_compare_direct); + + if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL + || tracemalloc_traces == NULL) + { + PyErr_NoMemory(); + return -1; + } + + unknown_filename = PyUnicode_FromString(""); + if (unknown_filename == NULL) + return -1; + PyUnicode_InternInPlace(&unknown_filename); + + tracemalloc_empty_traceback.nframe = 1; + tracemalloc_empty_traceback.frames[0].filename = unknown_filename; + tracemalloc_empty_traceback.frames[0].lineno = 0; + tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback); + + /* disable tracing allocations until tracemalloc is fully initialized */ + set_reentrant(1); + + /* by default, exclude tracemalloc.py */ +#ifdef MS_WINDOWS + filename = PyUnicode_FromString("*\\tracemalloc.py"); +#else + filename = PyUnicode_FromString("*/tracemalloc.py"); +#endif + if (filename == NULL) + return -1; + if (filter_init(&filter, 0, filename, -1, 0) < 0) + return -1; + if (tracemalloc_add_filter(&filter) < 0) + return -1; + + tracemalloc_config.init = 1; + return 0; +} + +static void +tracemalloc_deinit(void) +{ + if (tracemalloc_config.init != 1) + return; + tracemalloc_config.init = 2; + + tracemalloc_stop(); + + tracemalloc_clear_filters(); + + /* destroy hash tables */ + _Py_hashtable_destroy(tracemalloc_traces); + _Py_hashtable_destroy(tracemalloc_tracebacks); + _Py_hashtable_destroy(tracemalloc_filenames); + +#if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) + if (tables_lock == NULL) { + PyThread_free_lock(tables_lock); + tables_lock = NULL; + } +#endif + +#ifdef REENTRANT_THREADLOCAL + PyThread_delete_key(tracemalloc_reentrant_key); +#endif + + Py_XDECREF(unknown_filename); +} + +static int +tracemalloc_start(void) +{ + PyMemAllocator alloc; + + if (tracemalloc_init() < 0) + return -1; + + if (tracemalloc_config.installed) { + /* hook already installed: do nothing */ + return 0; + } + + if (tracemalloc_atexit_register() < 0) + return -1; + + tracemalloc_traced_memory = 0; + tracemalloc_max_traced_memory = 0; + +#ifdef TRACE_RAW_MALLOC + alloc.malloc = tracemalloc_raw_malloc; + alloc.realloc = tracemalloc_raw_realloc; + alloc.free = tracemalloc_free; + + alloc.ctx = &allocators.raw; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); +#endif + + alloc.malloc = tracemalloc_malloc_gil; + alloc.realloc = tracemalloc_realloc_gil; + alloc.free = tracemalloc_free; + + alloc.ctx = &allocators.mem; + PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + + alloc.ctx = &allocators.obj; + PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + + /* every is ready: start tracing Python memory allocations */ + tracemalloc_config.installed = 1; + set_reentrant(0); + + return 0; +} + +static void +tracemalloc_stop(void) +{ + if (!tracemalloc_config.installed) + return; + + /* stop tracing Python memory allocations */ + tracemalloc_config.installed = 0; + set_reentrant(1); + + /* unregister the hook on memory allocators */ +#ifdef TRACE_RAW_MALLOC + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); +#endif + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); + + /* release memory */ + tracemalloc_clear_traces(); +} + + +static PyObject* +tracemalloc_lineno_as_obj(int lineno) +{ + if (lineno >= 0) + return PyLong_FromLong(lineno); + else + Py_RETURN_NONE; +} + +typedef struct { + PyObject_HEAD + filter_t filter; +} FilterObject; + +/* Converter for PyArg_ParseTuple() to parse a filename */ +static int +parse_filename(PyObject* arg, void* addr) +{ + if (PyUnicode_Check(arg)) { + if (PyUnicode_READY(arg) < 0) + return 1; + *(PyObject **)addr = arg; + return 1; + } + else { + PyErr_SetString(PyExc_TypeError, "filename must be a str"); + return 0; + } +} + +/* Converter for PyArg_ParseTuple() to parse a line number */ +static int +parse_lineno(PyObject* arg, void* addr) +{ + int lineno; + + if (!PyLong_Check(arg)) { + PyErr_Format(PyExc_TypeError, + "lineno must be an int, not %s", + Py_TYPE(arg)->tp_name); + return 0; + } + + lineno = _PyLong_AsInt(arg); + if (lineno == -1 && PyErr_Occurred()) + return 0; + if (lineno < 0) { + PyErr_Format(PyExc_ValueError, + "lineno must be positive", + MAX_NFRAME); + return 0; + } + *(int *)addr = lineno; + return 1; +} + +/* Converter for PyArg_ParseTuple() to parse a line number, accepting None */ +static int +parse_filter_lineno(PyObject* arg, void* addr) +{ + if (arg == Py_None) { + *(int *)addr = -1; + return 1; + } + else { + return parse_lineno(arg, addr); + } +} + +PyDoc_STRVAR(pyfilter_doc, +"Filter(include: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False)"); + +static int +pyfilter_init(FilterObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"inclusive", "filename_pattern", + "lineno", "all_frames", NULL}; + int include; + PyObject *filename; + int lineno = -1; + int all_frames = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iO&|O&i:Filter", kwlist, + &include, + parse_filename, &filename, + parse_filter_lineno, &lineno, + &all_frames)) + return -1; + + filter_deinit(&self->filter); + + if (filter_init(&self->filter, include, filename, lineno, all_frames) < 0) + return -1; + + return 0; +} + +static void +pyfilter_dealloc(FilterObject *self) +{ + filter_deinit(&self->filter); + PyObject_FREE(self); +} + +static PyObject* +pyfilter_match(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *filename; + int lineno; + int match; + + if (!PyArg_ParseTuple(args, "O&O&:_match", + parse_filename, &filename, + parse_lineno, &lineno)) + return NULL; + + match = filter_match(&pyfilter->filter, filename, lineno); + return PyBool_FromLong(match); +} + +static int +parse_traceback(PyObject *pytraceback, + traceback_t *traceback, size_t buffer_size) +{ + PyObject *pyframe, *filename, *pylineno; + Py_ssize_t nframe, i; + int lineno; + + if (!PyTuple_Check(pytraceback)) { + PyErr_SetString(PyExc_TypeError, "traceback must be a tuple"); + return -1; + } + + nframe = PyTuple_GET_SIZE(pytraceback); + if (nframe > MAX_NFRAME) { + PyErr_SetString(PyExc_TypeError, "too many frames"); + return -1; + } + assert(TRACEBACK_SIZE(nframe) <= buffer_size); + + traceback->nframe = nframe; + for (i=0; i < nframe; i++) { + pyframe = PyTuple_GET_ITEM(pytraceback, i); + assert(pyframe != NULL); + if (!PyTuple_Check(pyframe) || Py_SIZE(pyframe) != 2) { + PyErr_SetString(PyExc_TypeError, "frames must be 2-tuples"); + return -1; + } + + filename = PyTuple_GET_ITEM(pyframe, 0); + assert(filename != NULL); + if (parse_filename(filename, &filename) != 1) + return -1; + + pylineno = PyTuple_GET_ITEM(pyframe, 1); + assert(pylineno != NULL); + if (parse_lineno(pylineno, &lineno) == 0) + return -1; + + /* borrowed reference to filename */ + traceback->frames[i].filename = filename; + traceback->frames[i].lineno = lineno; + } + + return 0; +} + +static PyObject* +pyfilter_match_traceback(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *pytraceback; + int match; + char stack_buffer[TRACEBACK_STACK_SIZE]; + traceback_t *traceback = (traceback_t *)stack_buffer; + + if (!PyArg_ParseTuple(args, "O:_match_traceback", &pytraceback)) + return NULL; + + if (parse_traceback(pytraceback, traceback, sizeof(stack_buffer)) < 0) + return NULL; + + match = filter_match_traceback(&pyfilter->filter, traceback); + return PyBool_FromLong(match); +} + +static PyObject* +pyfilter_match_filename(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *filename; + int match; + + if (!PyArg_ParseTuple(args, "O&:_match_filename", + parse_filename, &filename)) + return NULL; + + match = filter_match_filename(&pyfilter->filter, filename); + return PyBool_FromLong(match); +} + +static PyObject * +pyfilter_get_include(FilterObject *self, void *closure) +{ + return PyBool_FromLong(self->filter.include); +} + +static PyObject * +pyfilter_get_filename_pattern(FilterObject *self, void *closure) +{ + Py_INCREF(self->filter.filename_pattern); + return self->filter.filename_pattern; +} + +static PyObject * +pyfilter_get_lineno(FilterObject *self, void *closure) +{ + return tracemalloc_lineno_as_obj(self->filter.lineno); +} + +static PyObject * +pyfilter_get_all_frames(FilterObject *self, void *closure) +{ + return PyBool_FromLong(self->filter.all_frames); +} + +static PyObject* +pyfilter_repr(FilterObject *self) +{ + char lineno[30]; + + if (self->filter.lineno > 1) + PyOS_snprintf(lineno, sizeof(lineno), "%i", self->filter.lineno); + else + strcpy(lineno, "None"); + return PyUnicode_FromFormat("", + self->filter.include ? "True" : "False", + self->filter.filename_pattern, + lineno, + self->filter.all_frames ? "True" : "False"); +} + +static int +filter_compare(filter_t *f1, filter_t *f2) +{ + if (f1->include != f2->include) + return 0; + if (f1->lineno != f2->lineno) + return 0; + if (f1->all_frames != f2->all_frames) + return 0; + if (PyUnicode_Compare(f1->filename_pattern, f2->filename_pattern) != 0) + return 0; +#ifndef TRACE_NORMALIZE_FILENAME + assert(f1->use_joker == f2->use_joker); +#endif + return 1; +} + +static Py_hash_t +pyfilter_hash(FilterObject *self) +{ + Py_hash_t hash; + + hash = self->filter.pattern_hash; + hash ^= self->filter.lineno; + hash ^= ((Py_hash_t)self->filter.include << 20); + hash ^= ((Py_hash_t)self->filter.all_frames << 21); + return hash; +} + +static PyObject * +pyfilter_richcompare(FilterObject *self, FilterObject *other, int op) +{ + if (op == Py_EQ || op == Py_NE) { + int eq; + PyObject *res; + + eq = filter_compare(&self->filter, &other->filter); + if (op == Py_NE) + eq = !eq; + + if (eq) + res = Py_True; + else + res = Py_False; + Py_INCREF(res); + return res; + } + else { + Py_RETURN_NOTIMPLEMENTED; + } +} + +static PyGetSetDef pyfilter_getset[] = { + {"inclusive", (getter) pyfilter_get_include, NULL, + "Include or exclude the trace?"}, + {"filename_pattern", (getter) pyfilter_get_filename_pattern, NULL, + "Pattern matching a filename, can contain one " + "or many '*' joker characters"}, + {"lineno", (getter) pyfilter_get_lineno, NULL, + "Line number"}, + {"all_frames", (getter) pyfilter_get_all_frames, NULL, + "Check all frames or only the most recent frame?"}, + {NULL} +}; + +static PyMethodDef pyfilter_methods[] = { + {"_match", (PyCFunction)pyfilter_match, + METH_VARARGS, + PyDoc_STR("_match(filename: str, lineno: int) -> bool")}, + {"_match_traceback", (PyCFunction)pyfilter_match_traceback, + METH_VARARGS, + PyDoc_STR("_match_traceback(traceback) -> bool")}, + {"_match_filename", (PyCFunction)pyfilter_match_filename, + METH_VARARGS, + PyDoc_STR("_match_filename(filename: str) -> bool")}, + {NULL, NULL} /* sentinel */ +}; + +static PyTypeObject FilterType = { + /* The ob_type field must be initialized in the module init function + * to be portable to Windows without using C++. */ + PyVarObject_HEAD_INIT(NULL, 0) + "tracemalloc.Filter", /*tp_name*/ + sizeof(FilterObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)pyfilter_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + (getattrfunc)0, /*tp_getattr*/ + (setattrfunc)0, /*tp_setattr*/ + 0, /*tp_reserved*/ + (reprfunc)pyfilter_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + (hashfunc)pyfilter_hash, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + (getattrofunc)0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + pyfilter_doc, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + (richcmpfunc)pyfilter_richcompare, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + pyfilter_methods, /*tp_methods*/ + 0, /*tp_members*/ + pyfilter_getset, /* tp_getset */ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + (initproc)pyfilter_init, /* tp_init */ + 0, /*tp_alloc*/ + PyType_GenericNew, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ +}; + +PyDoc_STRVAR(tracemalloc_is_tracing_doc, + "is_tracing()->bool\n" + "\n" + "True if the tracemalloc module is tracing Python memory allocations,\n" + "False otherwise."); + +static PyObject* +py_tracemalloc_is_tracing(PyObject *self) +{ + return PyBool_FromLong(tracemalloc_config.installed); +} + +PyDoc_STRVAR(tracemalloc_clear_traces_doc, + "clear_traces()\n" + "\n" + "Clear traces of memory blocks allocated by Python."); + +static PyObject* +py_tracemalloc_clear_traces(PyObject *self) +{ + if (!tracemalloc_config.installed) + Py_RETURN_NONE; + + set_reentrant(1); + tracemalloc_clear_traces(); + set_reentrant(0); + + Py_RETURN_NONE; +} + +static PyObject* +frame_to_pyobject(frame_t *frame) +{ + PyObject *frame_obj, *lineno_obj; + + frame_obj = PyTuple_New(2); + if (frame_obj == NULL) + return NULL; + + if (frame->filename == NULL) + frame->filename = Py_None; + Py_INCREF(frame->filename); + PyTuple_SET_ITEM(frame_obj, 0, frame->filename); + + assert(frame->lineno >= 0); + lineno_obj = tracemalloc_lineno_as_obj(frame->lineno); + if (lineno_obj == NULL) { + Py_DECREF(frame_obj); + return NULL; + } + PyTuple_SET_ITEM(frame_obj, 1, lineno_obj); + + return frame_obj; +} + +static PyObject* +traceback_to_pyobject(traceback_t *traceback, _Py_hashtable_t *intern_table) +{ + int i; + PyObject *frames, *frame; + + if (intern_table != NULL) { + if (HASHTABLE_GET(intern_table, traceback, frames)) { + Py_INCREF(frames); + return frames; + } + } + + frames = PyTuple_New(traceback->nframe); + if (frames == NULL) + return NULL; + + for (i=0; i < traceback->nframe; i++) { + frame = frame_to_pyobject(&traceback->frames[i]); + if (frame == NULL) { + Py_DECREF(frames); + return NULL; + } + PyTuple_SET_ITEM(frames, i, frame); + } + + if (intern_table != NULL) { + if (HASHTABLE_SET(intern_table, traceback, frames) < 0) { + Py_DECREF(frames); + PyErr_NoMemory(); + return NULL; + } + /* intern_table keeps a new reference to frames */ + Py_INCREF(frames); + } + return frames; +} + +static PyObject* +trace_to_pyobject(trace_t *trace, _Py_hashtable_t *intern_tracebacks) +{ + PyObject *trace_obj = NULL; + PyObject *size, *traceback; + + trace_obj = PyTuple_New(2); + if (trace_obj == NULL) + return NULL; + + size = PyLong_FromSize_t(trace->size); + if (size == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyTuple_SET_ITEM(trace_obj, 0, size); + + traceback = traceback_to_pyobject(trace->traceback, intern_tracebacks); + if (traceback == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyTuple_SET_ITEM(trace_obj, 1, traceback); + + return trace_obj; +} + +typedef struct { + _Py_hashtable_t *traces; + _Py_hashtable_t *tracebacks; + PyObject *list; +} get_traces_t; + +static int +tracemalloc_get_traces_fill(hashtable_entry_t *entry, void *user_data) +{ + get_traces_t *get_traces = user_data; + trace_t *trace; + PyObject *tracemalloc_obj; + int res; + + trace = (trace_t *)_PY_HASHTABLE_ENTRY_DATA(entry); + + tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); + if (tracemalloc_obj == NULL) + return 1; + + res = PyList_Append(get_traces->list, tracemalloc_obj); + Py_DECREF(tracemalloc_obj); + if (res < 0) + return 1; + + return 0; +} + +static int +tracemalloc_pyobject_decref_cb(hashtable_entry_t *entry, void *user_data) +{ + PyObject *obj = (PyObject *)_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + Py_DECREF(obj); + return 0; +} + +PyDoc_STRVAR(tracemalloc_get_traces_doc, + "get_traces() -> list\n" + "\n" + "Get traces of all memory blocks allocated by Python.\n" + "Return a list of (size: int, traceback: tuple) tuples.\n" + "traceback is a tuple of (filename: str, lineno: int) tuples.\n" + "\n" + "Return an empty list if the tracemalloc module is disabled."); + +static PyObject* +py_tracemalloc_get_traces(PyObject *self, PyObject *obj) +{ + get_traces_t get_traces; + int err; + + if (!tracemalloc_config.installed) + return PyDict_New(); + + get_traces.traces = NULL; + get_traces.tracebacks = NULL; + get_traces.list = PyList_New(0); + if (get_traces.list == NULL) + goto error; + + get_traces.tracebacks = hashtable_new(sizeof(PyObject *), + _Py_hashtable_hash_ptr, + _Py_hashtable_compare_direct); + if (get_traces.tracebacks == NULL) { + PyErr_NoMemory(); + goto error; + } + + TABLES_LOCK(); + get_traces.traces = _Py_hashtable_copy(tracemalloc_traces); + TABLES_UNLOCK(); + + if (get_traces.traces == NULL) { + PyErr_NoMemory(); + goto error; + } + + set_reentrant(1); + err = _Py_hashtable_foreach(get_traces.traces, + tracemalloc_get_traces_fill, &get_traces); + set_reentrant(0); + if (err) + goto error; + + goto finally; + +error: + Py_CLEAR(get_traces.list); + +finally: + if (get_traces.tracebacks != NULL) { + _Py_hashtable_foreach(get_traces.tracebacks, + tracemalloc_pyobject_decref_cb, NULL); + _Py_hashtable_destroy(get_traces.tracebacks); + } + if (get_traces.traces != NULL) + _Py_hashtable_destroy(get_traces.traces); + + return get_traces.list; +} + +PyDoc_STRVAR(tracemalloc_get_object_traceback_doc, + "get_object_traceback(obj)\n" + "\n" + "Get the traceback where the Python object obj was allocated.\n" + "Return a tuple of (filename: str, lineno: int) tuples.\n" + "\n" + "Return None if the tracemalloc module is disabled or did not\n" + "trace the allocation of the object."); + +static PyObject* +py_tracemalloc_get_object_traceback(PyObject *self, PyObject *obj) +{ + PyTypeObject *type; + void *ptr; + trace_t trace; + int found; + + if (!tracemalloc_config.installed) + Py_RETURN_NONE; + + type = Py_TYPE(obj); + if (PyType_IS_GC(type)) + ptr = (void *)((char *)obj - sizeof(PyGC_Head)); + else + ptr = (void *)obj; + + TABLES_LOCK(); + found = HASHTABLE_GET(tracemalloc_traces, ptr, trace); + TABLES_UNLOCK(); + + if (!found) + Py_RETURN_NONE; + + return traceback_to_pyobject(trace.traceback, NULL); +} + +static PyObject* +tracemalloc_atexit(PyObject *self) +{ +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + tracemalloc_deinit(); + Py_RETURN_NONE; +} + +static PyMethodDef atexit_method = { + "_atexit", (PyCFunction)tracemalloc_atexit, METH_NOARGS, NULL}; + +static int +tracemalloc_atexit_register(void) +{ + PyObject *method = NULL, *atexit = NULL, *func = NULL; + PyObject *result; + int ret = -1; + + /* private functions */ + method = PyCFunction_New(&atexit_method, NULL); + if (method == NULL) + goto done; + + atexit = PyImport_ImportModule("atexit"); + if (atexit == NULL) { + if (!PyErr_Warn(PyExc_ImportWarning, + "atexit module is missing: " + "cannot automatically disable tracemalloc at exit")) + { + PyErr_Clear(); + return 0; + } + goto done; + } + + func = PyObject_GetAttrString(atexit, "register"); + if (func == NULL) + goto done; + + result = PyObject_CallFunction(func, "O", method); + if (result == NULL) + goto done; + Py_DECREF(result); + + ret = 0; + +done: + Py_XDECREF(method); + Py_XDECREF(func); + Py_XDECREF(atexit); + return ret; +} + +PyDoc_STRVAR(tracemalloc_start_doc, + "start()\n" + "\n" + "Start tracing Python memory allocations."); + +static PyObject* +py_tracemalloc_start(PyObject *self) +{ + if (tracemalloc_start() < 0) + return NULL; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_stop_doc, + "stop()\n" + "\n" + "Stop tracing Python memory allocations and clear traces\n" + "of memory blocks allocated by Python."); + +static PyObject* +py_tracemalloc_stop(PyObject *self) +{ + tracemalloc_stop(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_traceback_limit_doc, + "get_traceback_limit() -> int\n" + "\n" + "Get the maximum number of frames stored in the traceback\n" + "of a trace.\n" + "\n" + "By default, a trace of an allocated memory block only stores\n" + "the most recent frame: the limit is 1."); + +static PyObject* +py_tracemalloc_get_traceback_limit(PyObject *self) +{ + return PyLong_FromLong(tracemalloc_config.max_nframe); +} + +PyDoc_STRVAR(tracemalloc_set_traceback_limit_doc, + "set_traceback_limit(nframe: int)\n" + "\n" + "Set the maximum number of frames stored in the traceback of a trace."); + +static PyObject* +tracemalloc_set_traceback_limit(PyObject *self, PyObject *args) +{ + int nframe; + + if (!PyArg_ParseTuple(args, "i:set_traceback_limit", + &nframe)) + return NULL; + + if (nframe < 0 || nframe > MAX_NFRAME) { + PyErr_Format(PyExc_ValueError, + "the number of frames must be in range [0; %i]", + MAX_NFRAME); + return NULL; + } + tracemalloc_config.max_nframe = nframe; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_tracemalloc_memory_doc, + "get_tracemalloc_memory() -> int\n" + "\n" + "Get the memory usage in bytes of the tracemalloc module\n" + "used internally to trace memory allocations."); + +static PyObject* +tracemalloc_get_tracemalloc_memory(PyObject *self) +{ + size_t size; + PyObject *size_obj; + + /* hash tables */ + size = _Py_hashtable_size(tracemalloc_tracebacks); + size += _Py_hashtable_size(tracemalloc_filenames); + + TABLES_LOCK(); + size += _Py_hashtable_size(tracemalloc_traces); + TABLES_UNLOCK(); + + size_obj = PyLong_FromSize_t(size); + return Py_BuildValue("N", size_obj); +} + +PyDoc_STRVAR(tracemalloc_get_traced_memory_doc, + "get_traced_memory() -> int\n" + "\n" + "Get the current size and maximum size of memory blocks traced\n" + "by the tracemalloc module as a tuple: (size: int, max_size: int)."); + +static PyObject* +tracemalloc_get_traced_memory(PyObject *self) +{ + size_t size, max_size; + PyObject *size_obj, *max_size_obj; + + TABLES_LOCK(); + size = tracemalloc_traced_memory; + max_size = tracemalloc_max_traced_memory; + TABLES_UNLOCK(); + + size_obj = PyLong_FromSize_t(size); + max_size_obj = PyLong_FromSize_t(max_size); + return Py_BuildValue("NN", size_obj, max_size_obj); +} + +static int +tracemalloc_add_filter(filter_t *filter) +{ + size_t i, nfilter; + filter_list_t *filters; + filter_t *new_filters; + + if (filter->include) + filters = &tracemalloc_include_filters; + else + filters = &tracemalloc_exclude_filters; + + for(i=0; infilter; i++) { + if (filter_compare(&filters->filters[i], filter) == 1) { + /* filter already present, don't add a duplicate */ + return 0; + } + } + + nfilter = (filters->nfilter + 1); + new_filters = raw_realloc(filters->filters, nfilter * sizeof(filter_t)); + if (new_filters == NULL) { + PyErr_NoMemory(); + return -1; + } + + Py_INCREF(filter->filename_pattern); + new_filters[filters->nfilter] = *filter; + + filters->nfilter = nfilter; + filters->filters = new_filters; + return 0; +} + +PyDoc_STRVAR(tracemalloc_add_filter_doc, + "add_filter(filter)\n" + "\n" + "Add a new filter on Python memory allocations,\n" + "filter is a Filter instance."); + +static PyObject* +py_tracemalloc_add_filter(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter; + + if (!PyArg_ParseTuple(args, "O!:add_filter", + &FilterType, (PyObject **)&pyfilter)) + return NULL; + + if (tracemalloc_add_filter(&pyfilter->filter) < 0) + return NULL; + + Py_RETURN_NONE; +} + +static PyObject* +filter_as_obj(filter_t *filter) +{ + FilterObject *pyfilter; + + pyfilter = PyObject_New(FilterObject, &FilterType); + if (pyfilter == NULL) + return NULL; + + Py_INCREF(filter->filename_pattern); + pyfilter->filter = *filter; + return (PyObject *)pyfilter; +} + +PyDoc_STRVAR(tracemalloc_get_filters_doc, + "get_filters()\n" + "\n" + "Get the filters on Python memory allocations.\n" + "Return a list of Filter instances."); + +static size_t +tracemalloc_get_filters(PyObject *list, size_t first_index, + filter_list_t *filters) +{ + size_t i; + filter_t *filter; + PyObject *pyfilter; + + for (i=0; infilter; i++) { + filter = &filters->filters[i]; + + pyfilter = filter_as_obj(filter); + if (pyfilter == NULL) + return (size_t)-1; + + PyList_SET_ITEM(list, first_index + i, pyfilter); + } + return filters->nfilter; +} + +static PyObject* +py_tracemalloc_get_filters(PyObject *self) +{ + PyObject *filters = NULL; + size_t number; + + filters = PyList_New(tracemalloc_include_filters.nfilter + + tracemalloc_exclude_filters.nfilter); + if (filters == NULL) + return NULL; + + number = tracemalloc_get_filters(filters, 0, &tracemalloc_include_filters); + if (number == (size_t)-1) { + Py_DECREF(filters); + return NULL; + } + + number = tracemalloc_get_filters(filters, number, &tracemalloc_exclude_filters); + if (number == (size_t)-1) { + Py_DECREF(filters); + return NULL; + } + + return filters; +} + +PyDoc_STRVAR(tracemalloc_clear_filters_doc, + "clear_filters()\n" + "\n" + "Clear the filter list."); + +static PyObject* +py_tracemalloc_clear_filters(PyObject *self) +{ + tracemalloc_clear_filters(); + Py_RETURN_NONE; +} + +static PyMethodDef module_methods[] = { + {"is_tracing", (PyCFunction)py_tracemalloc_is_tracing, + METH_NOARGS, tracemalloc_is_tracing_doc}, + {"clear_traces", (PyCFunction)py_tracemalloc_clear_traces, + METH_NOARGS, tracemalloc_clear_traces_doc}, + {"get_traces", (PyCFunction)py_tracemalloc_get_traces, + METH_NOARGS, tracemalloc_get_traces_doc}, + {"get_object_traceback", (PyCFunction)py_tracemalloc_get_object_traceback, + METH_O, tracemalloc_get_object_traceback_doc}, + {"start", (PyCFunction)py_tracemalloc_start, + METH_NOARGS, tracemalloc_start_doc}, + {"stop", (PyCFunction)py_tracemalloc_stop, + METH_NOARGS, tracemalloc_stop_doc}, + {"get_traceback_limit", (PyCFunction)py_tracemalloc_get_traceback_limit, + METH_NOARGS, tracemalloc_get_traceback_limit_doc}, + {"set_traceback_limit", (PyCFunction)tracemalloc_set_traceback_limit, + METH_VARARGS, tracemalloc_set_traceback_limit_doc}, + {"get_tracemalloc_memory", (PyCFunction)tracemalloc_get_tracemalloc_memory, + METH_NOARGS, tracemalloc_get_tracemalloc_memory_doc}, + {"get_traced_memory", (PyCFunction)tracemalloc_get_traced_memory, + METH_NOARGS, tracemalloc_get_traced_memory_doc}, + {"add_filter", (PyCFunction)py_tracemalloc_add_filter, + METH_VARARGS, tracemalloc_add_filter_doc}, + {"get_filters", (PyCFunction)py_tracemalloc_get_filters, + METH_NOARGS, tracemalloc_get_filters_doc}, + {"clear_filters", (PyCFunction)py_tracemalloc_clear_filters, + METH_NOARGS, tracemalloc_clear_filters_doc}, + + /* sentinel */ + {NULL, NULL} +}; + +PyDoc_STRVAR(module_doc, +"Debug module to trace memory blocks allocated by Python."); + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "_tracemalloc", + module_doc, + 0, /* non-negative size to be able to unload the module */ + module_methods, + NULL, +}; + +PyMODINIT_FUNC +PyInit__tracemalloc(void) +{ + PyObject *m; + m = PyModule_Create(&module_def); + if (m == NULL) + return NULL; + + if (tracemalloc_init() < 0) + return NULL; + + if (PyType_Ready(&FilterType) < 0) + return NULL; + + Py_INCREF((PyObject*) &FilterType); + PyModule_AddObject(m, "Filter", (PyObject*)&FilterType); + + return m; +} + +static int +parse_sys_xoptions(PyObject *value) +{ + PyObject *valuelong = NULL, *max_nframe = NULL; + int nframe; + + if (value == Py_True) + return 1; + + assert(PyUnicode_Check(value)); + + if (PyUnicode_GetLength(value) == 0) + goto error; + + valuelong = PyLong_FromUnicodeObject(value, 10); + if (valuelong == NULL) + goto error; + + max_nframe = PyLong_FromLong(MAX_NFRAME); + if (max_nframe == NULL) + goto error; + + /* if 0 <= valuelong < MAX_NFRAME */ + if (_PyLong_Sign(valuelong) >= 0 + && PyObject_RichCompareBool(valuelong, max_nframe, Py_LE)) { + nframe = PyLong_AsUnsignedLongMask(valuelong); + if (nframe == (unsigned long)-1 && PyErr_Occurred()) + goto error; + } + else + nframe = MAX_NFRAME + 1; + + if (nframe > MAX_NFRAME) + goto error; + + Py_DECREF(valuelong); + Py_DECREF(max_nframe); + + assert(nframe >= 0); + return nframe; + +error: + Py_XDECREF(valuelong); + Py_XDECREF(max_nframe); + return -1; +} + +int +_PyTraceMalloc_Init(void) +{ + char *p; + int nframe; + +#ifdef WITH_THREAD + assert(PyGILState_Check()); +#endif + + if ((p = Py_GETENV("PYTHONTRACEMALLOC")) && *p != '\0') { + char *endptr = p; + unsigned long value; + + value = strtoul(p, &endptr, 10); + if (*endptr != '\0' + || value > MAX_NFRAME + || (errno == ERANGE && value == ULONG_MAX)) + { + Py_FatalError("PYTHONTRACEMALLOC must be an integer " + "in range [0; " STR(MAX_NFRAME) "]"); + return -1; + } + + nframe = (int)value; + } + else { + PyObject *xoptions, *key, *value; + + xoptions = PySys_GetXOptions(); + if (xoptions == NULL) + return -1; + + key = PyUnicode_FromString("tracemalloc"); + if (key == NULL) + return -1; + + value = PyDict_GetItemWithError(xoptions, key); + Py_DECREF(key); + if (value == NULL) { + if (PyErr_Occurred()) + return -1; + + /* -X tracemalloc is not used */ + return 0; + } + + nframe = parse_sys_xoptions(value); + Py_DECREF(value); + if (nframe < 0) { + Py_FatalError("-X tracemalloc=NFRAME: number of frame must be " + "an integer in range [0; " STR(MAX_NFRAME) "]"); + } + } + + tracemalloc_config.max_nframe = nframe; + return tracemalloc_start(); +} + diff -r 2ed8d500e113 -r 65e72bf01246 PC/config.c --- a/PC/config.c Sun Nov 03 13:53:12 2013 +0100 +++ b/PC/config.c Sun Nov 03 14:25:07 2013 +0100 @@ -13,6 +13,7 @@ extern PyObject* PyInit_binascii(void); extern PyObject* PyInit_cmath(void); extern PyObject* PyInit_errno(void); extern PyObject* PyInit_faulthandler(void); +extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); extern PyObject* PyInit_math(void); extern PyObject* PyInit__md5(void); @@ -102,6 +103,7 @@ struct _inittab _PyImport_Inittab[] = { {"msvcrt", PyInit_msvcrt}, {"_locale", PyInit__locale}, #endif + {"_tracemalloc", PyInit__tracemalloc}, /* XXX Should _winapi go in a WIN32 block? not WIN64? */ {"_winapi", PyInit__winapi}, diff -r 2ed8d500e113 -r 65e72bf01246 PCbuild/pythoncore.vcxproj --- a/PCbuild/pythoncore.vcxproj Sun Nov 03 13:53:12 2013 +0100 +++ b/PCbuild/pythoncore.vcxproj Sun Nov 03 14:25:07 2013 +0100 @@ -390,6 +390,7 @@ + @@ -531,6 +532,7 @@ + @@ -638,6 +640,7 @@ + @@ -682,4 +685,4 @@ - \ No newline at end of file + diff -r 2ed8d500e113 -r 65e72bf01246 Python/hashtable.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Python/hashtable.c Sun Nov 03 14:25:07 2013 +0100 @@ -0,0 +1,517 @@ +/* The implementation of the hash table (_Py_hashtable_t) is based on the cfuhash + project: + http://sourceforge.net/projects/libcfu/ + + Copyright of cfuhash: + ---------------------------------- + Creation date: 2005-06-24 21:22:40 + Authors: Don + Change log: + + Copyright (c) 2005 Don Owens + All rights reserved. + + This code is released under the BSD license: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + ---------------------------------- +*/ + +#include "Python.h" +#include "hashtable.h" + +#define HASHTABLE_MIN_SIZE 16 +#define HASHTABLE_HIGH 0.50 +#define HASHTABLE_LOW 0.10 +#define HASHTABLE_REHASH_FACTOR 2.0 / (HASHTABLE_LOW + HASHTABLE_HIGH) + +#define BUCKETS_HEAD(SLIST) ((hashtable_entry_t *)SLIST_HEAD(&(SLIST))) +#define TABLE_HEAD(HT, BUCKET) ((hashtable_entry_t *)SLIST_HEAD(&(HT)->buckets[BUCKET])) +#define ENTRY_NEXT(ENTRY) ((hashtable_entry_t *)SLIST_ITEM_NEXT(ENTRY)) +#define HASHTABLE_ITEM_SIZE(HT) (sizeof(hashtable_entry_t) + (HT)->data_size) + +/* Forward declaration */ +static void hashtable_rehash(_Py_hashtable_t *ht); + +static void +_Py_slist_init(_Py_slist_t *list) +{ + list->head = NULL; +} + +static void +_Py_slist_prepend(_Py_slist_t *list, _Py_slist_item_t *item) +{ + item->next = list->head; + list->head = item; +} + +static void +_Py_slist_remove(_Py_slist_t *list, _Py_slist_item_t *previous, + _Py_slist_item_t *item) +{ + if (previous != NULL) + previous->next = item->next; + else + list->head = item->next; +} + +Py_uhash_t +_Py_hashtable_hash_int(const void *key) +{ + return (Py_uhash_t)Py_POINTER_TO_INT(key); +} + +Py_uhash_t +_Py_hashtable_hash_ptr(const void *key) +{ + return (Py_uhash_t)_Py_HashPointer((void *)key); +} + +int +_Py_hashtable_compare_direct(const void *key, hashtable_entry_t *entry) +{ + return entry->key == key; +} + +/* makes sure the real size of the buckets array is a power of 2 */ +static size_t +round_size(size_t s) +{ + size_t i; + if (s < HASHTABLE_MIN_SIZE) + return HASHTABLE_MIN_SIZE; + i = 1; + while (i < s) + i <<= 1; + return i; +} + +_Py_hashtable_t * +_Py_hashtable_new_full(size_t data_size, size_t init_size, + hashtable_hash_func hash_func, + hashtable_compare_func compare_func, + hashtable_copy_data_func copy_data_func, + hashtable_free_data_func free_data_func, + hashtable_get_data_size_func get_data_size_func, + _Py_hashtable_allocator_t *allocator) +{ + _Py_hashtable_t *ht; + size_t buckets_size; + _Py_hashtable_allocator_t alloc; + + if (allocator == NULL) { + alloc.malloc = PyMem_RawMalloc; + alloc.free = PyMem_RawFree; + } + else + alloc = *allocator; + + ht = (_Py_hashtable_t *)alloc.malloc(sizeof(_Py_hashtable_t)); + if (ht == NULL) + return ht; + + ht->num_buckets = round_size(init_size); + ht->entries = 0; + ht->data_size = data_size; + + buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); + ht->buckets = alloc.malloc(buckets_size); + if (ht->buckets == NULL) { + alloc.free(ht); + return NULL; + } + memset(ht->buckets, 0, buckets_size); + + ht->hash_func = hash_func; + ht->compare_func = compare_func; + ht->copy_data_func = copy_data_func; + ht->free_data_func = free_data_func; + ht->get_data_size_func = get_data_size_func; + ht->alloc = alloc; + return ht; +} + +_Py_hashtable_t * +_Py_hashtable_new(size_t data_size, + hashtable_hash_func hash_func, + hashtable_compare_func compare_func) +{ + return _Py_hashtable_new_full(data_size, HASHTABLE_MIN_SIZE, + hash_func, compare_func, + NULL, NULL, NULL, NULL); +} + +size_t +_Py_hashtable_size(_Py_hashtable_t *ht) +{ + size_t size; + size_t hv; + + size = sizeof(_Py_hashtable_t); + + /* buckets */ + size += ht->num_buckets * sizeof(hashtable_entry_t *); + + /* entries */ + size += ht->entries * HASHTABLE_ITEM_SIZE(ht); + + /* data linked from entries */ + if (ht->get_data_size_func) { + for (hv = 0; hv < ht->num_buckets; hv++) { + hashtable_entry_t *entry; + + for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { + void *data; + + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + size += ht->get_data_size_func(data); + } + } + } + return size; +} + +#ifdef Py_DEBUG +void +_Py_hashtable_print_stats(_Py_hashtable_t *ht) +{ + size_t size; + size_t chain_len, max_chain_len, total_chain_len, nchains; + hashtable_entry_t *entry; + size_t hv; + double load; + + size = _Py_hashtable_size(ht); + + load = (double)ht->entries / ht->num_buckets; + + max_chain_len = 0; + total_chain_len = 0; + nchains = 0; + for (hv = 0; hv < ht->num_buckets; hv++) { + entry = TABLE_HEAD(ht, hv); + if (entry != NULL) { + chain_len = 0; + for (; entry; entry = ENTRY_NEXT(entry)) { + chain_len++; + } + if (chain_len > max_chain_len) + max_chain_len = chain_len; + total_chain_len += chain_len; + nchains++; + } + } + printf("hash table %p: entries=%zu/%zu (%.0f%%), ", + ht, ht->entries, ht->num_buckets, load * 100.0); + if (nchains) + printf("avg_chain_len=%.1f, ", (double)total_chain_len / nchains); + printf("max_chain_len=%zu, %zu kB\n", + max_chain_len, size / 1024); +} +#endif + +/* + Returns one if the entry was found, zero otherwise. If found, r is + changed to point to the data in the entry. +*/ +hashtable_entry_t * +_Py_hashtable_get_entry(_Py_hashtable_t *ht, const void *key) +{ + Py_uhash_t key_hash; + size_t index; + hashtable_entry_t *entry; + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { + if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + break; + } + + return entry; +} + +static int +_hashtable_pop_entry(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +{ + Py_uhash_t key_hash; + size_t index; + hashtable_entry_t *entry, *previous; + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + previous = NULL; + for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { + if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + break; + previous = entry; + } + + if (entry == NULL) + return 0; + + _Py_slist_remove(&ht->buckets[index], (_Py_slist_item_t *)previous, + (_Py_slist_item_t *)entry); + ht->entries--; + + if (data != NULL) + _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); + ht->alloc.free(entry); + + if ((float)ht->entries / (float)ht->num_buckets < HASHTABLE_LOW) + hashtable_rehash(ht); + return 1; +} + +/* Add a new entry to the hash. The key must not be present in the hash table. + Return 0 on success, -1 on memory error. */ +int +_Py_hashtable_set(_Py_hashtable_t *ht, const void *key, + void *data, size_t data_size) +{ + Py_uhash_t key_hash; + size_t index; + hashtable_entry_t *entry; + + assert(data != NULL || data_size == 0); +#ifndef NDEBUG + /* Don't write the assertion on a single line because it is interesting + to know the duplicated entry if the assertion failed. The entry can + be read using a debugger. */ + entry = _Py_hashtable_get_entry(ht, key); + assert(entry == NULL); +#endif + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + entry = ht->alloc.malloc(HASHTABLE_ITEM_SIZE(ht)); + if (entry == NULL) { + /* memory allocation failed */ + return -1; + } + + entry->key = (void *)key; + entry->key_hash = key_hash; + + assert(data_size == ht->data_size); + memcpy(_PY_HASHTABLE_ENTRY_DATA(entry), data, data_size); + + _Py_slist_prepend(&ht->buckets[index], (_Py_slist_item_t*)entry); + ht->entries++; + + if ((float)ht->entries / (float)ht->num_buckets > HASHTABLE_HIGH) + hashtable_rehash(ht); + return 0; +} + +/* Get data from an entry. Copy entry data into data and return 1 if the entry + exists, return 0 if the entry does not exist. */ +int +_Py_hashtable_get(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +{ + hashtable_entry_t *entry; + + assert(data != NULL); + + entry = _Py_hashtable_get_entry(ht, key); + if (entry == NULL) + return 0; + _Py_HASHTABLE_ENTRY_READ_DATA(ht, data, data_size, entry); + return 1; +} + +int +_Py_hashtable_pop(_Py_hashtable_t *ht, const void *key, void *data, size_t data_size) +{ + assert(data != NULL); + assert(ht->free_data_func == NULL); + return _hashtable_pop_entry(ht, key, data, data_size); +} + +/* Delete an entry. The entry must exist. */ +void +_Py_hashtable_delete(_Py_hashtable_t *ht, const void *key) +{ +#ifndef NDEBUG + int found = _hashtable_pop_entry(ht, key, NULL, 0); + assert(found); +#else + (void)_hashtable_pop_entry(ht, key, NULL, 0); +#endif +} + +/* Prototype for a pointer to a function to be called foreach + key/value pair in the hash by hashtable_foreach(). Iteration + stops if a non-zero value is returned. */ +int +_Py_hashtable_foreach(_Py_hashtable_t *ht, + int (*func) (hashtable_entry_t *entry, void *arg), + void *arg) +{ + hashtable_entry_t *entry; + size_t hv; + + for (hv = 0; hv < ht->num_buckets; hv++) { + for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { + int res = func(entry, arg); + if (res) + return res; + } + } + return 0; +} + +static void +hashtable_rehash(_Py_hashtable_t *ht) +{ + size_t buckets_size, new_size, bucket; + _Py_slist_t *old_buckets = NULL; + size_t old_num_buckets; + + new_size = round_size((size_t)(ht->entries * HASHTABLE_REHASH_FACTOR)); + if (new_size == ht->num_buckets) + return; + + old_num_buckets = ht->num_buckets; + + buckets_size = new_size * sizeof(ht->buckets[0]); + old_buckets = ht->buckets; + ht->buckets = ht->alloc.malloc(buckets_size); + if (ht->buckets == NULL) { + /* cancel rehash on memory allocation failure */ + ht->buckets = old_buckets ; + /* memory allocation failed */ + return; + } + memset(ht->buckets, 0, buckets_size); + + ht->num_buckets = new_size; + + for (bucket = 0; bucket < old_num_buckets; bucket++) { + hashtable_entry_t *entry, *next; + for (entry = BUCKETS_HEAD(old_buckets[bucket]); entry != NULL; entry = next) { + size_t entry_index; + + assert(ht->hash_func(entry->key) == entry->key_hash); + next = ENTRY_NEXT(entry); + entry_index = entry->key_hash & (new_size - 1); + + _Py_slist_prepend(&ht->buckets[entry_index], (_Py_slist_item_t*)entry); + } + } + + ht->alloc.free(old_buckets); +} + +void +_Py_hashtable_clear(_Py_hashtable_t *ht) +{ + hashtable_entry_t *entry, *next; + size_t i; + + for (i=0; i < ht->num_buckets; i++) { + for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { + next = ENTRY_NEXT(entry); + if (ht->free_data_func) + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->alloc.free(entry); + } + _Py_slist_init(&ht->buckets[i]); + } + ht->entries = 0; + hashtable_rehash(ht); +} + +void +_Py_hashtable_destroy(_Py_hashtable_t *ht) +{ + size_t i; + + for (i = 0; i < ht->num_buckets; i++) { + _Py_slist_item_t *entry = ht->buckets[i].head; + while (entry) { + _Py_slist_item_t *entry_next = entry->next; + if (ht->free_data_func) + ht->free_data_func(_Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry)); + ht->alloc.free(entry); + entry = entry_next; + } + } + + ht->alloc.free(ht->buckets); + ht->alloc.free(ht); +} + +/* Return a copy of the hash table */ +_Py_hashtable_t * +_Py_hashtable_copy(_Py_hashtable_t *src) +{ + _Py_hashtable_t *dst; + hashtable_entry_t *entry; + size_t bucket; + int err; + void *data, *new_data; + + dst = _Py_hashtable_new_full(src->data_size, src->num_buckets, + src->hash_func, src->compare_func, + src->copy_data_func, src->free_data_func, + src->get_data_size_func, &src->alloc); + if (dst == NULL) + return NULL; + + for (bucket=0; bucket < src->num_buckets; bucket++) { + entry = TABLE_HEAD(src, bucket); + for (; entry; entry = ENTRY_NEXT(entry)) { + if (src->copy_data_func) { + data = _Py_HASHTABLE_ENTRY_DATA_AS_VOID_P(entry); + new_data = src->copy_data_func(data); + if (new_data != NULL) + err = _Py_hashtable_set(dst, entry->key, + &new_data, src->data_size); + else + err = 1; + } + else { + data = _PY_HASHTABLE_ENTRY_DATA(entry); + err = _Py_hashtable_set(dst, entry->key, data, src->data_size); + } + if (err) { + _Py_hashtable_destroy(dst); + return NULL; + } + } + } + return dst; +} + diff -r 2ed8d500e113 -r 65e72bf01246 Python/pythonrun.c --- a/Python/pythonrun.c Sun Nov 03 13:53:12 2013 +0100 +++ b/Python/pythonrun.c Sun Nov 03 14:25:07 2013 +0100 @@ -92,6 +92,7 @@ extern int _PyLong_Init(void); extern void PyLong_Fini(void); extern int _PyFaulthandler_Init(void); extern void _PyFaulthandler_Fini(void); +extern int _PyTraceMalloc_Init(void); #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); @@ -442,6 +443,9 @@ void if (install_sigs) initsigs(); /* Signal handling stuff, including initintr() */ + if (_PyTraceMalloc_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize tracemalloc"); + initmain(interp); /* Module __main__ */ if (initstdio() < 0) Py_FatalError(