diff -r df2fdd42b375 -r 21f7c3df0f15 Doc/library/debug.rst --- a/Doc/library/debug.rst Mon Aug 26 22:28:21 2013 +0200 +++ b/Doc/library/debug.rst Tue Sep 17 00:59:50 2013 +0200 @@ -15,3 +15,4 @@ allowing you to identify bottlenecks in profile.rst timeit.rst trace.rst + tracemalloc.rst diff -r df2fdd42b375 -r 21f7c3df0f15 Doc/library/tracemalloc.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/tracemalloc.rst Tue Sep 17 00:59:50 2013 +0200 @@ -0,0 +1,911 @@ +:mod:`tracemalloc` --- Trace memory allocations +=============================================== + +.. module:: tracemalloc + :synopsis: Trace memory allocations. + +The tracemalloc module is a debug tool to trace memory blocks allocated by +Python. It provides the following information: + +* Compute the differences between two snapshots to detect memory leaks +* Statistics on allocated memory blocks per filename and per line number: + total size, number and average size of allocated memory blocks +* For each allocated memory block: its size and the traceback where the block + was allocated + +To trace most memory blocks allocated by Python, the module should be enabled +as early as possible by calling :func:`tracemalloc.enable` function, by setting +the :envvar:`PYTHONTRACEMALLOC` environment variable to ``1``, or by using +:option:`-X` ``tracemalloc`` command line option. + +By default, the :attr:`Trace.traceback` attribute only stores one +:class:`~tracemalloc.Frame` instance per allocated memory block. Use +:func:`set_traceback_limit` to store more frames. + +.. versionadded:: 3.4 + + +Output examples +=============== + +Cumulative top 5 of the biggest allocations grouped by filename, compact +output:: + + 2013-09-16 22:07:39: Cumulative top 5 allocations per filename + #1: : 193 MiB + #2: .../unittest/suite.py: 40 MiB + #3: .../unittest/case.py: 31 MiB + #4: .../test/test___all__.py: 25 MiB + #5: .../test/regrtest.py: 16 MiB + 428 more: 73 MiB + Tracemalloc memory: 39 MiB + Process memory: RSS=711 MiB, VMS=962 MiB + +Top 5 of the biggest allocations grouped by address:: + + 2013-09-16 22:07:39: Top 5 allocations per address + #1: memory block 0x459bcf0: size=1536 KiB + #2: memory block 0x236e2b0: size=96 KiB + #3: memory block 0x39d4350: size=48 KiB + #4: memory block 0x29834a0: size=48 KiB + #5: memory block 0x1c79230: size=48 KiB + 186186 more: size=23 MiB, average=134 B + Traced Python memory: size=25 MiB, average=144 B + Tracemalloc memory: 39 MiB + Process memory: RSS=711 MiB, VMS=962 MiB + +Top 10 of the biggest allocations grouped by line number, full output:: + + 2013-09-16 22:07:39: Top 10 allocations per filename and line number + #1: :274: size=12 MiB, count=104471, average=127 B + #2: .../tracemalloc/Lib/linecache.py:127: size=1012 KiB, count=8793, average=117 B + #3: :704: size=730 KiB, count=6226, average=120 B + #4: .../Lib/unittest/case.py:496: size=347 KiB, count=681, average=522 B + #5: .../tracemalloc/Lib/linecache.py:4527504: size=276 KiB, count=2707, average=104 B + #6: .../Lib/test/test_argparse.py:216: size=222 KiB, count=2001, average=113 B + #7: .../tracemalloc/Lib/linecache.py:698899640: size=177 KiB, count=1691, average=107 B + #8: .../Lib/test/test_argparse.py:203: size=159 KiB, count=685, average=238 B + #9: .../tracemalloc/Lib/sre_compile.py:508: size=119 KiB, count=251, average=487 B + #10: .../Lib/test/test_argparse.py:68: size=116 KiB, count=786, average=151 B + 13809 more: size=9 MiB, count=57498, average=178 B + Traced Python memory: size=25 MiB, count=185790, average=144 B + Tracemalloc memory: 39 MiB + Process memory: RSS=711 MiB, VMS=962 MiB + + +Usage +===== + +Display top 25 +-------------- + +Example displaying the top 25 lines allocating the most memory every minute:: + + import tracemalloc + tracemalloc.enable() + task = tracemalloc.DisplayTopTask(25) + task.start(60) + # ... run your application ... + + +By default, allocations are grouped by filename and line numbers and the top is +written into :data:`sys.stdout`. + +See :class:`DisplayTop` and :class:`DisplayTopTask` classes for more options. + + +Take a snapshot +--------------- + +The :class:`DisplayTopTask` class creates temporary snapshots which are lost +after the top is displayed. When you don't know what you are looking for, you +can take a snapshot of the allocated memory blocks to analyze it while the +application is running, or analyze it later. + +Example taking a snapshot and writing it into a file:: + + snapshot = tracemalloc.Snapshot.create() + snapshot.write(filename) + +Use the following command to display the snapshot file:: + + python -m tracemalloc snapshot.pickle + +See `Command line options`_ for more options. See also +:meth:`Snapshot.apply_filters` and :class:`DisplayTop.display_snapshot` +methods. + + +Compare snapshots +----------------- + +It is not easy to find a memory leak using a single snapshot. It is easier to +take multiple snapshots and compare them to see the differences. + +Example taking a snapshot every minute:: + + import tracemalloc + tracemalloc.enable() + take_snapshot = tracemalloc.TakeSnapshot() + take_snapshot.start(60.0) + # ... run your application ... + +By default, snapshot files are written in the current directory with the name +``tracemalloc-XXXX.pickle`` where ``XXX`` is a simple counter. + +Use the following command to compare snapshot files:: + + python -m tracemalloc snapshot1.pickle snapshot2.pickle ... + +See `Command line options`_, :class:`TakeSnapshot` and :class:`StatsDiff` +classes for more options. + + +API +=== + +Functions +--------- + +.. function:: add_filter(filter) + + Add a new filter on Python memory allocations, *filter* is a :class:`Filter` + instance. + + All inclusive filters are applied at once, a memory allocation is only + ignored if no inclusive filter match its trace. A memory allocation is + ignored if at least one exclusive filter matchs its trace. + + The new filter is not applied on already collected traces. Use + :func:`clear_traces` to ensure that all traces match the new filter. + +.. function:: add_include_filter(filename: str, lineno: int=None, traceback: bool=False) + + Add an inclusive filter: helper for :func:`add_filter` creating a + :class:`Filter` instance with :attr:`~Filter.include` attribute + set to ``True``. + + Example: ``tracemalloc.add_include_filter(tracemalloc.__file__)`` only + includes memory blocks allocated by the :mod:`tracemalloc` module. + + +.. function:: add_exclude_filter(filename: str, lineno: int=None, traceback: bool=False) + + Add an exclusive filter: helper for :func:`add_filter` creating a + :class:`Filter` instance with :attr:`~Filter.include` attribute + set to ``False``. + + Example: ``tracemalloc.add_exclude_filter(tracemalloc.__file__)`` ignores + memory blocks allocated by the :mod:`tracemalloc` module. + + +.. function:: clear_filters() + + Reset the filter list. + + +.. function:: clear_traces() + + Clear all traces and statistics on Python memory allocations, and reset the + :func:`get_traced_memory` counter. + + +.. function:: disable() + + Stop tracing Python memory allocations and stop the timer started by + :func:`start_timer`. + + See also :func:`enable` and :func:`is_enabled` functions. + + +.. function:: enable() + + Start tracing Python memory allocations. + + See also :func:`disable` and :func:`is_enabled` functions. + + +.. function:: get_filters() + + Get the filters on Python memory allocations as list of :class:`Filter` + instances. + + +.. function:: get_traceback_limit() + + Get the maximum number of :class:`Frame` instances stored in the + :attr:`~tracemalloc.Trace.traceback` attribute of a + :class:`~tracemalloc.Trace` instance. + + Use :func:`set_traceback_limit` to change the limit. + + +.. function:: get_object_address(obj) + + Get the address of the memory block of the specified Python object. + + +.. function:: get_object_trace(obj) + + Get the trace of a Python object *obj* as a :class:`~tracemalloc.Trace` + instance. + + The function only returns the trace of the memory block directly holding to + object. The :attr:`~Trace.size` attribute of the trace is smaller than the + total size of the object if the object is composed of more than one memory + block. + + Return ``None`` if the :mod:`tracemalloc` module did not trace the + allocation of the object. + + See also :func:`gc.get_referrers` and :func:`sys.getsizeof` functions. + + +.. function:: get_process_memory() + + Get the memory usage of the current process as a meminfo namedtuple with + two attributes: + + * ``rss``: Resident Set Size in bytes + * ``vms``: size of the virtual memory in bytes + + Return ``None`` if the platform is not supported. + + +.. function:: get_stats() + + Get statistics on traced Python memory blocks as a dictionary ``{filename + (str): {line_number (int): stats}}`` where *stats* in a + :class:`~tracemalloc.TraceStats` instance, *filename* and *line_number* can + be ``None``. + + Return an empty dictionary if the :mod:`tracemalloc` module is + disabled. + + +.. function:: get_traced_memory() + + Get the total size of all traced memory blocks allocated by Python. + + +.. function:: get_tracemalloc_size() + + Get the memory usage in bytes of the :mod:`tracemalloc` module. + + +.. function:: get_traces(obj) + + Get all traces of Python memory allocations as a dictionary ``{address + (int): trace}`` where *trace* is a :class:`~tracemalloc.Trace` instance. + + Return an empty dictionary if the :mod:`tracemalloc` module is disabled. + + +.. function:: is_enabled() + + ``True`` if the :mod:`tracemalloc` module is tracing Python memory + allocations, ``False`` otherwise. + + See also :func:`enable` and :func:`disable` functions. + + +.. function:: start_timer(delay: int, func: callable, args: tuple=(), kwargs: dict={}) + + Start a timer calling ``func(*args, **kwargs)`` every *delay* seconds. + Enable the :mod:`tracemalloc` module if it is disabled. The timer is based + on the Python memory allocator, it is not real time. *func* is called after + at least *delay* seconds, it is not called exactly after *delay* seconds if + no Python memory allocation occurred. The timer has a resolution of 1 + second. + + If the :func:`start_timer` function is called twice, previous parameters are + replaced. Call the :func:`stop_timer` function to stop the timer. + + The :meth:`DisplayTopTask.start` and :meth:`TakeSnapshot.start` methods use + the :func:`start_timer` function to run regulary a task. + + +.. function:: set_traceback_limit(limit: int) + + Set the maximum number of :class:`Frame` instances stored in the + :attr:`~tracemalloc.Trace.traceback` attribute of a + :class:`~tracemalloc.Trace` instance. Clear all traces and statistics on + Python memory allocations if the :mod:`tracemalloc` module is enabled, + + Storing the traceback of each memory allocation has an important overhead on + the memory usage. Example with the Python test suite: tracing all memory + allocations increases the memory usage by ``+50%`` when storing only 1 frame + and ``+150%`` when storing 10 frames. Use :func:`get_tracemalloc_size` to + measure the overhead and :func:`add_filter` to select which memory + allocations are traced. + + Use :func:`get_traceback_limit` to get the current limit. + + +.. function:: stop_timer() + + Stop the timer started by :func:`start_timer`. + + +DisplayTop +---------- + +.. class:: DisplayTop() + + Display the top of allocated memory blocks. + + .. method:: display_snapshot(snapshot, count=10, group_by="filename_lineno", cumulative=False, file=None) + + Display a snapshot of memory blocks allocated by Python, *snapshot* is a + :class:`Snapshot` instance. + + .. method:: display_top_diff(top_diff, count=10, file=None) + + Display differences between two :class:`GroupedStats` instances, + *top_diff* is a :class:`StatsDiff` instance. + + .. method:: display_top_stats(top_stats, count=10, file=None) + + Display the top of allocated memory blocks grouped by the + :attr:`~GroupedStats.group_by` attribute of *top_stats*, *top_stats* is a + :class:`GroupedStats` instance. + + .. attribute:: color + + If ``True``, always use colors. If ``False``, never use colors. The + default value is ``None``: use colors if the *file* parameter is a TTY + device. + + .. attribute:: compare_with_previous + + If ``True`` (default value), compare with the previous snapshot. If + ``False``, compare with the first snapshot. + + .. attribute:: filename_parts + + Number of displayed filename parts (int, default: ``3``). Extra parts + are replaced with ``'...'``. + + .. attribute:: show_average + + If ``True`` (default value), display the average size of memory blocks. + + .. attribute:: show_count + + If ``True`` (default value), display the number of allocated memory + blocks. + + .. attribute:: show_size + + If ``True`` (default value), display the size of memory blocks. + + +DisplayTopTask +-------------- + +.. class:: DisplayTopTask(count=10, group_by="filename_lineno", cumulative=False, file=sys.stdout, user_data_callback=None) + + Task taking temporary snapshots and displaying the top *count* memory + allocations grouped by *group_by*. + + Call the :meth:`start` method to start the task. + + .. method:: display() + + Take a snapshot and display the top *count* biggest allocated memory + blocks grouped by *group_by* using the :attr:`display_top` attribute. + + Return the snapshot, a :class:`Snapshot` instance. + + .. method:: start(delay: int) + + Start a task using the :func:`start_timer` function calling the + :meth:`display` method every *delay* seconds. + + .. method:: stop() + + Stop the task started by the :meth:`~DisplayTop.start` method using the + :func:`stop_timer` function. + + .. attribute:: count + + Maximum number of displayed memory blocks. + + .. attribute:: cumulative + + If ``True``, cumulate size and count of memory blocks of all frames of + each :class:`Trace` instance, not only the most recent frame. The default + value is ``False``. + + The option is ignored if the traceback limit is ``1``, see the + :func:`get_traceback_limit` function. + + .. attribute:: display_top + + Instance of :class:`DisplayTop`. + + .. attribute:: file + + The top is written into *file*. + + .. attribute:: group_by + + Determine how memory allocations are grouped: see :attr:`Snapshot.top_by` + for the available values. + + .. attribute:: user_data_callback + + Optional callback collecting user data (callable, default: ``None``). + See :meth:`Snapshot.create`. + + +Filter +------ + +.. class:: Filter(include: bool, pattern: str, lineno: int=None, traceback: bool=False) + + Filter to select which memory allocations are traced. Filters can be used to + reduce the memory usage of the :mod:`tracemalloc` module, which can be read + using :func:`get_tracemalloc_size`. + + .. method:: match_trace(trace) + + Return ``True`` if the :class:`~Trace` instance must be kept according to + the filter, ``False`` otherwise. + + .. method:: match(filename: str, lineno: int) + + Return ``True`` if the filename and line number must be kept according to + the filter, ``False`` otherwise. + + .. method:: match_filename(filename: str) + + Return ``True`` if the filename must be kept according to the filter, + ``False`` otherwise. + + .. method:: match_lineno(lineno: int) + + Return ``True`` if the line number must be kept according to the filter, + ``False`` otherwise. + + .. attribute:: include + + If *include* is ``True``, only trace memory blocks allocated in a file + with a name matching filename :attr:`pattern` at line number + :attr:`lineno`. If *include* is ``False``, ignore memory blocks allocated + in a file with a name matching filename :attr`pattern` at line number + :attr:`lineno`. + + .. attribute:: pattern + + The filename *pattern* can contain one or many ``*`` joker characters + which match any substring, including an empty string. The ``.pyc`` and + ``.pyo`` suffixes are replaced with ``.py``. On Windows, the comparison + is case insensitive and the alternative separator ``/`` is replaced with + the standard separator ``\``. + + .. attribute:: lineno + + Line number (``int``). If is is ``None`` or lesser than ``1``, it matches + any line number. + + .. attribute:: traceback + + If *traceback* is ``True``, all frames of the + :attr:`~tracemalloc.Trace.traceback` attribute of :class:`Trace` + instances are checked. If *traceback* is ``False``, only the most recent + frame is checked. + + This attribute only has an effect on the :meth:`match_trace` method and + only if the traceback limit is greater than ``1``. See the + :func:`get_traceback_limit` function. + + +Frame +----- + +.. class:: Frame + + Trace of a Python frame, used by :attr:`Trace.traceback` attribute. + + .. attribute:: filename + + Python filename, ``None`` if unknown. + + .. attribute:: lineno + + Python line number, ``None`` if unknown. + + +GroupedStats +------------ + +.. class:: GroupedStats(stats: dict, group_by: str, cumulative=False, timestamp=None, process_memory=None, tracemalloc_size=None) + + Top of allocated memory blocks grouped by on *group_by* as a dictionary. + + The :meth:`Snapshot.top_by` method creates a :class:`GroupedStats` instance. + + .. method:: compare_to(old_stats: GroupedStats=None) + + Compare to an older :class:`GroupedStats` instance. + Return a :class:`StatsDiff` instance. + + .. attribute:: cumulative + + If ``True``, cumulate size and count of memory blocks of all frames of + :class:`~tracemalloc.Trace`, not only the most recent frame. + + .. attribute:: group_by + + Determine how memory allocations were grouped. The type of :attr:`stats` + keys depends on *group_by*: + + ===================== ======================== ============== + group_by description key type + ===================== ======================== ============== + ``'filename'`` filename ``str`` + ``'filename_lineno'`` filename and line number ``(str, str)`` + ``'address'`` memory block address ``int`` + ===================== ======================== ============== + + See the *group_by* parameter of the :meth:`Snapshot.top_by` method. + + .. attribute:: stats + + Dictionary ``{key: stats}`` where the *key* type depends on the + :attr:`group_by` attribute and *stats* type is :class:`TraceStats`. + + .. attribute:: process_memory + + Result of the :func:`get_process_memory` function, can be ``None``. + + .. attribute:: timestamp + + Creation date and time of the snapshot, :class:`datetime.datetime` + instance. + + .. attribute:: tracemalloc_size + + The memory usage in bytes of the :mod:`tracemalloc` module, result of + the :func:`get_tracemalloc_size` function. + + +Snapshot +-------- + +.. class:: Snapshot + + Snapshot of memory blocks allocated by Python. + + Use :class:`TakeSnapshot` to take regulary snapshots. + + .. method:: apply_filters(filters) + + Apply a list filters on the :attr:`traces` and :attr:`stats` + dictionaries, *filters* is a list of :class:`Filter` instances. + + .. classmethod:: create(\*, with_traces=False, with_stats=True, user_data_callback=None) + + Take a snapshot of traces and/or statistics of allocated memory blocks. + + If *with_traces* is ``True``, :func:`get_traces` is called and its result + is stored in the :attr:`Snapshot.traces` attribute. This attribute + contains more information than :attr:`Snapshot.stats` and uses more + memory and more disk space. If *with_traces* is ``False``, + :attr:`Snapshot.traces` is set to ``None``. + + If *with_stats* is ``True``, :func:`get_stats` is called and its result + is stored in the :attr:`Snapshot.stats` attribute. If *with_stats* is + ``False``, :attr:`Snapshot.stats` is set to ``None``. + + *with_traces* and *with_stats* cannot be ``False`` at the same time. + + *user_data_callback* is an optional callable object. Its result should be + serializable by the :mod:`pickle` module, or :meth:`Snapshot.write` would + fail. If *user_data_callback* is set, it is called and the result is + stored in the :attr:`Snapshot.user_data` attribute. Otherwise, + :attr:`Snapshot.user_data` is set to ``None``. + + The :mod:`tracemalloc` module must be enabled to take a snapshot. See the + :func:`enable` function. + + .. classmethod:: load(filename) + + Load a snapshot from a file. + + .. method:: top_by(group_by: str, cumulative: bool=False) + + Compute top statistics grouped by *group_by* as a :class:`GroupedStats` + instance: + + ===================== ======================== ============== + group_by description key type + ===================== ======================== ============== + ``'filename'`` filename ``str`` + ``'filename_lineno'`` filename and line number ``(str, str)`` + ``'address'`` memory block address ``int`` + ===================== ======================== ============== + + If *cumulative* is ``True``, cumulate size and count of memory blocks of + all frames of each :class:`Trace` instance, not only the most recent + frame. The *cumulative* parameter is ignored if *group_by* is + ``'address'`` or if the traceback limit is ``1``. See the + :attr:`~Snapshot.traceback_limit` attribute. + + .. method:: write(filename) + + Write the snapshot into a file. + + .. attribute:: pid + + Identifier of the process which created the snapshot, result of + :func:`os.getpid`. + + .. attribute:: process_memory + + Memory usage of the current process, result of the + :func:`get_process_memory` function. It can be ``None``. + + .. attribute:: stats + + Statistics on traced Python memory, result of the :func:`get_stats` + function, if :meth:`create` was called with *with_stats* equals to + ``True``, ``None`` otherwise.:w + + .. attribute:: tracemalloc_size + + The memory usage in bytes of the :mod:`tracemalloc` module, result of + the :func:`get_tracemalloc_size` function. + + .. attribute:: traceback_limit + + The maximum number of frames stored in the + :attr:`~tracemalloc.tracemalloc.Trace.traceback` attribute of a + :class:`~tracemalloc.Trace`, result of the :func:`get_traceback_limit` + function. + + .. attribute:: traces + + Traces of Python memory allocations, result of the :func:`get_traces` + function, if :meth:`create` was called with *with_traces* equals to + ``True``, ``None`` otherwise. + + The :attr:`~Trace.traceback` attribute of each :class:`Trace` instance is + limited to :attr:`~Snapshot.traceback_limit` frames. + + .. attribute:: timestamp + + Creation date and time of the snapshot, :class:`datetime.datetime` + instance. + + .. attribute:: user_data + + Result of *user_data_callback* called in :meth:`Snapshot.create` + (default: ``None``). + + +StatsDiff +--------- + +.. class:: StatsDiff(differences, old_stats, new_stats) + + Differences between two :class:`GroupedStats` instances. By default, the + :attr:`differences` list is unsorted: call :meth:`sort` to sort it. + + The :meth:`GroupedStats.compare_to` method creates a :class:`StatsDiff` + instance. + + .. method:: sort() + + Sort the :attr:`differences` list from the biggest allocation to the + smallest. Sort by *size_diff*, *size*, *count_diff*, *count* and then by + *key*. + + .. attribute:: differences + + Differences between :attr:`old_stats` and :attr:`new_stats` as a list of + ``(size_diff, size, count_diff, count, key)`` tuples. *size_diff*, + *size*, *count_diff* and *count* are ``int``. The key type depends on the + :attr:`~GroupedStats.group_by` attribute of :attr:`new_stats`: + + ===================== ======================== ============== + group_by description key type + ===================== ======================== ============== + ``'filename'`` filename ``str`` + ``'filename_lineno'`` filename and line number ``(str, str)`` + ``'address'`` memory block address ``int`` + ===================== ======================== ============== + + See the :attr:`~GroupedStats.group_by` attribute of the + :class:`GroupedStats` class. + + .. attribute:: old_stats + + Old :class:`GroupedStats` instance, can be ``None``. + + .. attribute:: new_stats + + New :class:`GroupedStats` instance. + + +Trace +----- + +.. class:: Trace + + Debug information of a memory block allocated by Python. + + .. attribute:: size + + Size in bytes of the memory block. + + .. attribute:: traceback + + Traceback where the memory block was allocated as a list of + :class:`~tracemalloc.Frame` instances, most recent first. + + The list can be empty or incomplete if the :mod:`tracemalloc` module was + unable to retrieve the full traceback. + + The traceback is limited to :func:`get_traceback_limit` frames. Use + :func:`set_traceback_limit` to store more frames. + + +TraceStats +---------- + +.. class:: TraceStats + + Statistics on Python memory allocations. + + .. attribute:: size + + Total size in bytes of allocated memory blocks. + + .. attribute:: count + + Number of allocated memory blocks. + + +TakeSnapshot +------------ + +.. class:: TakeSnapshot + + Task taking snapshots of Python memory allocations and writing them into + files. + + .. method:: start(delay: int) + + Start a task calling the :meth:`take_snapshot` method every *delay* + seconds using the :func:`start_timer` function. + + .. method:: stop() + + Stop the task started by the :meth:`~TakeSnapshot.start` method using the + :func:`stop_timer` function. + + .. method:: take_snapshot() + + Take a snapshot and write it into a file + + .. attribute:: filename_template + + Template to create a filename. The template supports the following + variables: + + * ``$pid``: identifier of the current process + * ``$timestamp``: current date and time + * ``$counter``: counter starting at 1 and incremented at each snapshot + formatted as 4 digits + + The default template is ``tracemalloc-$counter.pickle``. + + .. attribute:: user_data_callback + + Optional callback collecting user data (callable, default: None). + See :meth:`Snapshot.create`. + + .. attribute:: with_traces + + Parameter passed to :meth:`Snapshot.create`. + + +Command line options +==================== + +The ``python -m tracemalloc`` command can be used to display, analyze and +compare snapshots. + +The command has the following options. + +``-a``, ``--address`` option: + + Group memory allocations by address, instead of grouping by line number. + +``-f``, ``--file`` option: + + Group memory allocations per filename, instead of grouping by line number. + +``-n NUMBER``, ``--number NUMBER`` option: + + Number of traces displayed per top (default: 10). + +``--first`` option: + + Compare with the first snapshot, instead of comparing with the + previous snapshot. + +``-c``, ``--cumulative`` option: + + Cumulate size and count of allocated memory blocks using all frames, not + only the most recent frame. The option has only an effect if the snapshot + contains traces and if the traceback limit was greater than ``1``. + +``-b ADDRESS``, ``--block=ADDRESS`` option: + + Get the memory block at address *ADDRESS*, display its size and the + traceback where it was allocated. + + The option can only be used on snapshots created with traces. + +``-t``, ``--traceback`` option: + + Group memory allocations by address, display the size and the traceback + of the *NUMBER* biggest allocated memory blocks. + + The option can only be used on snapshots created with traces. By default, + the traceback limit is ``1`` frame: set a greater limit with + :func:`set_traceback_limit` before taking snapshots to get more frames. + + See the ``--number`` option for *NUMBER*. + +``-i FILENAME[:LINENO]``, ``--include FILENAME[:LINENO]`` option: + + Only include traces of files with a name matching *FILENAME* pattern at + line number *LINENO*. Only check the most recent frame. The option can be + specified multiple times. + + See :func:`add_include_filter` for the syntax of a filter. + +``-I FILENAME[:LINENO]``, ``--include-traceback FILENAME[:LINENO]`` option: + + Similar to ``--include`` option, but check all frames of the traceback. + +``-x FILENAME[:LINENO]``, ``--exclude FILENAME[:LINENO]`` option: + + Exclude traces of files with a name matching *FILENAME* pattern at line + number *LINENO*. Only check the most recent frame. The option can be + specified multiple times. + + See :func:`add_exclude_filter` for the syntax of a filter. + +``-X FILENAME[:LINENO]``, ``--exclude-traceback FILENAME[:LINENO]`` option: + + Similar to ``--exclude`` option, but check all frames of the traceback. + +``-S``, ``--hide-size`` option: + + Hide the size of allocations. + +``-C``, ``--hide-count`` option: + + Hide the number of allocations. + +``-A``, ``--hide-average`` option: + + Hide the average size of allocations. + +``-P PARTS``, ``--filename-parts=PARTS`` option: + + Number of displayed filename parts (default: 3). + +``--color`` option: + + Always use colors, even if :data:`sys.stdout` is not a TTY device. + +``--no-color`` option: + + Never use colors, even if :data:`sys.stdout` is a TTY device. + diff -r df2fdd42b375 -r 21f7c3df0f15 Doc/using/cmdline.rst --- a/Doc/using/cmdline.rst Mon Aug 26 22:28:21 2013 +0200 +++ b/Doc/using/cmdline.rst Tue Sep 17 00:59:50 2013 +0200 @@ -381,6 +381,7 @@ Miscellaneous options * ``-X faulthandler`` to enable :mod:`faulthandler`; * ``-X showrefcount`` to enable the output of the total reference count and memory blocks (only works on debug builds); + * ``-X tracemalloc`` to enable :mod:`tracemalloc`. It also allows to pass arbitrary values and retrieve them through the :data:`sys._xoptions` dictionary. @@ -392,7 +393,7 @@ Miscellaneous options The ``-X faulthandler`` option. .. versionadded:: 3.4 - The ``-X showrefcount`` option. + The ``-X showrefcount`` and ``-X tracemalloc`` options. Options you shouldn't use @@ -591,6 +592,14 @@ conflict. .. versionadded:: 3.3 +.. envvar:: PYTHONTRACEMALLOC + + If this environment variable is set to a non-empty string, all memory + allocations made by Python are traced by the :mod:`tracemalloc` module. + + .. versionadded:: 3.4 + + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff -r df2fdd42b375 -r 21f7c3df0f15 Include/tracemalloc.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Include/tracemalloc.h Tue Sep 17 00:59:50 2013 +0200 @@ -0,0 +1,11 @@ +#ifndef Py_TRACEMALLOC_H +#define Py_TRACEMALLOC_H + +#include "Python.h" + +PyAPI_FUNC(int) PyTraceMalloc_Init(void); +PyAPI_FUNC(int) PyTraceMalloc_Enable(void); +PyAPI_FUNC(int) PyTraceMalloc_DisableTemporary(void); +PyAPI_FUNC(void) PyTraceMalloc_RestoreTemporary(int was_enabled); + +#endif /* Py_TRACEMALLOC_H */ diff -r df2fdd42b375 -r 21f7c3df0f15 Lib/test/regrtest.py --- a/Lib/test/regrtest.py Mon Aug 26 22:28:21 2013 +0200 +++ b/Lib/test/regrtest.py Tue Sep 17 00:59:50 2013 +0200 @@ -372,6 +372,25 @@ def main(tests=None, testdir=None, verbo directly to set the values that would normally be set by flags on the command line. """ + import tracemalloc + if tracemalloc.is_enabled(): + tracemalloc.add_exclude_filter(tracemalloc.__file__) + if 0: + tracemalloc.add_include_filter("", traceback=True) + tracemalloc.set_traceback_limit(15) + tracemalloc.clear_traces() + + if 0: + #top = tracemalloc.DisplayTop(25, group_per_file=True) + top = tracemalloc.DisplayTop(25) + top.start(1) + elif 1: + take = tracemalloc.TakeSnapshot() + take.with_traces = True + #take.with_stats = False + take.filename_template = "/tmp/tracemalloc-$pid-$counter.pickle" + take.start(10) + # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) diff -r df2fdd42b375 -r 21f7c3df0f15 Lib/test/test_tracemalloc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_tracemalloc.py Tue Sep 17 00:59:50 2013 +0200 @@ -0,0 +1,906 @@ +import datetime +import io +import os +import sys +import time +import tracemalloc +import unittest +from test.script_helper import assert_python_ok +from test import support + +EMPTY_STRING_SIZE = sys.getsizeof(b'') + +def get_frames(nframe, lineno_delta): + frames = [] + frame = sys._getframe(1) + for index in range(nframe): + code = frame.f_code + lineno = frame.f_lineno + lineno_delta + frame_trace = tracemalloc.Frame((code.co_filename, lineno)) + frames.append(frame_trace) + lineno_delta = 0 + frame = frame.f_back + return frames + +def allocate_bytes(size): + nframe = tracemalloc.get_traceback_limit() + frames = get_frames(nframe, 1) + data = b'x' * (size - EMPTY_STRING_SIZE) + return data, frames + + +class TestTracemallocEnabled(unittest.TestCase): + def setUp(self): + if tracemalloc.is_enabled(): + self.skipTest("tracemalloc must be disabled before the test") + tracemalloc.clear_filters() + tracemalloc.enable() + + def tearDown(self): + tracemalloc.disable() + tracemalloc.clear_filters() + + def test_get_object_trace(self): + tracemalloc.clear_traces() + size = 12345 + obj, obj_frames = allocate_bytes(size) + trace = tracemalloc.get_object_trace(obj) + self.assertIsInstance(trace, tracemalloc.Trace) + self.assertEqual(trace.size, size) + self.assertEqual(trace.traceback, obj_frames) + + def test_set_traceback_limit(self): + size = 10 + + nframe = tracemalloc.get_traceback_limit() + self.addCleanup(tracemalloc.set_traceback_limit, nframe) + + self.assertRaises(ValueError, tracemalloc.set_traceback_limit, -1) + self.assertRaises(ValueError, tracemalloc.set_traceback_limit, 0) + + tracemalloc.set_traceback_limit(1) + obj, obj_frames = allocate_bytes(size) + trace = tracemalloc.get_object_trace(obj) + self.assertEqual(len(trace.traceback), 1) + self.assertEqual(trace.traceback, obj_frames) + + tracemalloc.set_traceback_limit(10) + obj2, obj2_frames = allocate_bytes(size) + trace = tracemalloc.get_object_trace(obj2) + self.assertEqual(len(trace.traceback), 10) + self.assertEqual(trace.traceback, obj2_frames) + + def test_get_traces(self): + tracemalloc.clear_traces() + size = 12345 + obj, obj_frames = allocate_bytes(size) + traces = tracemalloc.get_traces() + + address = tracemalloc.get_object_address(obj) + self.assertIn(address, traces) + trace = traces[address] + + self.assertIsInstance(trace, tracemalloc.Trace) + self.assertEqual(trace.size, size) + self.assertEqual(trace.traceback, obj_frames) + + def test_get_process_memory(self): + tracemalloc.clear_traces() + obj_size = 5 * 1024 * 1024 + + orig = tracemalloc.get_process_memory() + if orig is None: + self.skipTest("get_process_memory is not supported") + self.assertGreater(orig.rss, 0) + self.assertGreater(orig.vms, 0) + + obj, obj_frames = allocate_bytes(obj_size) + curr = tracemalloc.get_process_memory() + self.assertGreaterEqual(curr.rss, orig.rss) + self.assertGreaterEqual(curr.vms, orig.vms) + + def test_get_traced_memory(self): + # get the allocation location to filter allocations + size = 12345 + obj, frames = allocate_bytes(size) + tracemalloc.add_include_filter(frames[0].filename, frames[0].lineno) + + # allocate one object + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + self.assertEqual(tracemalloc.get_traced_memory(), size) + + # clear_traces() must reset the python memory counter + tracemalloc.clear_traces() + self.assertEqual(tracemalloc.get_traced_memory(), 0) + + # allocate another object + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + self.assertEqual(tracemalloc.get_traced_memory(), size) + + # tracemalloc must be enabled to get the python memory + tracemalloc.disable() + self.assertEqual(tracemalloc.get_traced_memory(), 0) + + def test_get_stats(self): + tracemalloc.clear_traces() + total = 0 + count = 0 + objs = [] + for index in range(5): + size = 1234 + obj, obj_frames = allocate_bytes(size) + objs.append(obj) + total += size + count += 1 + + stats = tracemalloc.get_stats() + for filename, line_stats in stats.items(): + for lineno, line_stat in line_stats.items(): + # stats can be huge, one test per file should be enough + self.assertIsInstance(line_stat, tracemalloc.TraceStats) + break + + frame = obj_frames[0] + self.assertIn(frame.filename, stats) + line_stats = stats[frame.filename] + self.assertIn(frame.lineno, line_stats) + line_stat = line_stats[frame.lineno] + self.assertEqual(line_stat.size, total) + self.assertEqual(line_stat.count, count) + + def test_clear_traces(self): + tracemalloc.clear_traces() + size = 1234 + obj, obj_frames = allocate_bytes(size) + + stats = tracemalloc.get_stats() + frame = obj_frames[0] + line_stats = stats[frame.filename][frame.lineno] + self.assertEqual(line_stats.size, size) + self.assertEqual(line_stats.count, 1) + + tracemalloc.clear_traces() + stats2 = tracemalloc.get_stats() + self.assertNotIn(frame.lineno, stats2[frame.filename]) + + def test_timer(self): + calls = [] + def func(*args, **kw): + calls.append((args, kw)) + + # timer enabled + args = (1, 2, 3) + kwargs = {'arg': 4} + tracemalloc.start_timer(1, func, args, kwargs) + time.sleep(1) + obj, source = allocate_bytes(123) + self.assertEqual(len(calls), 1) + call = calls[0] + self.assertEqual(call, (args, kwargs)) + + # timer disabled + tracemalloc.stop_timer() + time.sleep(1) + obj2, source2 = allocate_bytes(123) + self.assertEqual(len(calls), 1) + + def test_is_enabled(self): + tracemalloc.clear_traces() + tracemalloc.disable() + self.assertFalse(tracemalloc.is_enabled()) + + tracemalloc.enable() + self.assertTrue(tracemalloc.is_enabled()) + + def test_snapshot(self): + def compute_nstats(stats): + return sum(len(line_stats) + for filename, line_stats in stats.items()) + + stats1 = tracemalloc.get_stats() + nstat1 = compute_nstats(stats1) + + # take a snapshot with traces + snapshot = tracemalloc.Snapshot.create(with_traces=True) + nstat2 = compute_nstats(snapshot.stats) + self.assertGreaterEqual(nstat2, nstat2) + self.assertEqual(snapshot.pid, os.getpid()) + if snapshot.process_memory is not None: + self.assertGreater(snapshot.process_memory.rss, 0) + self.assertGreater(snapshot.process_memory.vms, 0) + self.assertIsNone(snapshot.user_data) + + snapshot.write(support.TESTFN) + self.addCleanup(support.unlink, support.TESTFN) + snapshot2 = tracemalloc.Snapshot.load(support.TESTFN) + + self.assertEqual(snapshot2.timestamp, snapshot.timestamp) + self.assertEqual(snapshot2.pid, snapshot.pid) + self.assertEqual(snapshot2.traces, snapshot.traces) + self.assertEqual(snapshot2.stats, snapshot.stats) + self.assertEqual(snapshot2.process_memory, snapshot.process_memory) + self.assertEqual(snapshot2.tracemalloc_size, snapshot.tracemalloc_size) + self.assertEqual(snapshot2.traceback_limit, snapshot.traceback_limit) + self.assertEqual(snapshot2.user_data, snapshot.user_data) + + # without traces + snapshot2 = tracemalloc.Snapshot.create() + self.assertIsNone(snapshot2.traces) + + # tracemalloc must be enabled to take a snapshot + tracemalloc.disable() + with self.assertRaises(RuntimeError) as cm: + tracemalloc.Snapshot.create() + self.assertEqual(str(cm.exception), + "the tracemalloc module must be enabled " + "to take a snapshot") + + def test_take_snapshot(self): + with support.temp_cwd() as temp_dir: + take_snapshot = tracemalloc.TakeSnapshot() + for index in range(1, 4): + snapshot, filename = take_snapshot.take_snapshot() + self.assertEqual(filename, + 'tracemalloc-%04d.pickle' % index) + self.assertTrue(os.path.exists(filename)) + + def test_filters(self): + # test multiple inclusive filters + tracemalloc.add_include_filter('should never match 1') + tracemalloc.add_include_filter('should never match 2') + tracemalloc.add_include_filter(__file__) + tracemalloc.clear_traces() + size = 1000 + obj, obj_frames = allocate_bytes(size) + trace = tracemalloc.get_object_trace(obj) + self.assertIsInstance(trace, tracemalloc.Trace) + + # test exclusive filter, based on previous filters + frame = obj_frames[0] + tracemalloc.add_exclude_filter(frame.filename, frame.lineno) + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + trace = tracemalloc.get_object_trace(obj) + self.assertIsNone(trace) + + +class TestFilters(unittest.TestCase): + maxDiff = 2048 + def test_add_clear_filter(self): + old_filters = tracemalloc.get_filters() + try: + # test add_include_filter(), add_exclude_filter() + tracemalloc.clear_filters() + tracemalloc.add_include_filter("abc", 3) + tracemalloc.add_exclude_filter("12345", 0) + tracemalloc.add_exclude_filter("6789", None) + tracemalloc.add_exclude_filter("def#", 55) + tracemalloc.add_exclude_filter("trace", 123, True) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abc', 3, False), + tracemalloc.Filter(False, '12345', None, False), + tracemalloc.Filter(False, '6789', None, False), + tracemalloc.Filter(False, "def#", 55, False), + tracemalloc.Filter(False, "trace", 123, True)]) + + # test filename normalization (.pyc/.pyo) + tracemalloc.clear_filters() + tracemalloc.add_include_filter("abc.pyc") + tracemalloc.add_include_filter("name.pyo") + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abc.py', None, False), + tracemalloc.Filter(True, 'name.py', None, False) ]) + + # test filename normalization ('*' joker character) + tracemalloc.clear_filters() + tracemalloc.add_include_filter('a****b') + tracemalloc.add_include_filter('***x****') + tracemalloc.add_include_filter('1*2**3***4') + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'a*b', None, False), + tracemalloc.Filter(True, '*x*', None, False), + tracemalloc.Filter(True, '1*2*3*4', None, False)]) + + # Windows: test filename normalization (lower case, slash) + if os.name == "nt": + tracemalloc.clear_filters() + tracemalloc.add_include_filter("aBcD\xC9") + tracemalloc.add_include_filter("MODule.PYc") + tracemalloc.add_include_filter(r"path/to\file") + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abcd\xe9', None, False), + tracemalloc.Filter(True, 'module.py', None, False), + tracemalloc.Filter(True, r'path\to\file', None, False)]) + + # test clear_filters() + tracemalloc.clear_filters() + self.assertEqual(tracemalloc.get_filters(), []) + finally: + tracemalloc.clear_filters() + for filter_args in old_filters: + tracemalloc.add_filter(*filter_args) + + def test_trace_filter(self): + # test default values + f = tracemalloc.Filter(True, "abc") + self.assertEqual(f.include, True) + self.assertEqual(f.pattern, "abc") + self.assertIsNone(f.lineno) + self.assertEqual(f.traceback, False) + + f = tracemalloc.Filter(False, "test.py", 123, True) + self.assertEqual(f.include, False) + self.assertEqual(f.pattern, "test.py") + self.assertEqual(f.lineno, 123) + self.assertEqual(f.traceback, True) + + def test_filter_match(self): + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f.match("abc", 5)) + self.assertTrue(f.match("abc", None)) + self.assertFalse(f.match("12356", 5)) + self.assertFalse(f.match("12356", None)) + self.assertFalse(f.match(None, 5)) + self.assertFalse(f.match(None, None)) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f.match("abc", 5)) + self.assertFalse(f.match("abc", None)) + self.assertTrue(f.match("12356", 5)) + self.assertTrue(f.match("12356", None)) + self.assertTrue(f.match(None, 5)) + self.assertTrue(f.match(None, None)) + + f = tracemalloc.Filter(True, "abc", 5) + self.assertTrue(f.match("abc", 5)) + self.assertFalse(f.match("abc", 10)) + self.assertFalse(f.match("abc", None)) + self.assertFalse(f.match("12356", 5)) + self.assertFalse(f.match("12356", 10)) + self.assertFalse(f.match("12356", None)) + self.assertFalse(f.match(None, 5)) + self.assertFalse(f.match(None, 10)) + self.assertFalse(f.match(None, None)) + + f = tracemalloc.Filter(False, "abc", 5) + self.assertFalse(f.match("abc", 5)) + self.assertTrue(f.match("abc", 10)) + self.assertTrue(f.match("abc", None)) + self.assertTrue(f.match("12356", 5)) + self.assertTrue(f.match("12356", 10)) + self.assertTrue(f.match("12356", None)) + self.assertTrue(f.match(None, 5)) + self.assertTrue(f.match(None, 10)) + self.assertTrue(f.match(None, None)) + + def test_filter_match_lineno(self): + f = tracemalloc.Filter(True, "unused") + self.assertTrue(f.match_lineno(5)) + self.assertTrue(f.match_lineno(10)) + self.assertTrue(f.match_lineno(None)) + + f = tracemalloc.Filter(True, "unused", 5) + self.assertTrue(f.match_lineno(5)) + self.assertFalse(f.match_lineno(10)) + self.assertFalse(f.match_lineno(None)) + + f = tracemalloc.Filter(False, "unused") + self.assertTrue(f.match_lineno(5)) + self.assertTrue(f.match_lineno(10)) + self.assertTrue(f.match_lineno(None)) + + f = tracemalloc.Filter(False, "unused", 5) + self.assertFalse(f.match_lineno(5)) + self.assertTrue(f.match_lineno(10)) + self.assertTrue(f.match_lineno(None)) + + def test_filter_match_filename(self): + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f.match_filename("abc")) + self.assertFalse(f.match_filename("12356")) + self.assertFalse(f.match_filename(None)) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f.match_filename("abc")) + self.assertTrue(f.match_filename("12356")) + self.assertTrue(f.match_filename(None)) + + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f.match_filename("abc")) + self.assertFalse(f.match_filename("12356")) + self.assertFalse(f.match_filename(None)) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f.match_filename("abc")) + self.assertTrue(f.match_filename("12356")) + self.assertTrue(f.match_filename(None)) + + def test_filter_match_filename_joker(self): + def fnmatch(filename, pattern): + filter = tracemalloc.Filter(True, pattern) + return filter.match_filename(filename) + + # no * + self.assertTrue(fnmatch('abc', 'abc')) + self.assertFalse(fnmatch('abc', 'abcd')) + self.assertFalse(fnmatch('abc', 'def')) + + # a* + self.assertTrue(fnmatch('abc', 'a*')) + self.assertTrue(fnmatch('abc', 'abc*')) + self.assertFalse(fnmatch('abc', 'b*')) + self.assertFalse(fnmatch('abc', 'abcd*')) + + # a*b + self.assertTrue(fnmatch('abc', 'a*c')) + self.assertTrue(fnmatch('abcdcx', 'a*cx')) + self.assertFalse(fnmatch('abb', 'a*c')) + self.assertFalse(fnmatch('abcdce', 'a*cx')) + + # a*b*c + self.assertTrue(fnmatch('abcde', 'a*c*e')) + self.assertTrue(fnmatch('abcbdefeg', 'a*bd*eg')) + self.assertFalse(fnmatch('abcdd', 'a*c*e')) + self.assertFalse(fnmatch('abcbdefef', 'a*bd*eg')) + + # replace .pyc and .pyo suffix with .py + self.assertTrue(fnmatch('a.pyc', 'a.py')) + self.assertTrue(fnmatch('a.pyo', 'a.py')) + self.assertTrue(fnmatch('a.py', 'a.pyc')) + self.assertTrue(fnmatch('a.py', 'a.pyo')) + + if os.name == 'nt': + # case insensitive + self.assertTrue(fnmatch('aBC', 'ABc')) + self.assertTrue(fnmatch('aBcDe', 'Ab*dE')) + + self.assertTrue(fnmatch('a.pyc', 'a.PY')) + self.assertTrue(fnmatch('a.PYO', 'a.py')) + self.assertTrue(fnmatch('a.py', 'a.PYC')) + self.assertTrue(fnmatch('a.PY', 'a.pyo')) + else: + # case sensitive + self.assertFalse(fnmatch('aBC', 'ABc')) + self.assertFalse(fnmatch('aBcDe', 'Ab*dE')) + + self.assertFalse(fnmatch('a.pyc', 'a.PY')) + self.assertFalse(fnmatch('a.PYO', 'a.py')) + self.assertFalse(fnmatch('a.py', 'a.PYC')) + self.assertFalse(fnmatch('a.PY', 'a.pyo')) + + if os.name == 'nt': + # normalize alternate separator "/" to the standard separator "\" + self.assertTrue(fnmatch(r'a/b', r'a\b')) + self.assertTrue(fnmatch(r'a\b', r'a/b')) + self.assertTrue(fnmatch(r'a/b\c', r'a\b/c')) + self.assertTrue(fnmatch(r'a/b/c', r'a\b\c')) + else: + # there is no alternate separator + self.assertFalse(fnmatch(r'a/b', r'a\b')) + self.assertFalse(fnmatch(r'a\b', r'a/b')) + self.assertFalse(fnmatch(r'a/b\c', r'a\b/c')) + self.assertFalse(fnmatch(r'a/b/c', r'a\b\c')) + + # a******b + N = 10 ** 6 + self.assertTrue (fnmatch('a' * N, '*' * N)) + self.assertTrue (fnmatch('a' * N + 'c', '*' * N)) + self.assertTrue (fnmatch('a' * N, 'a' + '*' * N + 'a')) + self.assertTrue (fnmatch('a' * N + 'b', 'a' + '*' * N + 'b')) + self.assertFalse(fnmatch('a' * N + 'b', 'a' + '*' * N + 'c')) + + # a*a*a*a* + self.assertTrue(fnmatch('a' * 10, 'a*' * 10)) + self.assertFalse(fnmatch('a' * 10, 'a*' * 10 + 'b')) + with self.assertRaises(ValueError) as cm: + fnmatch('abc', 'a*' * 101) + self.assertEqual(str(cm.exception), + "too many joker characters in the filename pattern") + + def test_filter_match_trace(self): + t1 = tracemalloc.Trace((123, [ + tracemalloc.Frame(("a.py", 2)), + tracemalloc.Frame(("b.py", 3))])) + + t2 = tracemalloc.Trace((123, [ + tracemalloc.Frame(("b.py", 4)), + tracemalloc.Frame(("b.py", 5))])) + + f = tracemalloc.Filter(True, "b.py", traceback=True) + self.assertTrue(f.match_trace(t1)) + self.assertTrue(f.match_trace(t2)) + + f = tracemalloc.Filter(True, "b.py", traceback=False) + self.assertFalse(f.match_trace(t1)) + self.assertTrue(f.match_trace(t2)) + + f = tracemalloc.Filter(False, "b.py", traceback=True) + self.assertFalse(f.match_trace(t1)) + self.assertFalse(f.match_trace(t2)) + + f = tracemalloc.Filter(False, "b.py", traceback=False) + self.assertTrue(f.match_trace(t1)) + self.assertFalse(f.match_trace(t2)) + + +class TestCommandLine(unittest.TestCase): + def test_env_var(self): + # disabled by default + code = 'import tracemalloc; print(tracemalloc.is_enabled())' + ok, stdout, stderr = assert_python_ok('-c', code) + stdout = stdout.rstrip() + self.assertEqual(stdout, b'False') + + # PYTHON* environment varibles must be ignored when -E option is + # present + code = 'import tracemalloc; print(tracemalloc.is_enabled())' + ok, stdout, stderr = assert_python_ok('-E', '-c', code, PYTHONTRACEMALLOC='1') + stdout = stdout.rstrip() + self.assertEqual(stdout, b'False') + + # enabled by default + code = 'import tracemalloc; print(tracemalloc.is_enabled())' + ok, stdout, stderr = assert_python_ok('-c', code, PYTHONTRACEMALLOC='1') + stdout = stdout.rstrip() + self.assertEqual(stdout, b'True') + + def test_sys_xoptions(self): + # -X tracemalloc + code = 'import tracemalloc; print(tracemalloc.is_enabled())' + ok, stdout, stderr = assert_python_ok('-X', 'tracemalloc', '-c', code) + stdout = stdout.rstrip() + self.assertEqual(stdout, b'True') + + +class TestTop(unittest.TestCase): + def create_snapshots(self): + pid = 123 + traceback_limit = 5 + user_data = {} + + timestamp = datetime.datetime(2013, 9, 12, 15, 16, 17) + stats = { + 'a.py': {2: tracemalloc.TraceStats((10, 100)), + 5: tracemalloc.TraceStats((127, 2))}, + None: {None: tracemalloc.TraceStats((55, 1))}, + 'b.py': {None: tracemalloc.TraceStats((66, 1))}, + } + tracemalloc_size = 100 + process_memory = tracemalloc._meminfo(1024, 2048) + + timestamp2 = datetime.datetime(2013, 9, 12, 15, 16, 50) + stats2 = { + 'a.py': {2: tracemalloc.TraceStats((10, 100)), + 5: tracemalloc.TraceStats((200, 2))}, + 'b.py': {None: tracemalloc.TraceStats((66, 1))}, + } + process_memory2 = tracemalloc._meminfo(1500, 2048) + tracemalloc_size2 = 200 + snapshot = tracemalloc.Snapshot(timestamp, pid, None, stats, + process_memory, tracemalloc_size, + traceback_limit, user_data) + snapshot2 = tracemalloc.Snapshot(timestamp2, pid, None, stats2, + process_memory2, tracemalloc_size2, + traceback_limit, user_data) + return (snapshot, snapshot2) + + def create_snapshot_with_traces(self): + pid = 123 + traceback_limit = 5 + user_data = {} + + traceback_a_2 = [tracemalloc.Frame(('a.py', 2)), + tracemalloc.Frame(('b.py', 4))] + traceback_a_5 = [tracemalloc.Frame(('a.py', 5)), + tracemalloc.Frame(('b.py', 4))] + traceback_b_1 = [tracemalloc.Frame(('b.py', 1))] + traceback_none_none = [tracemalloc.Frame((None, None))] + + timestamp = datetime.datetime(2013, 9, 12, 15, 16, 17) + traces = { + 0x10001: tracemalloc.Trace((10, traceback_a_2)), + 0x10002: tracemalloc.Trace((10, traceback_a_2)), + 0x10003: tracemalloc.Trace((10, traceback_a_2)), + + 0x20001: tracemalloc.Trace((2, traceback_a_5)), + + 0x30001: tracemalloc.Trace((66, traceback_b_1)), + + 0x40001: tracemalloc.Trace((2, traceback_none_none)), + } + tracemalloc_size = 100 + process_memory = tracemalloc._meminfo(1024, 2048) + snapshot = tracemalloc.Snapshot(timestamp, pid, traces, None, + process_memory, tracemalloc_size, + traceback_limit, user_data) + + timestamp2 = datetime.datetime(2013, 9, 12, 15, 16, 50) + traces2 = { + 0x10001: tracemalloc.Trace((10, traceback_a_2)), + 0x10002: tracemalloc.Trace((10, traceback_a_2)), + 0x10003: tracemalloc.Trace((10, traceback_a_2)), + + 0x20001: tracemalloc.Trace((2, traceback_a_5)), + 0x20002: tracemalloc.Trace((5000, traceback_a_5)), + + 0x30001: tracemalloc.Trace((66, traceback_b_1)), + } + process_memory2 = tracemalloc._meminfo(1500, 2048) + tracemalloc_size2 = 200 + snapshot2 = tracemalloc.Snapshot(timestamp2, pid, traces2, None, + process_memory2, tracemalloc_size2, + traceback_limit, user_data) + return (snapshot, snapshot2) + + def test_snapshot_top_by_line(self): + snapshot, snapshot2 = self.create_snapshots() + + # stats per file and line + top_stats = snapshot.top_by('filename_lineno') + self.assertEqual(top_stats.stats, { + (None, None): tracemalloc.TraceStats((55, 1)), + ('a.py', 2): tracemalloc.TraceStats((10, 100)), + ('a.py', 5): tracemalloc.TraceStats((127, 2)), + ('b.py', None): tracemalloc.TraceStats((66, 1)), + }) + self.assertEqual(top_stats.group_by, 'filename_lineno') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, False) + self.assertEqual(top_stats.process_memory, snapshot.process_memory) + self.assertEqual(top_stats.tracemalloc_size, snapshot.tracemalloc_size) + + # stats per file and line (2) + top_stats2 = snapshot2.top_by('filename_lineno') + self.assertEqual(top_stats2.stats, { + ('a.py', 2): tracemalloc.TraceStats((10, 100)), + ('a.py', 5): tracemalloc.TraceStats((200, 2)), + ('b.py', None): tracemalloc.TraceStats((66, 1)), + }) + self.assertEqual(top_stats2.group_by, 'filename_lineno') + self.assertEqual(top_stats2.timestamp, snapshot2.timestamp) + self.assertEqual(top_stats2.cumulative, False) + self.assertEqual(top_stats2.process_memory, snapshot2.process_memory) + self.assertEqual(top_stats2.tracemalloc_size, snapshot2.tracemalloc_size) + + # stats diff per file and line + top_diff = top_stats2.compare_to(top_stats) + self.assertIsInstance(top_diff, tracemalloc.StatsDiff) + top_diff.sort() + self.assertEqual(top_diff.differences, [ + (73, 200, 0, 2, ('a.py', 5)), + (0, 66, 0, 1, ('b.py', 0)), + (0, 10, 0, 100, ('a.py', 2)), + (-55, 0, -1, 0, ('', 0)), + ]) + + def test_snapshot_top_by_file(self): + snapshot, snapshot2 = self.create_snapshots() + + # stats per file + top_stats = snapshot.top_by('filename') + self.assertEqual(top_stats.stats, { + None: tracemalloc.TraceStats((55, 1)), + 'a.py': tracemalloc.TraceStats((137, 102)), + 'b.py': tracemalloc.TraceStats((66, 1)), + }) + self.assertEqual(top_stats.group_by, 'filename') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, False) + self.assertEqual(top_stats.process_memory, snapshot.process_memory) + self.assertEqual(top_stats.tracemalloc_size, snapshot.tracemalloc_size) + + # stats per file (2) + top_stats2 = snapshot2.top_by('filename') + self.assertEqual(top_stats2.stats, { + 'a.py': tracemalloc.TraceStats((210, 102)), + 'b.py': tracemalloc.TraceStats((66, 1)), + }) + self.assertEqual(top_stats2.group_by, 'filename') + self.assertEqual(top_stats2.timestamp, snapshot2.timestamp) + self.assertEqual(top_stats2.cumulative, False) + self.assertEqual(top_stats2.process_memory, snapshot2.process_memory) + self.assertEqual(top_stats2.tracemalloc_size, snapshot2.tracemalloc_size) + + # stats diff per file + top_diff = top_stats2.compare_to(top_stats) + self.assertIsInstance(top_diff, tracemalloc.StatsDiff) + top_diff.sort() + self.assertEqual(top_diff.differences, [ + (73, 210, 0, 102, 'a.py'), + (0, 66, 0, 1, 'b.py'), + (-55, 0, -1, 0, ''), + ]) + + def test_snapshot_top_from_traces(self): + snapshot, snapshot2 = self.create_snapshot_with_traces() + + # stats per file and line + top_stats = snapshot.top_by('filename_lineno') + self.assertEqual(top_stats.stats, { + ('a.py', 2): tracemalloc.TraceStats((30, 3)), + (None, None): tracemalloc.TraceStats((2, 1)), + ('a.py', 5): tracemalloc.TraceStats((2, 1)), + ('b.py', 1): tracemalloc.TraceStats((66, 1)), + }) + self.assertEqual(top_stats.group_by, 'filename_lineno') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, False) + self.assertEqual(top_stats.process_memory, snapshot.process_memory) + self.assertEqual(top_stats.tracemalloc_size, snapshot.tracemalloc_size) + + # stats per file + top_stats = snapshot.top_by('filename') + self.assertEqual(top_stats.stats, { + None: tracemalloc.TraceStats((2, 1)), + 'a.py': tracemalloc.TraceStats((32, 4)), + 'b.py': tracemalloc.TraceStats((66, 1)), + }) + self.assertEqual(top_stats.group_by, 'filename') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, False) + self.assertEqual(top_stats.process_memory, snapshot.process_memory) + self.assertEqual(top_stats.tracemalloc_size, snapshot.tracemalloc_size) + + def test_snapshot_top_by_address(self): + snapshot, snapshot2 = self.create_snapshot_with_traces() + + # stats per address + top_stats = snapshot.top_by('address') + self.assertEqual(top_stats.stats, { + 65537: tracemalloc.TraceStats((10, 1)), + 65538: tracemalloc.TraceStats((10, 1)), + 65539: tracemalloc.TraceStats((10, 1)), + 131073: tracemalloc.TraceStats((2, 1)), + 196609: tracemalloc.TraceStats((66, 1)), + 262145: tracemalloc.TraceStats((2, 1)), + }) + self.assertEqual(top_stats.group_by, 'address') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, False) + self.assertEqual(top_stats.process_memory, snapshot.process_memory) + self.assertEqual(top_stats.tracemalloc_size, snapshot.tracemalloc_size) + + # stats per address (2) + top_stats2 = snapshot2.top_by('address') + self.assertEqual(top_stats2.stats, { + 65537: tracemalloc.TraceStats((10, 1)), + 65538: tracemalloc.TraceStats((10, 1)), + 65539: tracemalloc.TraceStats((10, 1)), + 131073: tracemalloc.TraceStats((2, 1)), + 131074: tracemalloc.TraceStats((5000, 1)), + 196609: tracemalloc.TraceStats((66, 1)), + }) + self.assertEqual(top_stats2.group_by, 'address') + self.assertEqual(top_stats2.timestamp, snapshot2.timestamp) + self.assertEqual(top_stats2.cumulative, False) + self.assertEqual(top_stats2.process_memory, snapshot2.process_memory) + self.assertEqual(top_stats2.tracemalloc_size, snapshot2.tracemalloc_size) + + # diff + top_diff = top_stats2.compare_to(top_stats) + self.assertIsInstance(top_diff, tracemalloc.StatsDiff) + top_diff.sort() + self.assertEqual(top_diff.differences, [ + (0, 5000, 0, 1, 131074), + (0, 66, 0, 1, 196609), + (0, 10, 0, 1, 65539), + (0, 10, 0, 1, 65538), + (0, 10, 0, 1, 65537), + (0, 2, 0, 1, 131073), + (-2, 0, -1, 0, 262145), + ]) + + snapshot, snapshot2 = self.create_snapshots() + with self.assertRaises(ValueError) as cm: + snapshot.top_by('address') + self.assertEqual(str(cm.exception), "need traces") + + def test_snapshot_top_cumulative(self): + snapshot, snapshot2 = self.create_snapshot_with_traces() + + # per file + top_stats = snapshot.top_by('filename', True) + self.assertEqual(top_stats.stats, { + None: tracemalloc.TraceStats((2, 1)), + 'a.py': tracemalloc.TraceStats((32, 4)), + 'b.py': tracemalloc.TraceStats((98, 5)), + }) + self.assertEqual(top_stats.group_by, 'filename') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, True) + self.assertEqual(top_stats.process_memory, snapshot.process_memory) + self.assertEqual(top_stats.tracemalloc_size, snapshot.tracemalloc_size) + + # per line + top_stats2 = snapshot.top_by('filename_lineno', True) + self.assertEqual(top_stats2.stats, { + (None, None): tracemalloc.TraceStats((2, 1)), + ('a.py', 2): tracemalloc.TraceStats((30, 3)), + ('a.py', 5): tracemalloc.TraceStats((2, 1)), + ('b.py', 4): tracemalloc.TraceStats((32, 4)), + ('b.py', 1): tracemalloc.TraceStats((66, 1)), + }) + self.assertEqual(top_stats2.group_by, 'filename_lineno') + self.assertEqual(top_stats2.timestamp, snapshot.timestamp) + self.assertEqual(top_stats2.cumulative, True) + self.assertEqual(top_stats2.process_memory, snapshot.process_memory) + self.assertEqual(top_stats2.tracemalloc_size, snapshot.tracemalloc_size) + + def test_display_top_by_line(self): + snapshot, snapshot2 = self.create_snapshots() + + # top per file (default options) + output = io.StringIO() + top = tracemalloc.DisplayTop() + top.display_snapshot(snapshot, file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:17: Top 4 allocations per filename and line number +#1: a.py:5: size=127 B, count=2, average=63 B +#2: b.py:?: size=66 B, count=1 +#3: ???:?: size=55 B, count=1 +#4: a.py:2: size=10 B, count=100, average=0 B +Traced Python memory: size=258 B, count=104, average=2 B +Tracemalloc memory: 100 B +Process memory: RSS=1024 B, VMS=2048 B + '''.strip() + '\n\n') + + # diff per line + output = io.StringIO() + top.display_snapshot(snapshot2, file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:50: Top 4 allocations per filename and line number (compared to 2013-09-12 15:16:17) +#1: a.py:5: size=200 B (+73 B), count=2 (+0), average=100 B +#2: b.py:?: size=66 B (+0 B), count=1 (+0) +#3: a.py:2: size=10 B (+0 B), count=100 (+0), average=0 B +#4: ???:?: size=0 B (-55 B), count=0 (-1) +Traced Python memory: size=276 B (+18 B), count=103 (-1), average=2 B +Tracemalloc memory: 200 B (+100 B) +Process memory: RSS=1500 B (+476 B), VMS=2048 B (+0 B) + '''.strip() + '\n\n') + + def test_display_top_by_file(self): + snapshot, snapshot2 = self.create_snapshots() + + # group per file + output = io.StringIO() + top = tracemalloc.DisplayTop() + top.display_snapshot(snapshot, group_by='filename', file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:17: Top 3 allocations per filename +#1: a.py: size=137 B, count=102, average=1 B +#2: b.py: size=66 B, count=1 +#3: ???: size=55 B, count=1 +Traced Python memory: size=258 B, count=104, average=2 B +Tracemalloc memory: 100 B +Process memory: RSS=1024 B, VMS=2048 B + '''.strip() + '\n\n') + + # diff per file + output = io.StringIO() + top.display_snapshot(snapshot2, group_by='filename', file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:50: Top 3 allocations per filename (compared to 2013-09-12 15:16:17) +#1: a.py: size=210 B (+73 B), count=102 (+0), average=2 B +#2: b.py: size=66 B (+0 B), count=1 (+0) +#3: ???: size=0 B (-55 B), count=0 (-1) +Traced Python memory: size=276 B (+18 B), count=103 (-1), average=2 B +Tracemalloc memory: 200 B (+100 B) +Process memory: RSS=1500 B (+476 B), VMS=2048 B (+0 B) + '''.strip() + '\n\n') + + +def test_main(): + support.run_unittest( + TestTracemallocEnabled, + TestFilters, + TestCommandLine, + TestTop) + +if __name__ == "__main__": + test_main() diff -r df2fdd42b375 -r 21f7c3df0f15 Lib/tracemalloc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/tracemalloc.py Tue Sep 17 00:59:50 2013 +0200 @@ -0,0 +1,975 @@ +from __future__ import with_statement +import collections +import datetime +import linecache +import operator +import os +import sys +import types +try: + from time import monotonic as _time_monotonic +except ImportError: + from time import time as _time_monotonic +pickle = None + +from _tracemalloc import * + +def _format_timestamp(timestamp): + return str(timestamp).split(".", 1)[0] + +def __format_size(size, sign=False): + for unit in ('B', 'KiB', 'MiB', 'GiB'): + if abs(size) < 5 * 1024: + if sign: + return "%+i %s" % (size, unit) + else: + return "%i %s" % (size, unit) + size /= 1024 + + if sign: + return "%+i TiB" % size + else: + return "%i TiB" % size + +if os.name != "nt": + _FORMAT_YELLOW = '\x1b[1;33m%s\x1b[0m' + _FORMAT_BOLD = '\x1b[1m%s\x1b[0m' + _FORMAT_CYAN = '\x1b[36m%s\x1b[0m' +else: + _FORMAT_YELLOW = _FORMAT_BOLD = _FORMAT_CYAN = "%s" +_PAGESIZE = os.sysconf("SC_PAGE_SIZE") + +def _format_size(size, color): + text = __format_size(size) + if color: + text = _FORMAT_YELLOW % text + return text + +def _format_size_diff(size, diff, color): + text = __format_size(size) + if diff is not None: + if color: + text = _FORMAT_BOLD % text + textdiff = __format_size(diff, sign=True) + if color: + textdiff = _FORMAT_YELLOW % textdiff + text += " (%s)" % textdiff + else: + if color: + text = _FORMAT_YELLOW % text + return text + +def _format_address(address, color): + address = '0x%x' % address + if color: + address = _FORMAT_BOLD % address + return address + +def _colorize_filename(filename): + path, basename = os.path.split(filename) + if path: + path += os.path.sep + return _FORMAT_CYAN % path + basename + +def _format_filename(filename, max_parts, color): + if filename: + parts = filename.split(os.path.sep) + if max_parts < len(parts): + parts = ['...'] + parts[-max_parts:] + filename = os.path.sep.join(parts) + else: + # filename is None or an empty string + filename = '???' + if color: + filename = _colorize_filename(filename) + return filename + +def _format_lineno(lineno): + if lineno: + return str(lineno) + else: + # lineno is None or an empty string + return '?' + +def _format_traceback(traceback, filename_parts, color): + if traceback is None: + return ('(empty traceback)',) + lines = ['Traceback (most recent call first):'] + if traceback is not None: + for frame in traceback: + filename = _format_filename(frame.filename, filename_parts, color) + lineno = _format_lineno(frame.lineno) + lines.append(' File "%s", line %s' % (filename, lineno)) + if frame.filename and frame.lineno: + line = linecache.getline(frame.filename, frame.lineno) + line = line.strip() + if line: + lines.append(' ' + line) + else: + lines.append(' File "%s", line %' % (frame.filename, frame.lineno)) + return lines + +_meminfo = collections.namedtuple('meminfo', 'rss vms') + +def get_process_memory(): + """ + Get the memory usage of the current process as a meminfo namedtuple with + two attributes: + + * rss: Resident Set Size in bytes + * vms: size of the virtual memory in bytes + + Return None if the platform is not supported. + """ + if get_process_memory.psutil_process is None: + try: + import psutil + except ImportError: + get_process_memory.psutil_process = False + else: + pid = os.getpid() + get_process_memory.psutil_process = psutil.Process(pid) + + if get_process_memory.psutil_process != False: + usage = get_process_memory.psutil_process.get_memory_info() + return _meminfo(usage.rss, usage.vms) + + if get_process_memory.support_proc == False: + return + + try: + fp = open("/proc/self/statm", "rb") + except IOError: + get_process_memory.support_proc = False + return None + + get_process_memory.support_proc = True + with fp: + statm = fp.readline().split() + vms = int(statm[0]) * _PAGESIZE + rss = int(statm[1]) * _PAGESIZE + return _meminfo(rss, vms) + +get_process_memory.support_proc = None +get_process_memory.psutil_process = None + +class StatsDiff: + def __init__(self, differences, old_stats, new_stats): + self.differences = differences + self.old_stats = old_stats + self.new_stats = new_stats + + def sort(self): + try: + self.differences.sort(reverse=True) + except: + import pprint + pprint.pprint(self.differences) + raise + + +class GroupedStats: + def __init__(self, stats, group_by, cumulative=False, timestamp=None, + process_memory=None, tracemalloc_size=None): + if group_by not in ('filename', 'filename_lineno', 'address'): + raise ValueError("invalid group_by value") + # dictionary {key: stats} where stats is a TraceStats instance + self.stats = stats + self.group_by = group_by + self.cumulative = cumulative + self.timestamp = timestamp + self.process_memory = process_memory + self.tracemalloc_size = tracemalloc_size + + def _create_key(self, key): + if self.group_by == 'filename': + if key is None: + return '' + elif self.group_by == 'filename_lineno': + filename, lineno = key + if filename is None: + filename = '' + if lineno is None: + lineno = 0 + return (filename, lineno) + return key + + def compare_to(self, old_stats=None): + if old_stats is not None: + previous_dict = old_stats.stats.copy() + + differences = [] + for key, stats in self.stats.items(): + size = stats.size + count = stats.count + previous = previous_dict.pop(key, None) + key = self._create_key(key) + if previous is not None: + diff = (size - previous.size, size, + count - previous.count, count, + key) + else: + diff = (0, size, 0, count, key) + differences.append(diff) + + for key, trace in previous_dict.items(): + key = self._create_key(key) + diff = (-trace.size, 0, -trace.count, 0, key) + differences.append(diff) + else: + differences = [ + (0, stats.size, 0, stats.count, self._create_key(key)) + for key, stats in self.stats.items()] + + return StatsDiff(differences, old_stats, self) + + +class DisplayTop: + def __init__(self): + self.show_size = True + self.show_count = True + self.show_average = True + self.filename_parts = 3 + self.color = None + self.compare_with_previous = True + self.previous_top_stats = None + + def _format_diff(self, diff, show_diff, show_count, color): + if not show_count and not self.show_average: + if show_diff: + return _format_size_diff(diff[1], diff[0], color) + else: + return _format_size(diff[1], color) + + parts = [] + if self.show_size: + if show_diff: + text = _format_size_diff(diff[1], diff[0], color) + else: + text = _format_size(diff[1], color) + parts.append("size=%s" % text) + if show_count and (diff[3] or diff[2]): + text = "count=%s" % diff[3] + if show_diff: + text += " (%+i)" % diff[2] + parts.append(text) + if (self.show_average + and diff[3] > 1): + parts.append('average=%s' % _format_size(diff[1] // diff[3], False)) + return ', '.join(parts) + + def _format_filename(self, key, color): + return _format_filename(key, self.filename_parts, color) + + def _format_address(self, key, color): + return 'memory block %s' % _format_address(key, color) + + def _format_filename_lineno(self, key, color): + filename, lineno = key + filename = _format_filename(filename, self.filename_parts, color) + lineno = _format_lineno(lineno) + return "%s:%s" % (filename, lineno) + + def display_top_diff(self, top_diff, count=10, file=None): + if file is None: + file = sys.stdout + log = file.write + if self.color is None: + color = file.isatty() + else: + color = self.color + diff_list = top_diff.differences + top_stats = top_diff.new_stats + previous_top_stats = top_diff.old_stats + has_previous = (top_diff.old_stats is not None) + if top_stats.group_by == 'address': + show_count = False + else: + show_count = self.show_count + + if top_stats.group_by == 'filename': + format_key = self._format_filename + per_text = "filename" + elif top_stats.group_by == 'address': + format_key = self._format_address + per_text = "address" + else: + format_key = self._format_filename_lineno + per_text = "filename and line number" + + # Write the header + nother = max(len(diff_list) - count, 0) + count = min(count, len(diff_list)) + if top_stats.cumulative: + text = "Cumulative top %s allocations per %s" % (count, per_text) + else: + text = "Top %s allocations per %s" % (count, per_text) + if color: + text = _FORMAT_CYAN % text + if self.previous_top_stats is not None: + text += ' (compared to %s)' % _format_timestamp(self.previous_top_stats.timestamp) + name = _format_timestamp(top_stats.timestamp) + if color: + name = _FORMAT_BOLD % name + file.write("%s: %s\n" % (name, text)) + + # Sort differences by size and then by count + top_diff.sort() + + # Display items + total = [0, 0, 0, 0, ''] + for index in range(0, count): + diff = diff_list[index] + key_text = format_key(diff[4], color) + diff_text = self._format_diff(diff, has_previous, show_count, color) + log("#%s: %s: %s\n" % (1 + index, key_text, diff_text)) + total[0] += diff[0] + total[1] += diff[1] + total[2] += diff[2] + total[3] += diff[3] + + other = tuple(total) + for index in range(count, len(diff_list)): + diff = diff_list[index] + total[0] += diff[0] + total[1] += diff[1] + total[2] += diff[2] + total[3] += diff[3] + + # Display "xxx more" + if nother > 0: + other = [ + total[0] - other[0], + total[1] - other[1], + total[2] - other[2], + total[3] - other[3], + '', + ] + other = self._format_diff(other, has_previous, show_count, color) + text = "%s more" % nother + if color: + text = _FORMAT_CYAN % text + log("%s: %s\n" % (text, other)) + + if not top_stats.cumulative: + text = self._format_diff(total, has_previous, show_count, color) + log("Traced Python memory: %s\n" % text) + + # Display tracemalloc size + size = top_stats.tracemalloc_size + if has_previous: + diff = size - previous_top_stats.tracemalloc_size + text = _format_size_diff(size, diff, color) + else: + text = _format_size(size, color) + log("Tracemalloc memory: %s\n" % text) + + # Display process memory + if top_stats.process_memory is not None: + if has_previous and previous_top_stats.process_memory is not None: + diff = top_stats.process_memory.rss - previous_top_stats.process_memory.rss + rss_text = _format_size_diff(top_stats.process_memory.rss, diff, color) + + diff = top_stats.process_memory.vms - previous_top_stats.process_memory.vms + vms_text = _format_size_diff(top_stats.process_memory.vms, diff, color) + else: + rss_text = _format_size(top_stats.process_memory.rss, color) + vms_text = _format_size(top_stats.process_memory.vms, color) + log("Process memory: RSS=%s, VMS=%s\n" % (rss_text, vms_text)) + + log("\n") + file.flush() + + # store the current top stats as the previous top stats for later + # comparison with a newer top stats + if self.compare_with_previous: + self.previous_top_stats = top_stats + else: + if self.previous_top_stats is None: + self.previous_top_stats = top_stats + + def display_top_stats(self, top_stats, count=10, file=None): + top_diff = top_stats.compare_to(self.previous_top_stats) + self.display_top_diff(top_diff, count=count, file=file) + + def display_snapshot(self, snapshot, count=10, + group_by="filename_lineno", cumulative=False, + file=None): + top_stats = snapshot.top_by(group_by, cumulative) + self.display_top_stats(top_stats, count=count, file=file) + + +class DisplayTopTask: + def __init__(self, count, group_by="filename_lineno", cumulative=False, + file=None, user_data_callback=None): + self.display_top = DisplayTop() + self.count = count + self.group_by = group_by + self.cumulative = cumulative + self.file = file + self.user_data_callback = user_data_callback + + def display(self): + with_traces = (self.cumulative + and (tracemalloc.get_traceback_limit() > 1)) + snapshot = Snapshot.create(with_traces=with_traces, + user_data_callback=self.user_data_callback) + self.display_top.display_snapshot(snapshot, + count=self.count, + group_by=self.group_by, + cumulative=self.cumulative, + file=self.file) + return snapshot + + def start(self, delay): + start_timer(int(delay), self.display) + + def stop(self): + stop_timer() + + +def _lazy_import_pickle(): + # lazy loader for the pickle module + global pickle + if pickle is None: + import pickle + return pickle + +def _get_first_frame(trace): + if trace.traceback: + frame = trace.traceback[0] + return (frame.filename, frame.lineno) + else: + return (None, None) + +def _compute_stats_frame(stats, group_per_file, size, frame): + if not group_per_file: + if frame is not None: + key = (frame.filename, frame.lineno) + else: + key = (None, None) + else: + if frame is not None: + key = frame.filename + else: + key = None + if key in stats: + trace_stats = stats[key] + size += trace_stats.size + count = trace_stats.count + 1 + else: + count = 1 + stats[key] = TraceStats((size, count)) + +class Snapshot: + FORMAT_VERSION = '3.4' + __slots__ = ('timestamp', 'pid', 'traces', 'stats', + 'process_memory', 'tracemalloc_size', 'traceback_limit', + 'user_data') + + def __init__(self, timestamp, pid, traces, stats, + process_memory, tracemalloc_size, traceback_limit, + user_data): + if traces is None and stats is None: + raise ValueError("traces and stats cannot be None at the same time") + self.timestamp = timestamp + self.pid = pid + self.traces = traces + self.stats = stats + self.process_memory = process_memory + self.tracemalloc_size = tracemalloc_size + self.traceback_limit = traceback_limit + self.user_data = user_data + + @classmethod + def create(cls, *, with_traces=False, with_stats=True, + user_data_callback=None): + if not is_enabled(): + raise RuntimeError("the tracemalloc module must be enabled " + "to take a snapshot") + if not with_traces and not with_stats: + raise ValueError("with_traces or with_stats must be True") + timestamp = datetime.datetime.now() + pid = os.getpid() + if with_traces: + traces = get_traces() + else: + traces = None + if with_stats: + stats = get_stats() + else: + stats = None + tracemalloc_size = get_tracemalloc_size() + traceback_limit = get_traceback_limit() + process_memory = get_process_memory() + if user_data_callback is not None: + user_data = user_data_callback() + else: + user_data = None + return cls(timestamp, pid, traces, stats, + process_memory, tracemalloc_size, traceback_limit, + user_data) + + @classmethod + def load(cls, filename): + pickle = _lazy_import_pickle() + with open(filename, "rb") as fp: + data = pickle.load(fp) + + try: + if data['format_version'] != cls.FORMAT_VERSION: + raise TypeError("unknown format version") + + timestamp = data['timestamp'] + pid = data['pid'] + traces = data.get('traces') + stats = data['stats'] + process_memory = data.get('process_memory') + tracemalloc_size = data['tracemalloc_size'] + traceback_limit = data['traceback_limit'] + user_data = data.get('user_data') + except KeyError: + raise TypeError("invalid file format") + + if process_memory is not None: + process_memory = _meminfo(*process_memory) + + if traces is not None: + new_traces = {} + for address, item in traces.items(): + size = item[0] + traceback = [] + for index in range(1, len(item), 2): + filename = item[index] + lineno = item[index + 1] + traceback.append(Frame((filename, lineno))) + new_traces[address] = Trace((size, traceback)) + traces = new_traces + + return cls(timestamp, pid, traces, stats, + process_memory, tracemalloc_size, traceback_limit, + user_data) + + def write(self, filename): + pickle = _lazy_import_pickle() + data = { + 'format_version': self.FORMAT_VERSION, + 'timestamp': self.timestamp, + 'pid': self.pid, + 'stats': self.stats, + 'tracemalloc_size': self.tracemalloc_size, + 'traceback_limit': self.traceback_limit, + 'user_data': self.user_data, + } + if self.traces is not None: + traces = {} + for address, trace in self.traces.items(): + item = [trace.size] + for frame in trace.traceback: + item.extend((frame.filename, frame.lineno)) + traces[address] = item + data['traces'] = traces + if self.process_memory is not None: + # use a simple tuple rather than a namedtuple + data['process_memory'] = tuple(self.process_memory) + + try: + with open(filename, "wb") as fp: + pickle.dump(data, fp, pickle.HIGHEST_PROTOCOL) + except: + # Remove corrupted pickle file + if os.path.exists(filename): + os.unlink(filename) + raise + + def _filter_traces(self, include, filters): + new_traces = {} + for address, trace in self.traces.items(): + if include: + match = any(trace_filter.match_trace(trace) + for trace_filter in filters) + else: + match = all(trace_filter.match_trace(trace) + for trace_filter in filters) + if match: + new_traces[address] = trace + return new_traces + + def _filter_stats(self, include, filters): + file_stats = {} + for filename, line_stats in self.stats.items(): + if include: + match = any(trace_filter.match_filename(filename) + for trace_filter in filters) + else: + match = all(trace_filter.match_filename(filename) + for trace_filter in filters) + if not match: + continue + + new_line_stats = {} + for lineno, line_stat in line_stats.items(): + if include: + match = any(trace_filter.match(filename, lineno) + for trace_filter in filters) + else: + match = all(trace_filter.match(filename, lineno) + for trace_filter in filters) + if match: + new_line_stats[lineno] = line_stat + + file_stats[filename] = new_line_stats + return file_stats + + def _apply_filters(self, include, filters): + if not filters: + return + if self.traces is not None: + self.traces = self._filter_traces(include, filters) + if self.stats is not None: + self.stats = self._filter_stats(include, filters) + + def apply_filters(self, filters): + include_filters = [] + exclude_filters = [] + for trace_filter in filters: + if trace_filter.include: + include_filters.append(trace_filter) + else: + exclude_filters.append(trace_filter) + self._apply_filters(True, include_filters) + self._apply_filters(False, exclude_filters) + + def top_by(self, group_by, cumulative=False): + stats = {} + if group_by == 'address': + cumulative = False + + if self.traces is None: + raise ValueError("need traces") + + for address, trace in self.traces.items(): + stats[address] = TraceStats((trace.size, 1)) + else: + if group_by == 'filename': + group_per_file = True + elif group_by == 'filename_lineno': + group_per_file = False + else: + raise ValueError("unknown group_by value: %r" % (group_by,)) + + if not cumulative and self.stats is not None: + for filename, line_dict in self.stats.items(): + if not group_per_file: + for lineno, line_stats in line_dict.items(): + key = (filename, lineno) + stats[key] = line_stats + else: + key = filename + size = count = 0 + for line_stats in line_dict.values(): + size += line_stats.size + count += line_stats.count + stats[key] = TraceStats((size, count)) + else: + for trace in self.traces.values(): + if trace.traceback: + if cumulative: + for frame in trace.traceback: + _compute_stats_frame(stats, group_per_file, trace.size, frame) + else: + _compute_stats_frame(stats, group_per_file, trace.size, trace.traceback[0]) + else: + _compute_stats_frame(stats, group_per_file, trace.size, None) + + return GroupedStats(stats, group_by, + cumulative, self.timestamp, + self.process_memory, self.tracemalloc_size) + + +class TakeSnapshot: + def __init__(self): + self.filename_template = "tracemalloc-$counter.pickle" + self.counter = 1 + self.with_traces = False + self.with_stats = True + self.user_data_callback = None + + def take_snapshot(self): + snapshot = Snapshot.create(with_traces=self.with_traces, + with_stats=self.with_stats, + user_data_callback=self.user_data_callback) + + filename = self.filename_template + filename = filename.replace("$pid", str(snapshot.pid)) + + timestamp = _format_timestamp(snapshot.timestamp) + timestamp = timestamp.replace(" ", "-") + filename = filename.replace("$timestamp", timestamp) + + filename = filename.replace("$counter", "%04i" % self.counter) + + snapshot.write(filename) + self.counter += 1 + return snapshot, filename + + def _task(self): + start = _time_monotonic() + snapshot, filename = self.take_snapshot() + dt = _time_monotonic() - start + sys.stderr.write("%s: Write a snapshot of memory allocations into %s (%.1f sec)\n" + % (_format_timestamp(snapshot.timestamp), filename, dt)) + sys.stderr.flush() + + def start(self, delay): + start_timer(int(delay), self._task) + + def stop(self): + stop_timer() + + +def main(): + from optparse import OptionParser + + parser = OptionParser(usage="%prog trace1.pickle [trace2.pickle trace3.pickle ...]") + parser.add_option("-a", "--address", + help="Group memory allocations by address, " + "instead of grouping by line number", + action="store_true", default=False) + parser.add_option("-f", "--file", + help="Group memory allocations per filename, " + "instead of grouping by line number", + action="store_true", default=False) + parser.add_option("-n", "--number", + help="Number of traces displayed per top (default: 10)", + type="int", action="store", default=10) + parser.add_option("--first", + help="Compare with the first trace, instead of with the previous trace", + action="store_true", default=False) + parser.add_option("-c", "--cumulative", + help="Cumulate size and count of memory blocks using " + "all frames, not only the most recent frame. The option has only " + "an effect if the snapshot contains traces and the traceback limit" + "was greater than 1", + action="store_true", default=False) + parser.add_option("-b", "--block", metavar="ADDRESS", + help="Get the memory block at address ADDRESS, display its size and " + "the traceback where it was allocated.", + action="store", type="int", default=None) + parser.add_option("-t", "--traceback", + help="Group memmory allocations by address and display the size and " + "the traceback of the NUMBER biggest allocated memory blocks", + action="store_true", default=False) + parser.add_option("-i", "--include", metavar="FILENAME[:LINENO]", + help="Only show memory block allocated in a file with a name matching " + "FILENAME pattern at line number LINENO. Ony check the most " + "recent frame. The option can be specified multiple times.", + action="append", type=str, default=[]) + parser.add_option("-I", "--include-traceback", metavar="FILENAME[:LINENO]", + help="Similar to --include, but check all frames of the traceback.", + action="append", type=str, default=[]) + parser.add_option("-x", "--exclude", metavar="FILENAME[:LINENO]", + help="Exclude filenames matching FILENAME pattern at line number " + "LINENO. Only check the most recent frame. The option can be " + "specified multiple times.", + action="append", type=str, default=[]) + parser.add_option("-X", "--exclude-traceback", metavar="FILENAME[:LINENO]", + help="Similar to --exclude, but check all frames of the traceback.", + action="append", type=str, default=[]) + parser.add_option("-S", "--hide-size", + help="Hide the size of allocations", + action="store_true", default=False) + parser.add_option("-C", "--hide-count", + help="Hide the number of allocations", + action="store_true", default=False) + parser.add_option("-A", "--hide-average", + help="Hide the average size of allocations", + action="store_true", default=False) + parser.add_option("-P", "--filename-parts", + help="Number of displayed filename parts (default: 3)", + type="int", action="store", default=3) + parser.add_option("--color", + help="Always use colors", + action="store_true", default=False) + parser.add_option("--no-color", + help="Never use colors", + action="store_true", default=False) + + options, filenames = parser.parse_args() + if not filenames: + parser.print_help() + sys.exit(1) + # remove duplicates + filenames = list(set(filenames)) + + if options.address or options.traceback: + group_by = "address" + elif options.file: + group_by = "filename" + else: + group_by = "filename_lineno" + + # use set() to delete duplicate filters + filters = set() + for include, values, traceback in ( + (True, options.include, False), + (True, options.include_traceback, True), + (False, options.exclude, False), + (False, options.exclude_traceback, True), + ): + for value in values: + if ':' in value: + pattern, lineno = value.rsplit(':', 1) + lineno = int(lineno) + else: + pattern = value + lineno = None + filters.add(Filter(include, pattern, lineno, traceback)) + + def log(message, *args): + if args: + message = message % args + sys.stderr.write(message + "\n") + sys.stderr.flush() + + snapshots = [] + for filename in filenames: + start = _time_monotonic() + try: + log("Load snapshot %s", filename) + snapshot = Snapshot.load(filename) + except Exception: + err = sys.exc_info()[1] + print("ERROR: Failed to load %s: [%s] %s" % (filename, type(err).__name__, err)) + sys.exit(1) + + info = [] + if snapshot.stats is not None: + info.append('%s files' % len(snapshot.stats)) + if snapshot.traces is not None: + info.append('%s traces (limit=%s frames)' + % (len(snapshot.traces), snapshot.traceback_limit)) + dt = _time_monotonic() - start + log("Load snapshot %s: %s (%.1f sec)", + filename, ', '.join(info), dt) + + if options.address is not None or options.traceback: + if snapshot.traces is None: + print("ERROR: The snapshot %s does not contain traces, " + "only stats" % filename) + sys.exit(1) + elif not options.cumulative: + # drop traces if we don't need them, because filtering + # them can be slow + log("Don't need traces: delete them") + snapshot.traces = None + + if filters: + start = _time_monotonic() + text = ("Apply %s filter%s on snapshot %s..." + % (len(filters), + 's' if not filters or len(filters) > 1 else '', + _format_timestamp(snapshot.timestamp))) + log(text) + snapshot.apply_filters(filters) + dt = _time_monotonic() - start + log(text + " done (%.1f sec)" % dt) + + snapshots.append(snapshot) + snapshots.sort(key=lambda snapshot: snapshot.timestamp) + + pids = set(snapshot.pid for snapshot in snapshots) + if len(pids) > 1: + pids = ', '.join(map(str, sorted(pids))) + print("WARNING: Traces generated by different processes: %s" % pids) + print("") + + stream = sys.stdout + if options.color: + color = True + elif options.no_color: + color = False + else: + color = stream.isatty() + + if options.block is not None: + address = options.block + + for snapshot in snapshots: + log("") + trace = snapshot.traces.get(address) + timestamp = _format_timestamp(snapshot.timestamp) + address = _format_address(address, color) + if color: + timestamp = _FORMAT_CYAN % timestamp + if trace is not None: + size = _format_size(trace.size, color) + else: + size = '(not found)' + if color: + size = _FORMAT_YELLOW % size + print("%s, memory block %s: %s" + % (timestamp, address, size)) + if trace is not None: + for line in _format_traceback(trace.traceback, options.filename_parts, color): + print(line) + + elif options.traceback: + for snapshot in snapshots: + log("Sort traces of snapshot %s", + _format_timestamp(snapshot.timestamp)) + traces = [(trace.size, address, trace) + for address, trace in snapshot.traces.items()] + traces.sort(reverse=True) + displayed_traces = traces[:options.number] + + timestamp = _format_timestamp(snapshot.timestamp) + number = len(displayed_traces) + if color: + number = _FORMAT_BOLD % number + log("") + print("%s: Traceback of the top %s biggest memory blocks" + % (timestamp, number)) + print() + + for size, address, trace in displayed_traces: + address = _format_address(address, color) + size = _format_size(trace.size, color) + print("Memory block %s: %s" % (address, size)) + for line in _format_traceback(trace.traceback, options.filename_parts, color): + print(line) + print() + + ignored = len(traces) - len(displayed_traces) + if ignored: + ignored_size = sum(size for size, address, trace in traces[options.number:]) + size = _format_size(ignored_size, color) + print("%s more memory blocks: size=%s" % (ignored, size)) + + else: + top = DisplayTop() + top.filename_parts = options.filename_parts + top.show_average = not options.hide_average + top.show_count = not options.hide_count + top.show_size = not options.hide_size + top.compare_with_previous = not options.first + top.color = color + + for snapshot in snapshots: + log("") + top.display_snapshot(snapshot, + count=options.number, + group_by=group_by, + cumulative=options.cumulative, + file=stream) + + print("%s snapshots" % len(snapshots)) + + +if __name__ == "__main__": + if 0: + import cProfile + cProfile.run('main()', sort='tottime') + else: + main() + diff -r df2fdd42b375 -r 21f7c3df0f15 Modules/Setup.dist --- a/Modules/Setup.dist Mon Aug 26 22:28:21 2013 +0200 +++ b/Modules/Setup.dist Tue Sep 17 00:59:50 2013 +0200 @@ -102,7 +102,7 @@ PYTHONPATH=$(COREPYTHONPATH) # various reasons; therefore they are listed here instead of in the # normal order. -# This only contains the minimal set of modules required to run the +# This only contains the minimal set of modules required to run the # setup.py script in the root of the Python source tree. posix posixmodule.c # posix (UNIX) system calls @@ -115,7 +115,7 @@ pwd pwdmodule.c # this is needed to fi _functools _functoolsmodule.c # Tools for working with functions and callable objects _operator _operator.c # operator.add() and similar goodies _collections _collectionsmodule.c # Container types -itertools itertoolsmodule.c # Functions creating iterators for efficient looping +itertools itertoolsmodule.c # Functions creating iterators for efficient looping atexit atexitmodule.c # Register functions to be run at interpreter-shutdown _stat _stat.c # stat.h interface @@ -132,12 +132,15 @@ zipimport zipimport.c # faulthandler module faulthandler faulthandler.c +# _tracemalloc module +_tracemalloc _tracemalloc.c + # The rest of the modules listed in this file are all commented out by # default. Usually they can be detected and built as dynamically # loaded modules by the new setup.py script added in Python 2.1. If -# you're on a platform that doesn't support dynamic loading, want to -# compile modules statically into the Python binary, or need to -# specify some odd set of compiler switches, you can uncomment the +# you're on a platform that doesn't support dynamic loading, want to +# compile modules statically into the Python binary, or need to +# specify some odd set of compiler switches, you can uncomment the # appropriate lines below. # ====================================================================== @@ -186,7 +189,7 @@ faulthandler faulthandler.c # supported...) #fcntl fcntlmodule.c # fcntl(2) and ioctl(2) -#spwd spwdmodule.c # spwd(3) +#spwd spwdmodule.c # spwd(3) #grp grpmodule.c # grp(3) #select selectmodule.c # select(2); not on ancient System V @@ -302,7 +305,7 @@ faulthandler faulthandler.c #_curses _cursesmodule.c -lcurses -ltermcap # Wrapper for the panel library that's part of ncurses and SYSV curses. -#_curses_panel _curses_panel.c -lpanel -lncurses +#_curses_panel _curses_panel.c -lpanel -lncurses # Modules that provide persistent dictionary-like semantics. You will diff -r df2fdd42b375 -r 21f7c3df0f15 Modules/_tracemalloc.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Modules/_tracemalloc.c Tue Sep 17 00:59:50 2013 +0200 @@ -0,0 +1,3121 @@ +/* The implementation of the hash table is based on the cfuhash project: + http://sourceforge.net/projects/libcfu/ + + Copyright of cfuhash: + ---------------------------------- + Creation date: 2005-06-24 21:22:40 + Authors: Don + Change log: + + Copyright (c) 2005 Don Owens + All rights reserved. + + This code is released under the BSD license: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + ---------------------------------- +*/ + +#include "Python.h" +#include "frameobject.h" +#include "pythread.h" +#include "osdefs.h" + +/* #define TRACE_RAW_MALLOC */ + +#if defined(HAVE_PTHREAD_ATFORK) && defined(WITH_THREAD) +# define TRACE_ATFORK +#endif + +#ifdef MS_WINDOWS +# define TRACE_CASE_INSENSITIVE +#endif + +#if defined(ALTSEP) || defined(TRACE_CASE_INSENSITIVE) +# define TRACE_NORMALIZE_FILENAME +#endif + +#ifdef TRACE_ATFORK +# include +#endif + +#ifdef Py_DEBUG +# define TRACE_DEBUG +#endif + +#define INT_TO_POINTER(value) ((void*)(Py_uintptr_t)(unsigned int)value) +#define POINTER_TO_INT(ptr) ((int)(Py_uintptr_t)ptr) +#define CFUHASH_MIN_SIZE 8 +#define CFUHASH_HIGH 0.75 +#define CFUHASH_LOW 0.25 + +typedef struct cfuhash_entry { + struct cfuhash_entry *next; + const void *key; + Py_uhash_t key_hash; + /* data folllows */ +} cfuhash_entry; + +#define CFUHASH_ENTRY_DATA(ENTRY) \ + ((char *)(ENTRY) + sizeof(cfuhash_entry)) +#define CFUHASH_ENTRY_DATA_AS_VOID_P(ENTRY) \ + (*(void **)CFUHASH_ENTRY_DATA(ENTRY)) + +#define CFUHASH_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + memcpy(DATA, CFUHASH_ENTRY_DATA(ENTRY), DATA_SIZE); \ + } while (0) + +#define CFUHASH_WRITE_DATA(TABLE, ENTRY, DATA, DATA_SIZE) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + memcpy(CFUHASH_ENTRY_DATA(ENTRY), DATA, DATA_SIZE); \ + } while (0) + +#define CFUHASH_GET_DATA(ENTRY, KEY, DATA) \ + cfuhash_get_data(ENTRY, KEY, &(DATA), sizeof(DATA)) + +#define CFUHASH_PUT_DATA(ENTRY, KEY, DATA) \ + cfuhash_put_data(ENTRY, KEY, &(DATA), sizeof(DATA)) + +typedef Py_uhash_t (*cfuhash_hash_func) (const void *key); +typedef int (*cfuhash_compare_func) (const void *key, cfuhash_entry *he); +typedef void* (*cfuhash_copy_data_func)(void *data); +typedef void (*cfuhash_free_data_func)(void *data); +typedef size_t (*cfuhash_get_data_size_func)(void *data); + +typedef struct cfuhash_table { + size_t num_buckets; + size_t entries; /* Total number of entries in the table. */ + size_t data_size; + cfuhash_entry **buckets; + cfuhash_hash_func hash_func; + cfuhash_compare_func compare_func; + cfuhash_copy_data_func copy_data_func; + cfuhash_free_data_func free_data_func; + cfuhash_get_data_size_func get_data_size_func; +} cfuhash_table_t; + +/* Forward declaration */ +static void _cfuhash_rehash(cfuhash_table_t *ht); + +#ifdef TRACE_DEBUG +static void +tracemalloc_error(const char *format, ...) +{ + va_list ap; + fprintf(stderr, "tracemalloc: "); + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + fprintf(stderr, "\n"); + fflush(stderr); +} +#endif + +static Py_uhash_t +cfuhash_hash_int(const void *key) +{ + return (Py_uhash_t)POINTER_TO_INT(key); +} + +static Py_uhash_t +cfuhash_hash_ptr(const void *key) +{ + return (Py_uhash_t)_Py_HashPointer((void *)key); +} + +static Py_uhash_t +cfuhash_hash_pyobject(const void *key) +{ + if (key != NULL) + return PyObject_Hash((PyObject*)key); + else + return 0; +} + +static int +cfuhash_cmp_unicode(const void *key, cfuhash_entry *he) +{ + if (key != NULL && he->key != NULL) + return PyUnicode_Compare((PyObject *)key, (PyObject *)he->key); + else + return key == he->key; +} + +static int +cfuhash_cmp_direct(const void *key, cfuhash_entry *he) +{ + return he->key != key; +} + +/* makes sure the real size of the buckets array is a power of 2 */ +static u_int +cfuhash_hash_size(u_int s) +{ + u_int i; + if (s < CFUHASH_MIN_SIZE) + return CFUHASH_MIN_SIZE; + i = 1; + while (i < s) + i <<= 1; + return i; +} + +static cfuhash_table_t * +cfuhash_new_full(size_t data_size, size_t size, + cfuhash_hash_func hash_func, + cfuhash_compare_func compare_func, + cfuhash_copy_data_func copy_data_func, + cfuhash_free_data_func free_data_func, + cfuhash_get_data_size_func get_data_size_func) +{ + cfuhash_table_t *ht; + + ht = (cfuhash_table_t *)malloc(sizeof(cfuhash_table_t)); + if (ht == NULL) + return ht; + + if (size > CFUHASH_MIN_SIZE) + ht->num_buckets = size; + else + ht->num_buckets = CFUHASH_MIN_SIZE; + ht->entries = 0; + ht->data_size = data_size; + ht->buckets = (cfuhash_entry **)calloc(ht->num_buckets, + sizeof(cfuhash_entry *)); + if (ht->buckets == NULL) { + free(ht); + return NULL; + } + + ht->hash_func = hash_func; + ht->compare_func = compare_func; + ht->copy_data_func = copy_data_func; + ht->free_data_func = free_data_func; + ht->get_data_size_func = get_data_size_func; + return ht; +} + +static cfuhash_table_t * +cfuhash_new(size_t data_size, + cfuhash_hash_func hash_func, + cfuhash_compare_func compare_func) +{ + return cfuhash_new_full(data_size, CFUHASH_MIN_SIZE, + hash_func, compare_func, + NULL, NULL, NULL); +} + +static size_t +cfuhash_get_size(cfuhash_table_t *ht) +{ + size_t size, entry_size; + + entry_size = sizeof(cfuhash_entry) + ht->data_size; + + size = sizeof(cfuhash_table_t); + size += ht->entries * entry_size; + if (ht->get_data_size_func) { + cfuhash_entry *entry; + size_t hv; + + for (hv = 0; hv < ht->num_buckets; hv++) { + for (entry = ht->buckets[hv]; entry; entry = entry->next) { + void *data = CFUHASH_ENTRY_DATA_AS_VOID_P(entry); + size += ht->get_data_size_func(data); + } + } + } + return size; +} + +/* + Returns one if the entry was found, zero otherwise. If found, r is + changed to point to the data in the entry. +*/ +static cfuhash_entry * +cfuhash_get_entry(cfuhash_table_t *ht, const void *key) +{ + Py_uhash_t key_hash; + size_t index; + cfuhash_entry *entry; + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + for (entry = ht->buckets[index]; entry != NULL; entry = entry->next) { + if (entry->key_hash == key_hash && ht->compare_func(key, entry) == 0) + break; + } + + return entry; +} + +static cfuhash_entry * +_cfuhash_pop_entry(cfuhash_table_t *ht, const void *key) +{ + Py_uhash_t key_hash; + size_t index; + cfuhash_entry *entry, *previous; + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + previous = NULL; + for (entry = ht->buckets[index]; entry != NULL; entry = entry->next) { + if (entry->key_hash == key_hash && ht->compare_func(key, entry) == 0) + break; + previous = entry; + } + + if (entry == NULL) + return NULL; + + if (previous != NULL) + previous->next = entry->next; + else + ht->buckets[index] = entry->next; + ht->entries--; + + if ((float)ht->entries / (float)ht->num_buckets < CFUHASH_LOW) + _cfuhash_rehash(ht); + + return entry; +} + +/* Add a new entry to the hash. Return 0 on success, -1 on memory error. */ +static int +cfuhash_put_data(cfuhash_table_t *ht, const void *key, + void *data, size_t data_size) +{ + Py_uhash_t key_hash; + size_t index; + cfuhash_entry *entry; + size_t size; + + assert(data != NULL); + assert(cfuhash_get_entry(ht, key) == NULL); + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + size = sizeof(cfuhash_entry) + ht->data_size; + entry = (cfuhash_entry *)malloc(size); + if (entry == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("memory allocation failed in cfuhash_put_data()"); +#endif + return -1; + } + + entry->next = ht->buckets[index]; + entry->key = (void *)key; + entry->key_hash = key_hash; + CFUHASH_WRITE_DATA(ht, entry, data, data_size); + + ht->buckets[index] = entry; + ht->entries++; + + if ((float)ht->entries / (float)ht->num_buckets > CFUHASH_HIGH) + _cfuhash_rehash(ht); + return 0; +} + +/* Get data from an entry. Copy entry data into data and return 1 if the entry + exists, return 0 if the entry does not exist. */ +static int +cfuhash_get_data(cfuhash_table_t *ht, const void *key, + void *data, size_t data_size) +{ + cfuhash_entry *entry; + + assert(data != NULL); + + entry = cfuhash_get_entry(ht, key); + if (entry == NULL) + return 0; + CFUHASH_READ_DATA(ht, data, data_size, entry); + return 1; +} + +static int +cfuhash_pop_data(cfuhash_table_t *ht, const void *key, + void *data, size_t data_size) +{ + cfuhash_entry *entry; + + assert(data != NULL); + assert(ht->free_data_func == NULL); + + entry = _cfuhash_pop_entry(ht, key); + if (entry == NULL) + return 0; + + CFUHASH_READ_DATA(ht, data, data_size, entry); + free(entry); + return 1; +} + +static void +cfuhash_delete_data(cfuhash_table_t *ht, const void *key) +{ + cfuhash_entry *entry = _cfuhash_pop_entry(ht, key); + assert (entry != NULL); + if (ht->free_data_func) + ht->free_data_func(CFUHASH_ENTRY_DATA_AS_VOID_P(entry)); + free(entry); +} + +/* Prototype for a pointer to a function to be called foreach + key/value pair in the hash by cfuhash_foreach(). Iteration + stops if a non-zero value is returned. */ +static int +cfuhash_foreach(cfuhash_table_t *ht, + int (*fe_fn) (cfuhash_entry *entry, void *arg), + void *arg) +{ + cfuhash_entry *entry; + size_t hv; + + for (hv = 0; hv < ht->num_buckets; hv++) { + for (entry = ht->buckets[hv]; entry; entry = entry->next) { + int res = fe_fn(entry, arg); + if (res) + return res; + } + } + return 0; +} + +static void +_cfuhash_rehash(cfuhash_table_t *ht) +{ + size_t new_size, bucket; + cfuhash_entry **new_buckets; + size_t old_num_buckets; + + new_size = cfuhash_hash_size(ht->entries * 2 / (CFUHASH_HIGH + CFUHASH_LOW)); + if (new_size == ht->num_buckets) + return; + old_num_buckets = ht->num_buckets; + + new_buckets = (cfuhash_entry **)calloc(new_size, sizeof(cfuhash_entry *)); + if (new_buckets == NULL) { + /* cancel rehash on memory allocation failure */ +#ifdef TRACE_DEBUG + tracemalloc_error("memory allocation failed in _cfuhash_rehash()"); +#endif + return; + } + + ht->num_buckets = new_size; + + for (bucket = 0; bucket < old_num_buckets; bucket++) { + cfuhash_entry *entry = ht->buckets[bucket]; + while (entry) { + cfuhash_entry *next_entry; + size_t entry_index; + + assert(ht->hash_func(entry->key) == entry->key_hash); + entry_index = entry->key_hash & (new_size - 1); + next_entry = entry->next; + entry->next = new_buckets[entry_index]; + new_buckets[entry_index] = entry; + entry = next_entry; + } + } + + free(ht->buckets); + ht->buckets = new_buckets; +} + +static void +cfuhash_clear(cfuhash_table_t *ht) +{ + cfuhash_entry *entry; + size_t i; + + for (i=0; i < ht->num_buckets; i++) { + entry = ht->buckets[i]; + while (entry) { + cfuhash_entry *hep = entry; + entry = entry->next; + if (ht->free_data_func) + ht->free_data_func(CFUHASH_ENTRY_DATA_AS_VOID_P(hep)); + free(hep); + } + ht->buckets[i] = NULL; + } + ht->entries = 0; + _cfuhash_rehash(ht); +} + +static void +cfuhash_destroy(void *ptr) +{ + cfuhash_table_t *ht = (cfuhash_table_t *)ptr; + size_t i; + + for (i = 0; i < ht->num_buckets; i++) { + if (ht->buckets[i]) { + cfuhash_entry *entry = ht->buckets[i]; + while (entry) { + cfuhash_entry *entry_next = entry->next; + if (ht->free_data_func) + ht->free_data_func(CFUHASH_ENTRY_DATA_AS_VOID_P(entry)); + free(entry); + entry = entry_next; + } + } + } +#ifdef Py_DEBUG + memset(ht->buckets, 0xdb, ht->num_buckets * sizeof(ht->buckets[0])); +#endif + free(ht->buckets); +#ifdef Py_DEBUG + memset(ht, 0xdb, sizeof(*ht)); +#endif + free(ht); +} + +/* Return a copy of the hash table */ +static cfuhash_table_t * +cfuhash_copy(cfuhash_table_t *src) +{ + cfuhash_table_t *dst; + cfuhash_entry *entry; + size_t bucket; + int err; + void *data, *new_data; + + dst = cfuhash_new_full(src->data_size, src->num_buckets, + src->hash_func, src->compare_func, + src->copy_data_func, src->free_data_func, + src->get_data_size_func); + if (dst == NULL) + return NULL; + + for (bucket=0; bucket < src->num_buckets; bucket++) { + entry = src->buckets[bucket]; + for (; entry; entry = entry->next) { + if (src->copy_data_func) { + data = CFUHASH_ENTRY_DATA_AS_VOID_P(entry); + new_data = src->copy_data_func(data); + if (new_data != NULL) + err = cfuhash_put_data(dst, entry->key, + &new_data, src->data_size); + else + err = 1; + } + else { + data = CFUHASH_ENTRY_DATA(entry); + err = cfuhash_put_data(dst, entry->key, data, src->data_size); + } + if (err) { + cfuhash_destroy(dst); + return NULL; + } + } + } + return dst; +} + +/* arbitrary limit of the number of frames in a traceback to not allocate + too much memory on the stack (see below how the constant is used) */ +#define MAX_NFRAME 100 + +/* arbitrary limit the depth of the recursive matching function + to avoid a stack overflow */ +#define MAX_NJOKER 100 + +static struct { + PyMemAllocator mem; + PyMemAllocator raw; + PyMemAllocator obj; +} allocators; + +typedef struct { + PyObject *filename; + int lineno; +} tracemalloc_frame_t; + +struct { + /* tracemalloc_init() was already called? */ + int init; + /* is the module enabled? */ + int enabled; + /* guard to avoid reentrant calls to tracemalloc_malloc() + and tracemalloc_realloc() */ + int reentrant; + /* limit of the number of frames in a traceback, 1 by default */ + int max_nframe; +} tracemalloc_config = {0, 0, 0, 1}; + +typedef struct { + /* include (1) or exclude (0) matching frame? */ + int include; + PyObject *pattern; +#ifndef TRACE_NORMALIZE_FILENAME + int use_joker; +#endif + /* ignore any line number if lineno < 1 */ + int lineno; + /* use the whole traceback, or only the most recent frame? */ + int traceback; +} tracemalloc_filter_t; + +typedef struct { + size_t nfilter; + tracemalloc_filter_t *filters; +} tracemalloc_filters_t; + +#ifdef WITH_THREAD +static PyThread_type_lock tracemalloc_filters_lock = NULL; +#endif + +static tracemalloc_filters_t tracemalloc_include_filters; +static tracemalloc_filters_t tracemalloc_exclude_filters; + +typedef struct { + size_t size; + int nframe; + /* data follows: + + PyObject *filenames[tracemalloc_config->nframe]; + int linenos[tracemalloc_config->nframe]; + + Filenames are borrowed references: the tracemalloc_filenames table + keeps a reference to the filename. + + Use TRACE_FILENAMES() and TRACE_LINENOS() to access these fields. + */ +} tracemalloc_trace_t; + +#define TRACE_SIZE(NFRAME) \ + (sizeof(tracemalloc_trace_t) \ + + NFRAME * sizeof(PyObject*) \ + + NFRAME * sizeof(int)) +#define TRACE_FILENAMES(TRACE) \ + ((PyObject**)((char*)(TRACE) \ + + sizeof(tracemalloc_trace_t))) +#define TRACE_LINENOS(TRACE, NFRAME) \ + ((int *)((char*)TRACE_FILENAMES(TRACE) \ + + (NFRAME) * sizeof(PyObject*))) + +static struct { + int enabled; + int delay; + time_t next_trigger; + PyObject *callback; + PyObject *args; + PyObject *kwargs; +} tracemalloc_timer; + +typedef struct { + size_t size; + size_t count; +} tracemalloc_trace_stat_t; + +/* Hash table used to intern filenames */ +static size_t tracemalloc_python_memory = 0; + +/* Hash table used to intern filenames */ +static cfuhash_table_t *tracemalloc_filenames = NULL; + +/* Statistics on Python memory allocations per file and per line: + {filename: PyObject* => {lineno: int => stat: tracemalloc_trace_stat_t} */ +static cfuhash_table_t *tracemalloc_file_stats = NULL; + +/* pointer (void*) => trace (tracemalloc_trace_t*) */ +static cfuhash_table_t *tracemalloc_allocs = NULL; + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) +# define TRACE_LOCK() PyThread_acquire_lock(tracemalloc_lock, 1) +# define TRACE_UNLOCK() PyThread_release_lock(tracemalloc_lock) +PyThread_type_lock tracemalloc_lock; +#else +# define TRACE_LOCK() +# define TRACE_UNLOCK() +#endif + +static void +tracemalloc_get_frame(PyFrameObject *frame, + PyObject **filename_p, int *lineno_p) +{ + PyCodeObject *code; + PyObject *filename; + cfuhash_entry *entry; + char dummy_value = 1; + + *filename_p = NULL; + *lineno_p = PyFrame_GetLineNumber(frame); + + code = frame->f_code; + if (code == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error( + "failed to get the code object of " + "the a frame (thread %li)\n", + PyThread_get_thread_ident()); +#endif + return; + } + + if (code->co_filename == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error( + "failed to get the filename of the code object " + "(thread %li)\n", + PyThread_get_thread_ident()); +#endif + return; + } + + filename = code->co_filename; + assert(filename != NULL); + + if (!PyUnicode_CheckExact(filename)) { +#ifdef TRACE_DEBUG + tracemalloc_error("filename is not an unicode string\n"); +#endif + return; + } + if (!PyUnicode_IS_READY(filename)) { + /* Don't make a Unicode string ready to avoid reentrant calls + to tracemalloc_malloc() or tracemalloc_realloc() */ +#ifdef TRACE_DEBUG + tracemalloc_error("filename is not a ready unicode string\n"); +#endif + return; + } + + /* intern the filename */ + TRACE_LOCK(); + entry = cfuhash_get_entry(tracemalloc_filenames, filename); + if (entry != NULL) { + filename = (PyObject*)entry->key; + } + else { + /* tracemalloc_filenames is responsible to keep a reference + to the filename */ + Py_INCREF(filename); + if (CFUHASH_PUT_DATA(tracemalloc_filenames, + filename, dummy_value) < 0) { + TRACE_UNLOCK(); +#ifdef TRACE_DEBUG + tracemalloc_error("failed to intern the filename\n"); +#endif + return; + } + } + TRACE_UNLOCK(); + + /* the tracemalloc_filenames table keeps a reference to the filename */ + *filename_p = filename; +} + +static void +tracemalloc_get_frames_gil(tracemalloc_trace_t *trace) +{ + PyThreadState *tstate; + PyFrameObject *frame; + + trace->nframe = 0; + + tstate = PyGILState_GetThisThreadState(); + if (tstate == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error( + "failed to get the current thread state (thread %li)\n", + PyThread_get_thread_ident()); +#endif + return; + } + + frame = tstate->frame; + if (frame == NULL) { + /* during startup and finalization, PyMem_Malloc() may be called + without any Python frame */ + return; + } + + do { + tracemalloc_get_frame(frame, + &TRACE_FILENAMES(trace)[trace->nframe], + &TRACE_LINENOS(trace, tracemalloc_config.max_nframe)[trace->nframe]); + trace->nframe++; + frame = frame->f_back; + if (trace->nframe == tracemalloc_config.max_nframe) + break; + } while (frame != NULL); +} + +static void +tracemalloc_get_frames(tracemalloc_trace_t *trace, int gil_held) +{ +#ifdef TRACE_RAW_MALLOC + if (!gil_held) { + PyGILState_STATE gil_state; + + /* PyGILState_Ensure() may call PyMem_RawMalloc() indirectly. + Traceback of test_capi._test_thread_state(), most recent first: + + PyGILState_Ensure () + tracemalloc_get_filename () + ... + tracemalloc_raw_malloc () + PyMem_RawMalloc () + new_threadstate () + PyThreadState_New () + PyGILState_Ensure () + _make_call () + _make_call_from_thread () Modules/_testcapimodule.c + */ + TRACE_LOCK(); + tracemalloc_config.reentrant = 1; + TRACE_UNLOCK(); + + gil_state = PyGILState_Ensure(); + + TRACE_LOCK(); + tracemalloc_config.reentrant = 0; + TRACE_UNLOCK(); + + tracemalloc_get_frames_gil(trace); + PyGILState_Release(gil_state); + } + else { + tracemalloc_get_frames_gil(trace); + } +#else + assert(gil_held); + tracemalloc_get_frames_gil(trace); +#endif +} + +static int +tracemalloc_timer_call(void *user_data) +{ + PyObject *result; + + result = PyEval_CallObjectWithKeywords(tracemalloc_timer.callback, + tracemalloc_timer.args, + tracemalloc_timer.kwargs); + + tracemalloc_timer.enabled = 1; + tracemalloc_timer.next_trigger = time(NULL) + tracemalloc_timer.delay; + + if (!result) + return -1; + Py_DECREF(result); + return 0; +} + +static void +tracemalloc_timer_check(void) +{ + int res; + + if (time(NULL) < tracemalloc_timer.next_trigger) + return; + + /* don't schedule a new call before the previous call is done */ + tracemalloc_timer.enabled = 0; + + res = Py_AddPendingCall(tracemalloc_timer_call, NULL); + if (res != 0) + return; +} + +static void +tracemalloc_update_stats(tracemalloc_trace_t *trace, int is_alloc) +{ + tracemalloc_trace_stat_t local_trace_stat; + tracemalloc_trace_stat_t *trace_stats; + cfuhash_table_t *line_hash; + void *line_key; + cfuhash_entry *line_entry; + PyObject *filename; + int lineno; + + if (is_alloc) { + assert(tracemalloc_python_memory <= PY_SIZE_MAX - trace->size); + tracemalloc_python_memory += trace->size; + } + else { + assert(tracemalloc_python_memory >= trace->size); + tracemalloc_python_memory -= trace->size; + } + + if (trace->nframe >= 1) { + filename = TRACE_FILENAMES(trace)[0]; + lineno = TRACE_LINENOS(trace, trace->nframe)[0]; + } + else { + filename = NULL; + lineno = -1; + } + + if (!CFUHASH_GET_DATA(tracemalloc_file_stats, filename, line_hash)) { + if (!is_alloc) { + /* clear_traces() was called, or tracemalloc_update_stats() failed + to store the allocation */ + return; + } + + line_hash = cfuhash_new(sizeof(tracemalloc_trace_stat_t), + cfuhash_hash_int, cfuhash_cmp_direct); + if (line_hash == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to allocate a hash table for lines " + "for a new filename"); +#endif + return; + } + + if (CFUHASH_PUT_DATA(tracemalloc_file_stats, + filename, line_hash) < 0) { + cfuhash_destroy(line_hash); + return; + } + } + + line_key = INT_TO_POINTER(lineno); + line_entry = cfuhash_get_entry(line_hash, line_key); + if (line_entry != NULL) { + assert(line_hash->data_size == sizeof(*trace_stats)); + trace_stats = (tracemalloc_trace_stat_t *)CFUHASH_ENTRY_DATA(line_entry); + + if (is_alloc) { + trace_stats->size += trace->size; + assert(trace_stats->count != PY_SIZE_MAX); + trace_stats->count++; + } + else { + assert(trace_stats->count != 0); + trace_stats->size -= trace->size; + trace_stats->count--; + assert(trace_stats->count != 0 || trace_stats->size == 0); + + if (trace_stats->count == 0) { + assert(!tracemalloc_config.reentrant); + + cfuhash_delete_data(line_hash, line_key); + if (line_hash->entries == 0) + cfuhash_delete_data(tracemalloc_file_stats, filename); + } + } + } + else { + if (!is_alloc) { + /* clear_traces() was called, or tracemalloc_update_stats() failed + to store the allocation */ + return; + } + + local_trace_stat.size = trace->size; + local_trace_stat.count = 1; + CFUHASH_PUT_DATA(line_hash, line_key, local_trace_stat); + } +} + +#ifdef TRACE_NORMALIZE_FILENAME +static Py_UCS4 +tracemalloc_normalize_filename(Py_UCS4 ch) +{ +#ifdef ALTSEP + if (ch == ALTSEP) + return SEP; +#endif +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + return ch; +} +#endif + +typedef struct { + PyObject *filename, *pattern; + int file_kind, pat_kind; + void *file_data, *pat_data; + Py_ssize_t file_len, pat_len; +} tracemalloc_match_t; + +static int +tracemalloc_match_filename_joker(tracemalloc_match_t *match, + Py_ssize_t file_pos, Py_ssize_t pat_pos) +{ + Py_UCS4 ch1, ch2; + + while (file_pos < match->file_len) { + ch1 = PyUnicode_READ(match->file_kind, match->file_data, file_pos); + ch2 = PyUnicode_READ(match->pat_kind, match->pat_data, pat_pos); + if (ch2 == '*') { + int found; + + do { + pat_pos++; + if (pat_pos >= match->pat_len) { + /* 'abc' always match '*' */ + return 1; + } + ch2 = PyUnicode_READ(match->pat_kind, match->pat_data, + pat_pos); + } while (ch2 == '*'); + + do { + found = tracemalloc_match_filename_joker(match, + file_pos, pat_pos); + if (found) + break; + file_pos++; + } while (file_pos < match->file_len); + + return found; + } + +#ifdef TRACE_NORMALIZE_FILENAME + ch1 = tracemalloc_normalize_filename(ch1); +#endif + if (ch1 != ch2) + return 0; + + file_pos++; + pat_pos++; + } + + if (pat_pos != match->pat_len) { + if ((pat_pos + 1) == match->pat_len) { + ch2 = PyUnicode_READ(match->pat_kind, match->pat_data, pat_pos); + if (ch2 == '*') { + /* 'abc' matchs 'abc*' */ + return 1; + } + } + return 0; + } + return 1; +} + +static int +tracemalloc_endswith_pyc_pyo(PyObject *filename) +{ + void* data; + int kind; + Py_UCS4 ch; + Py_ssize_t len; + + len = PyUnicode_GetLength(filename); + if (len < 4) + return 0; + + data = PyUnicode_DATA(filename); + kind = PyUnicode_KIND(filename); + + if (PyUnicode_READ(kind, data, len-4) != '.') + return 0; + ch = PyUnicode_READ(kind, data, len-3); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if (ch != 'p') + return 0; + ch = PyUnicode_READ(kind, data, len-2); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if (ch != 'y') + return 0; + ch = PyUnicode_READ(kind, data, len-1); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if ((ch != 'c' && ch != 'o')) + return 0; + return 1; +} + +static int +tracemalloc_match_filename(tracemalloc_filter_t *filter, PyObject *filename) +{ + Py_ssize_t len; + tracemalloc_match_t match; + +#ifndef TRACE_NORMALIZE_FILENAME + if (!filter->use_joker) { + if (!tracemalloc_endswith_pyc_pyo(filename)) { + int res = PyUnicode_Compare(filename, filter->pattern); + return (res == 0); + } + else { + len = PyUnicode_GetLength(filename); + + /* don't compare last character */ + return PyUnicode_Tailmatch(filename, filter->pattern, + 0, len - 1, 1); + } + } +#endif + + len = PyUnicode_GetLength(filename); + + /* replace "a.pyc" and "a.pyo" with "a.py" */ + if (tracemalloc_endswith_pyc_pyo(filename)) + len--; + + match.filename = filename; + match.file_kind = PyUnicode_KIND(match.filename); + match.file_data = PyUnicode_DATA(match.filename); + match.file_len = len; + + match.pattern = filter->pattern; + match.pat_kind = PyUnicode_KIND(match.pattern); + match.pat_data = PyUnicode_DATA(match.pattern); + match.pat_len = PyUnicode_GET_LENGTH(match.pattern); + return tracemalloc_match_filename_joker(&match, 0, 0); +} + +static int +tracemalloc_filter_match_filename(tracemalloc_filter_t *filter, + PyObject *filename) +{ + int match; + + if (filename == NULL) + return !filter->include; + + match = tracemalloc_match_filename(filter, filename); + return match ^ !filter->include; +} + +static int +tracemalloc_filter_match_lineno(tracemalloc_filter_t *filter, + int lineno) +{ + int match; + + if (filter->lineno < 1) + return 1; + + if (lineno < 1) + return !filter->include; + + match = (lineno == filter->lineno); + return match ^ !filter->include; +} + +static int +tracemalloc_filter_match(tracemalloc_filter_t *filter, + PyObject *filename, int lineno) +{ + int match; + + if (filename == NULL) + return !filter->include; + + match = tracemalloc_match_filename(filter, filename); + if (filter->include) { + if (!match) + return 0; + } + else { + /* exclude */ + if (!match) + return 1; + else if (filter->lineno < 1) + return 0; + } + + return tracemalloc_filter_match_lineno(filter, lineno); +} + +static int +tracemalloc_filter_match_trace(tracemalloc_filter_t *filter, + tracemalloc_trace_t *trace) +{ + int i; + PyObject *filename; + int lineno; + int nframe; + int match; + + nframe = trace->nframe; + if (nframe == 0) { + return tracemalloc_filter_match(filter, NULL, -1); + } + else if (!filter->traceback) { + filename = TRACE_FILENAMES(trace)[0]; + lineno = TRACE_LINENOS(trace, nframe)[0]; + + return tracemalloc_filter_match(filter, filename, lineno); + } + else { + for (i = 0; i < nframe; i++) { + filename = TRACE_FILENAMES(trace)[i]; + lineno = TRACE_LINENOS(trace, nframe)[i]; + + match = tracemalloc_filter_match(filter, filename, lineno); + if (match) { + if (filter->include) + return 1; + } + else { + if (!filter->include) + return 0; + } + } + return !filter->include; + } +} + +static int +tracemalloc_filters_match(tracemalloc_filters_t *filters, int include, + tracemalloc_trace_t *trace) +{ + size_t i; + tracemalloc_filter_t *filter; + int match; + + if (filters->nfilter == 0) + return 1; + + for (i=0; infilter; i++) { + filter = &filters->filters[i]; + match = tracemalloc_filter_match_trace(filter, trace); + if (include) { + if (match) + return 1; + } + else { + if (!match) + return 0; + } + } + return !include; +} + +static int +tracemalloc_trace_match_filters(tracemalloc_trace_t *trace) +{ + int match; +#ifdef WITH_THREAD + PyThread_acquire_lock(tracemalloc_filters_lock, 1); +#endif + match = 1; + if (!tracemalloc_filters_match(&tracemalloc_include_filters, 1, trace)) + match = 0; + if (match && !tracemalloc_filters_match(&tracemalloc_exclude_filters, 0, trace)) + match = 0; +#ifdef WITH_THREAD + PyThread_release_lock(tracemalloc_filters_lock); +#endif + return match; +} + +static void +tracemalloc_log_alloc(void *ptr, size_t size, int gil_held) +{ + size_t trace_size; + char stack_buffer[ TRACE_SIZE(MAX_NFRAME) ]; + tracemalloc_trace_t *trace; + + trace_size = TRACE_SIZE(tracemalloc_config.max_nframe); + assert(trace_size <= sizeof(stack_buffer)); + trace = (tracemalloc_trace_t *)stack_buffer; + + trace->size = size; + tracemalloc_get_frames(trace, gil_held); + + if (!tracemalloc_trace_match_filters(trace)) + return; + + TRACE_LOCK(); + tracemalloc_update_stats(trace, 1); + cfuhash_put_data(tracemalloc_allocs, ptr, trace, trace_size); + TRACE_UNLOCK(); +} + +static void +tracemalloc_log_free(void *ptr) +{ + size_t trace_size; + char stack_buffer[ TRACE_SIZE(MAX_NFRAME) ]; + tracemalloc_trace_t *trace; + + trace_size = TRACE_SIZE(tracemalloc_config.max_nframe); + assert(trace_size <= sizeof(stack_buffer)); + trace = (tracemalloc_trace_t *)stack_buffer; + + TRACE_LOCK(); + if (cfuhash_pop_data(tracemalloc_allocs, ptr, trace, trace_size)) + tracemalloc_update_stats(trace, 0); + TRACE_UNLOCK(); +} + +static void* +tracemalloc_malloc(void *ctx, size_t size, int gil_held) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; + void *ptr; + + TRACE_LOCK(); + if (tracemalloc_config.reentrant) { + TRACE_UNLOCK(); + return alloc->malloc(alloc->ctx, size); + } + + /* PyObjet_Malloc() calls PyMem_Malloc() for allocations + larger than 512 bytes */ + tracemalloc_config.reentrant = 1; + TRACE_UNLOCK(); + + ptr = alloc->malloc(alloc->ctx, size); + + TRACE_LOCK(); + tracemalloc_config.reentrant = 0; + TRACE_UNLOCK(); + + if (ptr != NULL) + tracemalloc_log_alloc(ptr, size, gil_held); + + if (tracemalloc_timer.enabled) + tracemalloc_timer_check(); + + return ptr; +} + +static void* +tracemalloc_realloc(void *ctx, void *ptr, size_t new_size, int gil_held) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; + void *ptr2; + + TRACE_LOCK(); + if (tracemalloc_config.reentrant) { + TRACE_UNLOCK(); + return alloc->realloc(alloc->ctx, ptr, new_size); + } + + /* PyObjet_Realloc() calls PyMem_Realloc() for allocations + larger than 512 bytes */ + tracemalloc_config.reentrant = 1; + TRACE_UNLOCK(); + + ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); + + TRACE_LOCK(); + tracemalloc_config.reentrant = 0; + TRACE_UNLOCK(); + + if (ptr2 != NULL) { + if (ptr != NULL) + tracemalloc_log_free(ptr); + + tracemalloc_log_alloc(ptr2, new_size, gil_held); + } + + if (tracemalloc_timer.enabled) + tracemalloc_timer_check(); + + return ptr2; +} + +static void +tracemalloc_free(void *ctx, void *ptr) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; + + if (ptr != NULL) { + alloc->free(alloc->ctx, ptr); + tracemalloc_log_free(ptr); + } + + if (tracemalloc_timer.enabled) + tracemalloc_timer_check(); +} + +static void* +tracemalloc_malloc_gil(void *ctx, size_t size) +{ + return tracemalloc_malloc(ctx, size, 1); +} + +static void* +tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) +{ + return tracemalloc_realloc(ctx, ptr, new_size, 1); +} + +#ifdef TRACE_RAW_MALLOC +static void* +tracemalloc_raw_malloc(void *ctx, size_t size) +{ + return tracemalloc_malloc(ctx, size, 0); +} + +static void* +tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) +{ + return tracemalloc_realloc(ctx, ptr, new_size, 0); +} +#endif + +static int +tracemalloc_filter_init(tracemalloc_filter_t *filter, + int include, PyObject *pattern, int lineno, + int traceback) +{ + Py_ssize_t len, len2; + Py_ssize_t i, j; + PyObject *new_pattern; + Py_UCS4 maxchar, ch; + int kind, kind2; + void *data, *data2; + int previous_joker; + size_t njoker; + + if (PyUnicode_READY(pattern) < 0) + return -1; + + len = PyUnicode_GetLength(pattern); + kind = PyUnicode_KIND(pattern); + data = PyUnicode_DATA(pattern); + + if (tracemalloc_endswith_pyc_pyo(pattern)) + len--; + + maxchar = 0; + len2 = 0; + njoker = 0; + previous_joker = 0; + for (i=0; i < len; i++) { + ch = PyUnicode_READ(kind, data, i); +#ifdef TRACE_NORMALIZE_FILENAME + ch = tracemalloc_normalize_filename(ch); +#endif + if (!previous_joker || ch != '*') { + previous_joker = (ch == '*'); + if (previous_joker) + njoker++; + maxchar = Py_MAX(maxchar, ch); + len2++; + } + else { + /* skip consecutive joker character */ + } + } + + if (njoker > MAX_NJOKER) { + PyErr_SetString(PyExc_ValueError, + "too many joker characters in the filename pattern"); + return -1; + } + + new_pattern = PyUnicode_New(len2, maxchar); + if (new_pattern == NULL) + return -1; + kind2 = PyUnicode_KIND(new_pattern); + data2 = PyUnicode_DATA(new_pattern); + + j = 0; + previous_joker = 0; + for (i=0; i < len; i++) { + ch = PyUnicode_READ(kind, data, i); +#ifdef TRACE_NORMALIZE_FILENAME + ch = tracemalloc_normalize_filename(ch); +#endif + if (!previous_joker || ch != '*') { + previous_joker = (ch == '*'); + PyUnicode_WRITE(kind2, data2, j, ch); + j++; + } + else { + /* skip consecutive joker character */ + } + } + assert(j == len2); + + assert(_PyUnicode_CheckConsistency(new_pattern, 1)); + pattern = new_pattern; + + filter->include = include; + filter->pattern = pattern; +#ifndef TRACE_NORMALIZE_FILENAME + filter->use_joker = (njoker != 0); +#endif + if (lineno >= 1) + filter->lineno = lineno; + else + filter->lineno = -1; + filter->traceback = traceback; + return 0; +} + +static void +tracemalloc_filter_deinit(tracemalloc_filter_t *filter) +{ + Py_CLEAR(filter->pattern); +} + +static void +tracemalloc_filters_init(tracemalloc_filters_t *filters) +{ + filters->nfilter = 0; + filters->filters = NULL; +} + +static void +tracemalloc_filters_clear(tracemalloc_filters_t *filters) +{ + size_t i; + + if (filters->filters == NULL) { + assert(filters->nfilter == 0); + return; + } + + for (i=0; infilter; i++) + tracemalloc_filter_deinit(&filters->filters[i]); + + filters->nfilter = 0; + free(filters->filters); + filters->filters = NULL; +} + +static void +tracemalloc_clear_filters(void) +{ +#ifdef WITH_THREAD + PyThread_acquire_lock(tracemalloc_filters_lock, 1); +#endif + tracemalloc_filters_clear(&tracemalloc_include_filters); + tracemalloc_filters_clear(&tracemalloc_exclude_filters); +#ifdef WITH_THREAD + PyThread_release_lock(tracemalloc_filters_lock); +#endif +} + +static int +tracemalloc_clear_filename(cfuhash_entry *entry, void *user_data) +{ + PyObject *filename = (PyObject *)entry->key; + Py_DECREF(filename); + return 0; +} + +static void +tracemalloc_clear_traces(void) +{ + cfuhash_clear(tracemalloc_file_stats); + cfuhash_clear(tracemalloc_allocs); + + cfuhash_foreach(tracemalloc_filenames, tracemalloc_clear_filename, NULL); + cfuhash_clear(tracemalloc_filenames); + tracemalloc_python_memory = 0; +} + +static void +tracemalloc_disable(void) +{ + TRACE_LOCK(); + if (!tracemalloc_config.enabled) { + TRACE_UNLOCK(); + return; + } + tracemalloc_config.enabled = 0; + tracemalloc_config.reentrant = 1; + + /* unregister the hook on memory allocators */ +#ifdef TRACE_RAW_MALLOC + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); +#endif + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); + + /* release memory */ + tracemalloc_clear_traces(); + + TRACE_UNLOCK(); +} + +static void +tracemalloc_deinit(void) +{ + tracemalloc_disable(); + tracemalloc_clear_filters(); + + /* destroy hash tables */ + cfuhash_destroy(tracemalloc_file_stats); + tracemalloc_file_stats = NULL; + + cfuhash_destroy(tracemalloc_allocs); + tracemalloc_allocs = NULL; + + cfuhash_destroy(tracemalloc_filenames); + tracemalloc_filenames = NULL; + +#ifdef WITH_THREAD +#ifdef TRACE_RAW_MALLOC + PyThread_free_lock(tracemalloc_lock); +#endif + PyThread_free_lock(tracemalloc_filters_lock); +#endif +} + +#ifdef TRACE_ATFORK +static void +tracemalloc_atfork(void) +{ + tracemalloc_deinit(); +} +#endif + +PyDoc_STRVAR(tracemalloc_frame_structseq__doc__, +"Frame: trace of a Python frame."); + +static PyStructSequence_Field tracemalloc_frame_structseq_fields[] = { + {"filename", "Python filename, None if unknown"}, + {"lineno", "Python line number, None if unknown"}, + {0} +}; + +static PyStructSequence_Desc tracemalloc_frame_structseq_desc = { + "tracemalloc.Frame", /* name */ + tracemalloc_frame_structseq__doc__, /* doc */ + tracemalloc_frame_structseq_fields, + 2 +}; + +static PyTypeObject FrameType; + + +PyDoc_STRVAR(tracemalloc_structseq__doc__, +"Trace: debug information of an allocated memory block."); + +static PyStructSequence_Field tracemalloc_structseq_fields[] = { + {"size", "size in bytes of the memory block"}, + {"traceback", "traceback where the memory block was allocated as " + "a list of tracemalloc.frame instances (most recent first). " + "The list can be empty or incomplete if the tracemalloc " + "module was unable to retrieve the full traceback. " + "For efficiency, the traceback is truncated to 10 frames."}, + {0} +}; + +static PyStructSequence_Desc tracemalloc_structseq_desc = { + "tracemalloc.Trace", /* name */ + tracemalloc_structseq__doc__, /* doc */ + tracemalloc_structseq_fields, + 2 +}; + +static PyTypeObject TraceType; + + +PyDoc_STRVAR(trace_stat_structseq__doc__, +"TraceStats: statistics on Python memory allocations\n" +"of a specific line number."); + +static PyStructSequence_Field trace_stat_structseq_fields[] = { + {"size", "total size in bytes of all memory blocks allocated on the line"}, + {"count", "number of memory blocks allocated on the line"}, + {0} +}; + +static PyStructSequence_Desc trace_stat_structseq_desc = { + "tracemalloc.TraceStats", /* name */ + trace_stat_structseq__doc__, /* doc */ + trace_stat_structseq_fields, + 2 +}; + +static PyTypeObject LineStatsType; + +static int +tracemalloc_init(void) +{ +#ifdef TRACE_ATFORK + int res; +#endif + + if (tracemalloc_config.init) + return 0; + + if (PyStructSequence_InitType2(&TraceType, + &tracemalloc_structseq_desc) < 0) + return -1; + if (PyStructSequence_InitType2(&FrameType, + &tracemalloc_frame_structseq_desc) < 0) + return -1; + if (PyStructSequence_InitType2(&LineStatsType, + &trace_stat_structseq_desc) < 0) + return -1; + +#ifdef TRACE_ATFORK + res = pthread_atfork(NULL, NULL, tracemalloc_atfork); + if (res != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } +#endif + + tracemalloc_filters_init(&tracemalloc_include_filters); + tracemalloc_filters_init(&tracemalloc_exclude_filters); + +#ifdef WITH_THREAD +#ifdef TRACE_RAW_MALLOC + tracemalloc_lock = PyThread_allocate_lock(); + if (PyThread_allocate_lock() == NULL) { + PyErr_SetString(PyExc_RuntimeError, "can't allocate lock"); + return -1; + } +#endif + + if (tracemalloc_filters_lock == NULL) { + tracemalloc_filters_lock = PyThread_allocate_lock(); + if (PyThread_allocate_lock() == NULL) { + PyErr_SetString(PyExc_RuntimeError, "can't allocate lock"); + return -1; + } + } +#endif + + if (tracemalloc_filenames == NULL) { + tracemalloc_filenames = cfuhash_new(sizeof(char), + cfuhash_hash_pyobject, + cfuhash_cmp_unicode); + if (tracemalloc_filenames == NULL) { + PyErr_NoMemory(); + return -1; + } + } + + if (tracemalloc_file_stats == NULL) { + tracemalloc_file_stats = cfuhash_new_full(sizeof(cfuhash_table_t *), + 0, + cfuhash_hash_ptr, + cfuhash_cmp_direct, + (cfuhash_copy_data_func)cfuhash_copy, + cfuhash_destroy, + (cfuhash_get_data_size_func)cfuhash_get_size); + if (tracemalloc_file_stats == NULL) { + PyErr_NoMemory(); + return -1; + } + } + + if (tracemalloc_allocs == NULL) { + size_t trace_size; + trace_size = TRACE_SIZE(tracemalloc_config.max_nframe); + tracemalloc_allocs = cfuhash_new(trace_size, + cfuhash_hash_ptr, cfuhash_cmp_direct); + if (tracemalloc_allocs == NULL) { + PyErr_NoMemory(); + return -1; + } + } + + tracemalloc_config.init = 1; + return 0; +} + +static int +tracemalloc_enable(void) +{ + PyMemAllocator alloc; + + if (tracemalloc_config.enabled) { + /* hook already installed: do nothing */ + return 0; + } + + if (tracemalloc_init() < 0) + return -1; + + tracemalloc_python_memory = 0; + +#ifdef TRACE_RAW_MALLOC + alloc.malloc = tracemalloc_raw_malloc; + alloc.realloc = tracemalloc_raw_realloc; + alloc.free = tracemalloc_free; + + alloc.ctx = &allocators.raw; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); +#endif + + alloc.malloc = tracemalloc_malloc_gil; + alloc.realloc = tracemalloc_realloc_gil; + alloc.free = tracemalloc_free; + + alloc.ctx = &allocators.mem; + PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + + alloc.ctx = &allocators.obj; + PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + + TRACE_LOCK(); + tracemalloc_config.enabled = 1; + tracemalloc_config.reentrant = 0; + TRACE_UNLOCK(); + return 0; +} + +typedef struct { + cfuhash_table_t *tracemalloc_file_stats; + cfuhash_table_t *line_hash; + PyObject *file_dict; + PyObject *line_dict; +} tracemalloc_get_stats_t; + +static PyObject* +tracemalloc_lineno_as_obj(int lineno) +{ + if (lineno > 0) + return PyLong_FromLong(lineno); + else + Py_RETURN_NONE; +} + +static PyObject* +tracemalloc_trace_stat_to_pyobject(tracemalloc_trace_stat_t *trace_stats) +{ + PyObject *size, *count, *line_obj; + + line_obj = PyStructSequence_New(&LineStatsType); + if (line_obj == NULL) + return NULL; + + size = PyLong_FromSize_t(trace_stats->size); + if (size == NULL) { + Py_DECREF(line_obj); + return NULL; + } + PyStructSequence_SET_ITEM(line_obj, 0, size); + + count = PyLong_FromSize_t(trace_stats->count); + if (count == NULL) { + Py_DECREF(line_obj); + return NULL; + } + PyStructSequence_SET_ITEM(line_obj, 1, count); + + return line_obj; +} + +static int +tracemalloc_get_stats_fill_line(cfuhash_entry *entry, void *user_data) +{ + int lineno; + tracemalloc_trace_stat_t trace_stats; + tracemalloc_get_stats_t *get_stats = user_data; + PyObject *key, *line_obj; + int err; + + lineno = POINTER_TO_INT(entry->key); + + CFUHASH_READ_DATA(get_stats->line_hash, + &trace_stats, sizeof(trace_stats), entry); + + key = tracemalloc_lineno_as_obj(lineno); + if (key == NULL) + return 1; + + line_obj = tracemalloc_trace_stat_to_pyobject(&trace_stats); + if (line_obj == NULL) { + Py_XDECREF(key); + return 1; + } + + err = PyDict_SetItem(get_stats->line_dict, key, line_obj); + Py_DECREF(key); + Py_DECREF(line_obj); + return err; +} + +static int +tracemalloc_get_stats_fill_file(cfuhash_entry *entry, void *user_data) +{ + PyObject *filename = (PyObject *)entry->key; + tracemalloc_get_stats_t *get_stats = user_data; + int err = 1; + int res; + + CFUHASH_READ_DATA(get_stats->tracemalloc_file_stats, + &get_stats->line_hash, sizeof(get_stats->line_hash), + entry); + + if (filename == NULL) + filename = Py_None; + + get_stats->line_dict = PyDict_New(); + if (get_stats->line_dict == NULL) + goto done; + + res = cfuhash_foreach(get_stats->line_hash, + tracemalloc_get_stats_fill_line, user_data); + if (res) + goto done; + + res = PyDict_SetItem(get_stats->file_dict, filename, get_stats->line_dict); + Py_CLEAR(get_stats->line_dict); + if (res < 0) + goto done; + + err = 0; + +done: + Py_XDECREF(get_stats->line_dict); + return err; +} + +typedef struct { + PyObject_HEAD + tracemalloc_filter_t filter; +} FilterObject; + +/* Converter for PyArg_ParseTuple() to parse a filename, accepting None */ +static int +tracemalloc_parse_filename(PyObject* arg, void* addr) +{ + PyObject *filename; + + if (arg == Py_None) { + filename = NULL; + } + else if (PyUnicode_Check(arg)) { + filename = arg; + } + else { + PyErr_SetString(PyExc_TypeError, "filename must be a str or None"); + return 0; + } + *(PyObject **)addr = filename; + return 1; +} + +/* Converter for PyArg_ParseTuple() to parse a line number, accepting None */ +static int +tracemalloc_parse_lineno(PyObject* arg, void* addr) +{ + int lineno; + + if (arg == Py_None) { + lineno = -1; + } + else { + lineno = _PyLong_AsInt(arg); + if (lineno == -1 && PyErr_Occurred()) + return 0; + } + *(int *)addr = lineno; + return 1; +} + +static int +tracemalloc_pyfilter_init(FilterObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"include", "filename", "lineno", "traceback", 0}; + int include; + PyObject *filename; + int lineno = -1; + int traceback = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iO|O&i:str", kwlist, + &include, &filename, + tracemalloc_parse_lineno, &lineno, + &traceback)) + return -1; + + tracemalloc_filter_deinit(&self->filter); + + if (tracemalloc_filter_init(&self->filter, include, filename, lineno, + traceback) < 0) + return -1; + + return 0; +} + +static void +tracemalloc_pyfilter_dealloc(PyObject *self) +{ + FilterObject *pyfilter = (FilterObject *)self; + tracemalloc_filter_deinit(&pyfilter->filter); + PyObject_FREE(self); +} + +static PyObject* +tracemalloc_pyfilter_match(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *filename; + int lineno = -1; + int match; + + if (!PyArg_ParseTuple(args, "O&O&:match", + tracemalloc_parse_filename, &filename, + tracemalloc_parse_lineno, &lineno)) + return NULL; + + match = tracemalloc_filter_match(&pyfilter->filter, filename, lineno); + return PyBool_FromLong(match); +} + +static int +parse_trace(PyObject *pytrace, tracemalloc_trace_t *trace, size_t buffer_size) +{ + PyObject *pysize, *traceback, *pyframe, *filename, *pylineno; + Py_ssize_t nframe, i; + int lineno; + + pysize = PyStructSequence_GET_ITEM(pytrace, 0); + assert(pysize != NULL); + trace->size = PyLong_AsSsize_t(pysize); + if (trace->size == -1 && PyErr_Occurred()) + return -1; + + traceback = PyStructSequence_GET_ITEM(pytrace, 1); + assert(traceback != NULL); + if (!PyList_Check(traceback)) { + PyErr_SetString(PyExc_TypeError, "traceback must be a list"); + return -1; + } + + nframe = PyList_GET_SIZE(traceback); + if (nframe > MAX_NFRAME || TRACE_SIZE(nframe) > buffer_size) { + PyErr_SetString(PyExc_TypeError, "too many frames"); + return -1; + } + + trace->nframe = nframe; + for (i=0; i < nframe; i++) { + pyframe = PyList_GET_ITEM(traceback, i); + assert(pyframe != NULL); + if (Py_TYPE(pyframe) != &FrameType) { + PyErr_SetString(PyExc_TypeError, "frames must be Frame instances"); + return -1; + } + + filename = PyStructSequence_GET_ITEM(pyframe, 0); + assert(filename != NULL); + pylineno = PyStructSequence_GET_ITEM(pyframe, 1); + assert(pylineno != NULL); + if (tracemalloc_parse_lineno(pylineno, &lineno) == 0) + return -1; + + /* borrowed reference to filename */ + TRACE_FILENAMES(trace)[i] = filename; + TRACE_LINENOS(trace, nframe)[i] = lineno; + } + + return 0; +} + +static PyObject* +tracemalloc_pyfilter_match_trace(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *pytrace; + int match; + char stack_buffer[ TRACE_SIZE(MAX_NFRAME) ]; + tracemalloc_trace_t *trace; + + if (!PyArg_ParseTuple(args, "O!:match_trace", + &TraceType, &pytrace)) + return NULL; + + trace = (tracemalloc_trace_t *)stack_buffer; + + if (parse_trace(pytrace, trace, sizeof(stack_buffer)) < 0) + return NULL; + + match = tracemalloc_filter_match_trace(&pyfilter->filter, trace); + return PyBool_FromLong(match); +} + +static PyObject* +tracemalloc_pyfilter_match_filename(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *filename; + int match; + + if (!PyArg_ParseTuple(args, "O&:match_filename", + tracemalloc_parse_filename, &filename)) + return NULL; + + match = tracemalloc_filter_match_filename(&pyfilter->filter, filename); + return PyBool_FromLong(match); +} + +static PyObject* +tracemalloc_pyfilter_match_lineno(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + int lineno; + int match; + + if (!PyArg_ParseTuple(args, "O&:match_lineno", + tracemalloc_parse_lineno, &lineno)) + return NULL; + + match = tracemalloc_filter_match_lineno(&pyfilter->filter, lineno); + return PyBool_FromLong(match); +} + +static PyObject * +tracemalloc_pyfilter_get_include(FilterObject *self, void *closure) +{ + return PyBool_FromLong(self->filter.include); +} + +static PyObject * +tracemalloc_pyfilter_get_pattern(FilterObject *self, void *closure) +{ + Py_INCREF(self->filter.pattern); + return self->filter.pattern; +} + +static PyObject * +tracemalloc_pyfilter_get_lineno(FilterObject *self, void *closure) +{ + return tracemalloc_lineno_as_obj(self->filter.lineno); +} + +static PyObject * +tracemalloc_pyfilter_get_traceback(FilterObject *self, void *closure) +{ + return PyBool_FromLong(self->filter.traceback); +} + +static PyObject* +tracemalloc_pyfilter_repr(FilterObject *self) +{ + char lineno[30]; + if (self->filter.lineno > 1) + PyOS_snprintf(lineno, sizeof(lineno), "%i", self->filter.lineno); + else + strcpy(lineno, "None"); + return PyUnicode_FromFormat("", + self->filter.include ? "True" : "False", + self->filter.pattern, + lineno, + self->filter.traceback ? "True" : "False"); +} + +static int +tracemalloc_pyfilter_compare(tracemalloc_filter_t *f1, + tracemalloc_filter_t *f2) +{ + if (f1->include != f2->include) + return 0; + if (PyUnicode_Compare(f1->pattern, f2->pattern) != 0) + return 0; + if (f1->lineno != f2->lineno) + return 0; + if (f1->traceback != f2->traceback) + return 0; +#ifndef TRACE_NORMALIZE_FILENAME + assert(f1->use_joker ==f2->use_joker); +#endif + return 1; +} + +static Py_hash_t +tracemalloc_pyfilter_hash(FilterObject *self) +{ + Py_hash_t hash; + + hash = PyObject_Hash(self->filter.pattern); + hash ^= self->filter.lineno; + hash ^= ((Py_hash_t)self->filter.include << 20); + hash ^= ((Py_hash_t)self->filter.traceback << 21); + return hash; +} + +static PyObject * +tracemalloc_pyfilter_richcompare(FilterObject *self, FilterObject *other, int op) +{ + if (op == Py_EQ || op == Py_NE) { + int eq; + PyObject *res; + + eq = tracemalloc_pyfilter_compare(&self->filter, &other->filter); + if (op == Py_NE) + eq = !eq; + + if (eq) + res = Py_True; + else + res = Py_False; + Py_INCREF(res); + return res; + } + else { + Py_RETURN_NOTIMPLEMENTED; + } +} + +static PyGetSetDef tracemalloc_pyfilter_getset[] = { + {"include", (getter) tracemalloc_pyfilter_get_include, NULL, + "Include or exclude the trace?"}, + {"pattern", (getter) tracemalloc_pyfilter_get_pattern, NULL, + "Pattern matching a filename, can contain one " + "or many '*' joker characters"}, + {"lineno", (getter) tracemalloc_pyfilter_get_lineno, NULL, + "Line number"}, + {"traceback", (getter) tracemalloc_pyfilter_get_traceback, NULL, + "Check the whole traceback, or only the most recent frame?"}, + {NULL} +}; + +static PyMethodDef tracemalloc_pyfilter_methods[] = { + {"match", (PyCFunction)tracemalloc_pyfilter_match, + METH_VARARGS, + PyDoc_STR("match(filename: str, lineno: int) -> bool")}, + {"match_trace", (PyCFunction)tracemalloc_pyfilter_match_trace, + METH_VARARGS, + PyDoc_STR("match_trace(trace) -> bool")}, + {"match_filename", (PyCFunction)tracemalloc_pyfilter_match_filename, + METH_VARARGS, + PyDoc_STR("match_filename(filename: str) -> bool")}, + {"match_lineno", (PyCFunction)tracemalloc_pyfilter_match_lineno, + METH_VARARGS, + PyDoc_STR("match_lineno(lineno: int) -> bool")}, + {NULL, NULL} /* sentinel */ +}; + +PyDoc_STRVAR(tracemalloc_pyfilter_doc, +"Filter(include: bool, filename: str, lineno: int=None, traceback: bool=False)"); + +static PyTypeObject FilterType = { + /* The ob_type field must be initialized in the module init function + * to be portable to Windows without using C++. */ + PyVarObject_HEAD_INIT(NULL, 0) + "tracemalloc.Filter", /*tp_name*/ + sizeof(FilterObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)tracemalloc_pyfilter_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + (getattrfunc)0, /*tp_getattr*/ + (setattrfunc)0, /*tp_setattr*/ + 0, /*tp_reserved*/ + (reprfunc)tracemalloc_pyfilter_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + (hashfunc)tracemalloc_pyfilter_hash, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + (getattrofunc)0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + tracemalloc_pyfilter_doc, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + (richcmpfunc)tracemalloc_pyfilter_richcompare, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + tracemalloc_pyfilter_methods, /*tp_methods*/ + 0, /*tp_members*/ + tracemalloc_pyfilter_getset, /* tp_getset */ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + (initproc)tracemalloc_pyfilter_init, /* tp_init */ + 0, /*tp_alloc*/ + PyType_GenericNew, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ +}; + +static PyObject* +py_tracemalloc_is_enabled(PyObject *self) +{ + return PyBool_FromLong(tracemalloc_config.enabled); +} + +PyDoc_STRVAR(tracemalloc_clear_traces_doc, + "clear_traces()\n" + "\n" + "Clear all traces and statistics of memory allocations."); + +static PyObject* +py_tracemalloc_clear_traces(PyObject *self) +{ + if (tracemalloc_config.enabled) { + TRACE_LOCK(); + tracemalloc_clear_traces(); + TRACE_UNLOCK(); + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_stats_doc, +"get_stats() -> dict\n" +"\n" +"Get statistics on Python memory allocations per Python filename and per\n" +"line number.\n" +"\n" +"Return a dictionary {filename: str -> {line_number: int -> stats}}\n" +"where stats in a TraceStats instance.\n" +"\n" +"Return an empty dictionary if the module tracemalloc is disabled."); + +static PyObject* +tracemalloc_get_stats(PyObject *self) +{ + tracemalloc_get_stats_t get_stats; + int err; + + get_stats.file_dict = PyDict_New(); + if (get_stats.file_dict == NULL) + return NULL; + + if (!tracemalloc_config.enabled) + return get_stats.file_dict; + + TRACE_LOCK(); + get_stats.tracemalloc_file_stats = cfuhash_copy(tracemalloc_file_stats); + TRACE_UNLOCK(); + + if (get_stats.tracemalloc_file_stats == NULL) { + PyErr_NoMemory(); + return NULL; + } + + err = cfuhash_foreach(get_stats.tracemalloc_file_stats, + tracemalloc_get_stats_fill_file, &get_stats); + if (err) + Py_CLEAR(get_stats.file_dict); + + cfuhash_destroy(get_stats.tracemalloc_file_stats); + return get_stats.file_dict; +} + +static PyObject* +tracemalloc_frame_to_pyobject(PyObject *filename, int lineno) +{ + PyObject *frame_obj, *lineno_obj; + + frame_obj = PyStructSequence_New(&FrameType); + if (frame_obj == NULL) + return NULL; + + if (filename == NULL) + filename = Py_None; + Py_INCREF(filename); + PyStructSequence_SET_ITEM(frame_obj, 0, filename); + + lineno_obj = tracemalloc_lineno_as_obj(lineno); + if (lineno_obj == NULL) { + Py_DECREF(frame_obj); + return NULL; + } + PyStructSequence_SET_ITEM(frame_obj, 1, lineno_obj); + + return frame_obj; +} + +static PyObject* +tracemalloc_trace_to_pyobject(tracemalloc_trace_t *trace) +{ + PyObject *trace_obj = NULL; + PyObject *size, *frames, *frame; + int i; + + trace_obj = PyStructSequence_New(&TraceType); + if (trace_obj == NULL) + return NULL; + + size = PyLong_FromSize_t(trace->size); + if (size == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyStructSequence_SET_ITEM(trace_obj, 0, size); + + frames = PyList_New(trace->nframe); + if (frames == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + for (i=0; i < trace->nframe; i++) { + frame = tracemalloc_frame_to_pyobject(TRACE_FILENAMES(trace)[i], + TRACE_LINENOS(trace, trace->nframe)[i]); + if (frame == NULL) { + Py_DECREF(trace_obj); + Py_DECREF(frames); + return NULL; + } + PyList_SET_ITEM(frames, i, frame); + } + PyStructSequence_SET_ITEM(trace_obj, 1, frames); + + return trace_obj; +} + +typedef struct { + cfuhash_table_t *tracemalloc_allocs ; + PyObject *dict; +} tracemalloc_get_traces_t; + +static int +tracemalloc_get_traces_fill(cfuhash_entry *entry, void *user_data) +{ + tracemalloc_get_traces_t *get_traces = user_data; + const void *ptr; + tracemalloc_trace_t *trace; + PyObject *key_obj, *tracemalloc_obj; + int res; + + ptr = entry->key; + trace = (tracemalloc_trace_t *)CFUHASH_ENTRY_DATA(entry); + + key_obj = PyLong_FromVoidPtr((void *)ptr); + if (key_obj == NULL) + return 1; + + tracemalloc_obj = tracemalloc_trace_to_pyobject(trace); + if (tracemalloc_obj == NULL) { + Py_DECREF(key_obj); + return 1; + } + + res = PyDict_SetItem(get_traces->dict, key_obj, tracemalloc_obj); + Py_DECREF(key_obj); + Py_DECREF(tracemalloc_obj); + if (res < 0) + return 1; + return 0; +} + +PyDoc_STRVAR(tracemalloc_get_traces_doc, +"get_stats() -> dict\n" +"\n" +"Get all traces of allocated Python memory blocks.\n" +"Return a dictionary: {pointer: int -> trace: structseq).\n" +"Return an empty dictionary if the tracemalloc module is disabled."); + +static PyObject* +py_tracemalloc_get_traces(PyObject *self, PyObject *obj) +{ + tracemalloc_get_traces_t get_traces; + int err; + + get_traces.dict = PyDict_New(); + if (get_traces.dict == NULL) + return NULL; + + if (!tracemalloc_config.enabled) + return get_traces.dict; + + TRACE_LOCK(); + get_traces.tracemalloc_allocs = cfuhash_copy(tracemalloc_allocs); + TRACE_UNLOCK(); + + if (get_traces.tracemalloc_allocs == NULL) { + Py_DECREF(get_traces.dict); + PyErr_NoMemory(); + return NULL; + } + + err = cfuhash_foreach(get_traces.tracemalloc_allocs, + tracemalloc_get_traces_fill, &get_traces); + if (err) + Py_CLEAR(get_traces.dict); + + cfuhash_destroy(get_traces.tracemalloc_allocs); + return get_traces.dict; +} + +void* +tracemalloc_get_object_address(PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + if (PyType_IS_GC(type)) + return (void *)((char *)obj - sizeof(PyGC_Head)); + else + return (void *)obj; +} + +PyDoc_STRVAR(tracemalloc_get_object_address_doc, +"get_object_address(obj) -> int\n" +"\n" +"Return the address of the memory block of the specified\n" +"Python object."); + +static PyObject* +py_tracemalloc_get_object_address(PyObject *self, PyObject *obj) +{ + void *ptr; + ptr = tracemalloc_get_object_address(obj); + return PyLong_FromVoidPtr(ptr); +} + +PyDoc_STRVAR(tracemalloc_get_object_trace_doc, +"get_object_trace(obj) -> trace\n" +"\n" +"Get the trace of the Python object 'obj' as trace structseq.\n" +"Return None if tracemalloc module did not save the location\n" +"when the object was allocated, for example if tracemalloc was disabled."); + +static PyObject* +py_tracemalloc_get_object_trace(PyObject *self, PyObject *obj) +{ + void *ptr; + size_t trace_size; + char stack_buffer[ TRACE_SIZE(MAX_NFRAME) ]; + tracemalloc_trace_t *trace; + + trace_size = TRACE_SIZE(tracemalloc_config.max_nframe); + assert(trace_size <= sizeof(stack_buffer)); + trace = (tracemalloc_trace_t *)stack_buffer; + + if (!tracemalloc_config.enabled) + Py_RETURN_NONE; + + ptr = tracemalloc_get_object_address(obj); + + TRACE_LOCK(); + if (!cfuhash_get_data(tracemalloc_allocs, ptr, trace, trace_size)) { + TRACE_UNLOCK(); + + Py_RETURN_NONE; + } + TRACE_UNLOCK(); + + return tracemalloc_trace_to_pyobject(trace); +} + +static void +tracemalloc_timer_stop(void) +{ + tracemalloc_timer.enabled = 0; + Py_CLEAR(tracemalloc_timer.callback); + Py_CLEAR(tracemalloc_timer.args); + Py_CLEAR(tracemalloc_timer.kwargs); +} + +static int +tracemalloc_atexit_register(PyObject *module) +{ + PyObject *method = NULL, *atexit = NULL, *func = NULL; + PyObject *result; + int ret = -1; + + method = PyObject_GetAttrString(module, "_atexit"); + if (method == NULL) + goto done; + + atexit = PyImport_ImportModule("atexit"); + if (atexit == NULL) { + if (!PyErr_Warn(PyExc_ImportWarning, + "atexit module is missing: " + "cannot automatically disable tracemalloc at exit")) + { + PyErr_Clear(); + return 0; + } + goto done; + } + + func = PyObject_GetAttrString(atexit, "register"); + if (func == NULL) + goto done; + + result = PyObject_CallFunction(func, "O", method); + if (result == NULL) + goto done; + Py_DECREF(result); + + ret = 0; + +done: + Py_XDECREF(method); + Py_XDECREF(func); + Py_XDECREF(atexit); + return ret; +} + +PyDoc_STRVAR(tracemalloc_start_timer_doc, + "start_timer(delay: int,\n" + " callback: callable, args: tuple=None, kwargs: dict=None)\n" + "\n" + "Start a timer: call the 'callback' every 'delay' seconds\n" + "when the memory allocator is used."); + +static PyObject* +py_tracemalloc_start_timer(PyObject *self, PyObject *args) +{ + int delay; + PyObject *callback; + PyObject *cb_args = NULL; + PyObject *kwargs = NULL; + + if (!PyArg_ParseTuple(args, "iO|OO:start_timer", + &delay, &callback, &cb_args, &kwargs)) + return NULL; + + if (tracemalloc_enable() < 0) { + PyErr_NoMemory(); + return NULL; + } + + if (delay < 1) { + PyErr_SetString(PyExc_ValueError, "delay must be greater than 0"); + return NULL; + } + + if (!PyCallable_Check(callback)) { + PyErr_Format(PyExc_TypeError, + "callback must be a callable object, not %s", + Py_TYPE(callback)->tp_name); + return NULL; + } + + if (cb_args != NULL && !PyTuple_Check(cb_args)) { + PyErr_SetString(PyExc_TypeError, + "argument list must be a tuple"); + return NULL; + } + + if (kwargs != NULL && !PyDict_Check(kwargs)) { + PyErr_SetString(PyExc_TypeError, + "keyword list must be a dictionary"); + return NULL; + } + + /* Disable temporary the timer because Py_CLEAR may call it */ + tracemalloc_timer_stop(); + + Py_INCREF(callback); + tracemalloc_timer.callback = callback; + Py_XINCREF(cb_args); + tracemalloc_timer.args = cb_args; + Py_XINCREF(kwargs); + tracemalloc_timer.kwargs = kwargs; + + tracemalloc_timer.delay = delay; + tracemalloc_timer.next_trigger = time(NULL) + delay; + tracemalloc_timer.enabled = 1; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_stop_timer_doc, + "stop_timer()\n" + "\n" + "Stop the timer started by start_timer()."); + +PyObject* +py_tracemalloc_stop_timer(PyObject *self) +{ + if (tracemalloc_config.enabled) + tracemalloc_timer_stop(); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_enable_doc, + "enable()\n" + "\n" + "Start tracing Python memory allocations."); + +static PyObject* +py_tracemalloc_enable(PyObject *self) +{ + if (tracemalloc_enable() < 0) { + PyErr_NoMemory(); + return NULL; + } + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_disable_doc, + "disable()\n" + "\n" + "Stop tracing Python memory allocations."); + +static PyObject* +py_tracemalloc_disable(PyObject *self) +{ + tracemalloc_disable(); + Py_RETURN_NONE; +} + +static PyObject* +py_tracemalloc_atexit(PyObject *self) +{ + tracemalloc_deinit(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_traceback_limit_doc, + "get_traceback_limit() -> int\n" + "\n" + "Get the maximum number of frames stored in a trace of a memory\n" + "allocation."); + +static PyObject* +py_tracemalloc_get_traceback_limit(PyObject *self) +{ + return PyLong_FromLong(tracemalloc_config.max_nframe); +} + +PyDoc_STRVAR(tracemalloc_set_traceback_limit_doc, + "set_traceback_limit(nframe: int)\n" + "\n" + "Set the maximum number of frames stored in the traceback attribute\n" + "of a trace of a memory allocation.\n" + "\n" + "If the tracemalloc is enabled, all traces and statistics of memory\n" + "allocations are cleared."); + +static PyObject* +py_tracemalloc_set_traceback_limit(PyObject *self, PyObject *args) +{ + int nframe; + cfuhash_table_t *new_trace_allocs; + size_t trace_size; + + if (!PyArg_ParseTuple(args, "i:set_traceback_limit", + &nframe)) + return NULL; + + if (nframe < 1 || nframe > MAX_NFRAME) { + PyErr_Format(PyExc_ValueError, + "the number of frames must be in range [1; %i]", + MAX_NFRAME); + return NULL; + } + + trace_size = TRACE_SIZE(nframe); + new_trace_allocs = cfuhash_new(trace_size, + cfuhash_hash_ptr, cfuhash_cmp_direct); + if (new_trace_allocs == NULL) { + PyErr_NoMemory(); + return NULL; + } + + TRACE_LOCK(); + if (tracemalloc_config.enabled) + tracemalloc_clear_traces(); + + tracemalloc_config.max_nframe = nframe; + cfuhash_destroy(tracemalloc_allocs); + tracemalloc_allocs = new_trace_allocs; + TRACE_UNLOCK(); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_tracemalloc_size_doc, + "get_tracemalloc_size() -> int\n" + "\n" + "Get the memory usage in bytes of the _tracemalloc module."); + +static PyObject* +py_tracemalloc_get_tracemalloc_size(PyObject *self) +{ + size_t usage; + + usage = cfuhash_get_size(tracemalloc_allocs); + usage += cfuhash_get_size(tracemalloc_filenames); + usage += cfuhash_get_size(tracemalloc_file_stats); + return PyLong_FromSize_t(usage); +} + +PyDoc_STRVAR(tracemalloc_get_traced_memory_doc, + "get_traced_memory() -> int\n" + "\n" + "Get the total size in bytes of all memory blocks allocated\n" + "by Python currently."); + +static PyObject* +tracemalloc_get_traced_memory(PyObject *self) +{ + return PyLong_FromSize_t(tracemalloc_python_memory); +} + +static int +tracemalloc_add_filter(tracemalloc_filter_t *filter) +{ + size_t nfilter; + tracemalloc_filters_t *filters; + tracemalloc_filter_t *new_filters; + + if (filter->include) + filters = &tracemalloc_include_filters; + else + filters = &tracemalloc_exclude_filters; + +#ifdef WITH_THREAD + PyThread_acquire_lock(tracemalloc_filters_lock, 1); +#endif + nfilter = (filters->nfilter + 1); + new_filters = realloc(filters->filters, + nfilter * sizeof(tracemalloc_filter_t)); + if (new_filters == NULL) { +#ifdef WITH_THREAD + PyThread_release_lock(tracemalloc_filters_lock); +#endif + PyErr_NoMemory(); + return -1; + } + new_filters[filters->nfilter] = *filter; + + filters->nfilter = nfilter; + filters->filters = new_filters; +#ifdef WITH_THREAD + PyThread_release_lock(tracemalloc_filters_lock); +#endif + return 0; +} + +PyDoc_STRVAR(tracemalloc_add_filter_doc, + "add_filter(include: bool, filename: str, lineno: int=None, traceback: bool=True)\n" + "\n" + "Add a filter. If include is True, only trace memory blocks allocated\n" + "in a file with a name matching filename at line number lineno. If\n" + "include is True, don't trace memory blocks allocated in a file with a\n" + "name matching filename at line number lineno.\n" + "\n" + "The filename can contain one or many '*' joker characters which\n" + "matchs any substring, including an empty string. The '.pyc' and '.pyo'\n" + "suffixes are automatically replaced with '.py'. On Windows, the\n" + "comparison is case insensitive and the alternative separator '/' is\n" + "replaced with the standard separator '\'.\n" + "\n" + "If lineno is None or lesser than 1, it matches any line number."); + +static PyObject* +py_tracemalloc_add_filter(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter; + + if (!PyArg_ParseTuple(args, "O!:add_filter", + &FilterType, (PyObject **)&pyfilter)) + return NULL; + + if (tracemalloc_add_filter(&pyfilter->filter) < 0) + return NULL; + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_add_include_filter_doc, + "add_include_filter(filename: str, lineno: int=None, traceback: bool=False)"); + +static PyObject* +py_tracemalloc_add_include_filter(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"filename", "lineno", "traceback", 0}; + PyObject *filename; + int lineno = -1; + int traceback = 0; + tracemalloc_filter_t filter; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "U|O&i:add_include_filter", kwlist, + &filename, + &tracemalloc_parse_lineno, &lineno, + &traceback)) + return NULL; + + if (tracemalloc_filter_init(&filter, 1, filename, lineno, traceback) < 0) + return NULL; + + if (tracemalloc_add_filter(&filter) < 0) { + tracemalloc_filter_deinit(&filter); + return NULL; + } + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_add_exclude_filter_doc, + "add_exclude_filter(filename: str, lineno: int=None, traceback: bool=False)"); + +static PyObject* +py_tracemalloc_add_exclude_filter(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"filename", "lineno", "traceback", 0}; + PyObject *filename; + int lineno = -1; + int traceback = 0; + tracemalloc_filter_t filter; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "U|O&i:add_exclude_filter", kwlist, + &filename, + &tracemalloc_parse_lineno, &lineno, + &traceback)) + return NULL; + + if (tracemalloc_filter_init(&filter, 0, filename, lineno, traceback) < 0) + return NULL; + + if (tracemalloc_add_filter(&filter) < 0) { + tracemalloc_filter_deinit(&filter); + return NULL; + } + Py_RETURN_NONE; +} + +static PyObject* +tracemalloc_filter_as_obj(tracemalloc_filter_t *filter) +{ + FilterObject *pyfilter; + + pyfilter = PyObject_New(FilterObject, &FilterType); + if (pyfilter == NULL) + return NULL; + + Py_INCREF(filter->pattern); + pyfilter->filter = *filter; + return (PyObject *)pyfilter; +} + +PyDoc_STRVAR(tracemalloc_get_filters_doc, + "get_filters()\n" + "\n" + "Get the filters as list of (include: bool, filename: str, lineno: int)" + "tuples.\n" + "\n" + "If *lineno* is ``None``, a filter matchs any line number."); + +static size_t +tracemalloc_get_filters(PyObject *list, size_t first_index, + tracemalloc_filters_t *filters) +{ + size_t i; + tracemalloc_filter_t *filter; + PyObject *pyfilter; + + for (i=0; infilter; i++) { + filter = &filters->filters[i]; + + pyfilter = tracemalloc_filter_as_obj(filter); + if (pyfilter == NULL) + return -1; + + PyList_SET_ITEM(list, first_index + i, pyfilter); + } + return filters->nfilter; +} + +static PyObject* +py_tracemalloc_get_filters(PyObject *self) +{ + PyObject *filters = NULL; + size_t number; + + TRACE_LOCK(); + tracemalloc_config.reentrant = 1; + TRACE_UNLOCK(); + +#ifdef WITH_THREAD + PyThread_acquire_lock(tracemalloc_filters_lock, 1); +#endif + + filters = PyList_New(tracemalloc_include_filters.nfilter + + tracemalloc_exclude_filters.nfilter); + if (filters == NULL) + goto exit; + + number = tracemalloc_get_filters(filters, 0, &tracemalloc_include_filters); + if (number == (size_t)-1) { + Py_CLEAR(filters); + goto exit; + } + + number = tracemalloc_get_filters(filters, number, &tracemalloc_exclude_filters); + if (number == (size_t)-1) { + Py_CLEAR(filters); + goto exit; + } + +exit: +#ifdef WITH_THREAD + PyThread_release_lock(tracemalloc_filters_lock); +#endif + + TRACE_LOCK(); + tracemalloc_config.reentrant = 0; + TRACE_UNLOCK(); + + return filters; +} + +PyDoc_STRVAR(tracemalloc_clear_filters_doc, + "clear_filters()\n" + "\n" + "Reset the filter list."); + +static PyObject* +py_tracemalloc_clear_filters(PyObject *self) +{ + tracemalloc_clear_filters(); + Py_RETURN_NONE; +} + +static PyMethodDef module_methods[] = { + {"is_enabled", (PyCFunction)py_tracemalloc_is_enabled, METH_NOARGS, + PyDoc_STR("is_enabled()->bool")}, + {"clear_traces", (PyCFunction)py_tracemalloc_clear_traces, METH_NOARGS, + tracemalloc_clear_traces_doc}, + {"get_stats", (PyCFunction)tracemalloc_get_stats, METH_NOARGS, + tracemalloc_get_stats_doc}, + {"get_traces", (PyCFunction)py_tracemalloc_get_traces, METH_NOARGS, + tracemalloc_get_traces_doc}, + {"get_object_address", (PyCFunction)py_tracemalloc_get_object_address, METH_O, + tracemalloc_get_object_address_doc}, + {"get_object_trace", (PyCFunction)py_tracemalloc_get_object_trace, METH_O, + tracemalloc_get_object_trace_doc}, + {"start_timer", py_tracemalloc_start_timer, METH_VARARGS, + tracemalloc_start_timer_doc}, + {"stop_timer", (PyCFunction)py_tracemalloc_stop_timer, METH_NOARGS, + tracemalloc_stop_timer_doc}, + {"enable", (PyCFunction)py_tracemalloc_enable, METH_NOARGS, + tracemalloc_enable_doc}, + {"disable", (PyCFunction)py_tracemalloc_disable, METH_NOARGS, + tracemalloc_disable_doc}, + {"get_traceback_limit", (PyCFunction)py_tracemalloc_get_traceback_limit, + METH_NOARGS, tracemalloc_get_traceback_limit_doc}, + {"set_traceback_limit", (PyCFunction)py_tracemalloc_set_traceback_limit, + METH_VARARGS, tracemalloc_set_traceback_limit_doc}, + {"get_tracemalloc_size", (PyCFunction)py_tracemalloc_get_tracemalloc_size, + METH_NOARGS, tracemalloc_get_tracemalloc_size_doc}, + {"get_traced_memory", (PyCFunction)tracemalloc_get_traced_memory, + METH_NOARGS, tracemalloc_get_traced_memory_doc}, + {"add_filter", (PyCFunction)py_tracemalloc_add_filter, + METH_VARARGS, tracemalloc_add_filter_doc}, + {"add_include_filter", (PyCFunction)py_tracemalloc_add_include_filter, + METH_VARARGS | METH_KEYWORDS, tracemalloc_add_include_filter_doc}, + {"add_exclude_filter", (PyCFunction)py_tracemalloc_add_exclude_filter, + METH_VARARGS | METH_KEYWORDS, tracemalloc_add_exclude_filter_doc}, + {"get_filters", (PyCFunction)py_tracemalloc_get_filters, METH_NOARGS, + tracemalloc_get_filters_doc}, + {"clear_filters", (PyCFunction)py_tracemalloc_clear_filters, METH_NOARGS, + tracemalloc_clear_filters_doc}, + + /* private functions */ + {"_atexit", (PyCFunction)py_tracemalloc_atexit, METH_NOARGS}, + + /* sentinel */ + {NULL, NULL} +}; + +PyDoc_STRVAR(module_doc, +"_tracemalloc module."); + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "_tracemalloc", + module_doc, + 0, /* non-negative size to be able to unload the module */ + module_methods, + NULL, +}; + +PyMODINIT_FUNC +PyInit__tracemalloc(void) +{ + PyObject *m; + m = PyModule_Create(&module_def); + if (m == NULL) + return NULL; + + if (tracemalloc_init() < 0) + return NULL; + + if (PyType_Ready(&FilterType) < 0) + return NULL; + + Py_INCREF((PyObject*) &TraceType); + PyModule_AddObject(m, "Trace", (PyObject*)&TraceType); + + Py_INCREF((PyObject*) &FilterType); + PyModule_AddObject(m, "Filter", (PyObject*)&FilterType); + + Py_INCREF((PyObject*) &FrameType); + PyModule_AddObject(m, "Frame", (PyObject*)&FrameType); + + Py_INCREF((PyObject*) &LineStatsType); + PyModule_AddObject(m, "TraceStats", (PyObject*)&LineStatsType); + + if (tracemalloc_atexit_register(m) < 0) + return NULL; + return m; +} + +int +PyTraceMalloc_Enable(void) +{ + return tracemalloc_enable(); +} + +int +PyTraceMalloc_Init(void) +{ + char *p; + PyObject *xoptions, *key; + int has_key; + + if ((p = Py_GETENV("PYTHONTRACEMALLOC")) && *p != '\0') { + /* enable */ + } + else { + xoptions = PySys_GetXOptions(); + if (xoptions == NULL) + return -1; + + key = PyUnicode_FromString("tracemalloc"); + if (key == NULL) + return -1; + + has_key = PyDict_Contains(xoptions, key); + Py_DECREF(key); + if (has_key < 0) + return -1; + if (!has_key) + return 0; + } + + return PyTraceMalloc_Enable(); +} + diff -r df2fdd42b375 -r 21f7c3df0f15 Python/pythonrun.c --- a/Python/pythonrun.c Mon Aug 26 22:28:21 2013 +0200 +++ b/Python/pythonrun.c Tue Sep 17 00:59:50 2013 +0200 @@ -15,6 +15,7 @@ #include "ast.h" #include "marshal.h" #include "osdefs.h" +#include "tracemalloc.h" #ifdef HAVE_SIGNAL_H #include @@ -342,6 +343,9 @@ void if (_PyStructSequence_Init() < 0) Py_FatalError("Py_Initialize: can't initialize structseq"); + if (PyTraceMalloc_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize tracemalloc"); + bimod = _PyBuiltin_Init(); if (bimod == NULL) Py_FatalError("Py_Initialize: can't initialize builtins modules");