diff -r 50e4d0c62c2b -r ec121a72e848 Doc/c-api/veryhigh.rst --- a/Doc/c-api/veryhigh.rst Sun Oct 06 13:24:52 2013 +0200 +++ b/Doc/c-api/veryhigh.rst Sun Oct 06 17:53:23 2013 +0200 @@ -166,6 +166,14 @@ the same library that the Python runtime resulting string. For example, The :mod:`readline` module sets this hook to provide line-editing and tab-completion features. + The result must be a string allocated by :c:func:`PyMem_RawMalloc` or + :c:func:`PyMem_RawRealloc`, or *NULL* if an error occurred. + + .. versionchanged:: 3.4 + The result must be allocated by :c:func:`PyMem_RawMalloc` or + :c:func:`PyMem_RawRealloc`, instead of being allocated by + :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. + .. c:function:: struct _node* PyParser_SimpleParseString(const char *str, int start) diff -r 50e4d0c62c2b -r ec121a72e848 Doc/library/debug.rst --- a/Doc/library/debug.rst Sun Oct 06 13:24:52 2013 +0200 +++ b/Doc/library/debug.rst Sun Oct 06 17:53:23 2013 +0200 @@ -15,3 +15,4 @@ allowing you to identify bottlenecks in profile.rst timeit.rst trace.rst + tracemalloc.rst diff -r 50e4d0c62c2b -r ec121a72e848 Doc/library/tracemalloc.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Doc/library/tracemalloc.rst Sun Oct 06 17:53:23 2013 +0200 @@ -0,0 +1,1215 @@ +:mod:`tracemalloc` --- Trace memory allocations +=============================================== + +.. module:: tracemalloc + :synopsis: Trace memory allocations. + +The tracemalloc module is a debug tool to trace memory blocks allocated by +Python. It provides the following information: + +* Compute the differences between two snapshots to detect memory leaks +* Statistics on allocated memory blocks per filename and per line number: + total size, number and average size of allocated memory blocks +* Traceback where a memory block was allocated + +To trace most memory blocks allocated by Python, the module should be enabled +as early as possible by setting the :envvar:`PYTHONTRACEMALLOC` environment +variable to ``1``, or by using :option:`-X` ``tracemalloc`` command line +option. The :func:`tracemalloc.enable` function can also be called to start +tracing Python memory allocations. + +By default, a trace of an allocated memory block only stores one frame. Use the +:func:`set_traceback_limit` function to store more frames. + +Python memory blocks allocated in the :mod:`tracemalloc` module are also traced +by default. Use ``add_exclude_filter(tracemalloc.__file__)`` to ignore these +these memory allocations. + +At fork, the module is automatically disabled in the child process. + +.. versionadded:: 3.4 + + +Example of top outputs +====================== + +Cumulative top 5 of the biggest allocations grouped by filename, compact +output:: + + 2013-10-03 11:34:39: Cumulative top 5 allocations per filename + #1: .../Lib/test/regrtest.py: 554 MiB + #2: .../Lib/unittest/suite.py: 499 MiB + #3: : 401 MiB + #4: .../test/support/__init__.py: 349 MiB + #5: .../tracemalloc/Lib/runpy.py: 255 MiB + 1330 more: 822 MiB + +Top 5 of the biggest allocations grouped by address, compact output:: + + 2013-10-03 11:34:39: Top 5 allocations per address + #1: memory block 0x805e7010: size=80 MiB + #2: memory block 0x9b531010: size=12 MiB + #3: memory block 0x1a9b2838: size=1536 KiB + #4: memory block 0x19dbfd88: size=253 KiB + #5: memory block 0xa9fdcf0: size=252 KiB + 645844 more: size=56 MiB, average=92 B + Traced Python memory: size=151 MiB, average=245 B + +Top 10 of the biggest allocations grouped by line number, full output:: + + 2013-10-03 11:34:39: Top 10 allocations per filename and line number + #1: .../tracemalloc/Lib/lzma.py:120: size=93 MiB, count=13, average=7 MiB + #2: :704: size=24 MiB, count=357474, average=73 B + #3: .../Lib/unittest/case.py:496: size=2997 KiB, count=7942, average=386 B + #4: .../tracemalloc/Lib/linecache.py:127: size=2054 KiB, count=26474, average=79 B + #5: .../Lib/test/test_datetime.py:32: size=1248 KiB, count=27, average=46 KiB + #6: :274: size=989 KiB, count=12989, average=77 B + #7: .../Lib/test/test_zipfile.py:1319: size=858 KiB, count=5, average=171 KiB + #8: .../Lib/test/test_enumerate.py:150: size=852 KiB, count=29607, average=29 B + #9: .../Lib/unittest/case.py:306: size=309 KiB, count=2504, average=126 B + #10: .../Lib/test/test_zipfile.py:1508: size=307 KiB, count=12, average=25 KiB + 51150 more: size=24 MiB, count=208802, average=120 B + Traced Python memory: size=151 MiB, count=645849, average=245 B + + gc.objects: 2688709 + process_memory.rss: 828 MiB + process_memory.vms: 887 MiB + pymalloc.allocated: 277 MiB + pymalloc.blocks: 5125645 + pymalloc.fragmentation: 4.2% + pymalloc.free: 12 MiB + pymalloc.max_size: 563 MiB + pymalloc.size: 294 MiB + tracemalloc.arena_size: 294 MiB + tracemalloc.module.fragmentation: 19.2% + tracemalloc.module.free: 14 MiB + tracemalloc.module.size: 77 MiB + tracemalloc.traced.max_size: 182 MiB + tracemalloc.traced.size: 151 MiB + tracemalloc.traces: 645849 + unicode_interned.len: 48487 + unicode_interned.size: 1536 KiB + + +Usage +===== + +Display top 25 +-------------- + +Example displaying once the top 50 lines allocating the most memory:: + + import tracemalloc + tracemalloc.enable() + # ... run your application ... + tracemalloc.DisplayTop().display(50) + +By default, allocations are grouped by filename and line numbers and the top is +written into :data:`sys.stdout`. + +See the :class:`DisplayTop` class for more options. + + +Display top with differences +---------------------------- + +To watch the evolution of memory allocations, the top allocations can be +displayed regulary using a task. Example displaying the top 50 files when the +traced memory is increased or decreased by more than 5 MB, or every minute, +with a compact output (no count, no average, no metric):: + + import tracemalloc + task = tracemalloc.DisplayTopTask(25, group_by='filename') + task.display_top.count = False + task.display_top.average = False + task.display_top.metrics = False + task.set_memory_threshold(5 * 1024 * 1024) + task.set_delay(60) + tracemalloc.enable() + task.schedule() + # ... run your application ... + +See the :class:`DisplayTopTask` class for more options. + + +Take a snapshot +--------------- + +The :class:`DisplayTopTask` class creates temporary snapshots which are lost +after the top is displayed. When you don't know what you are looking for, you +can take a snapshot of the allocated memory blocks to analyze it while the +application is running, or analyze it later. + +Example taking a snapshot with traces and writing it into a file:: + + import tracemalloc + tracemalloc.enable() + # ... run your application ... + snapshot = tracemalloc.Snapshot.create(traces=True) + snapshot.write('snapshot.pickle') + +Use the following command to display the snapshot file:: + + python -m tracemalloc snapshot.pickle + +See `Command line options`_ for more options. See also +:meth:`Snapshot.apply_filters` and :meth:`DisplayTop.display_snapshot` +methods. + + +Compare snapshots +----------------- + +It is not always easy to find a memory leak using a single snapshot. It is +easier to take multiple snapshots and compare them to see the differences. + +Example taking a snapshot with traces when the traced memory is increased or +decreased by more than 5 MB, or every minute:: + + import tracemalloc + task = tracemalloc.TakeSnapshotTask(traces=True) + task.set_memory_threshold(5 * 1024 * 1024) + task.set_delay(60) + tracemalloc.enable() + task.schedule() + # ... run your application ... + +By default, snapshot files are written in the current directory with the name +``tracemalloc-XXXX.pickle`` where ``XXXX`` is a simple counter. + +Use the following command to compare snapshot files:: + + python -m tracemalloc tracemalloc-0001.pickle tracemalloc-0002.pickle ... + +See `Command line options`_, and :class:`TakeSnapshotTask` and :class:`StatsDiff` +classes for more options. + + +API +=== + +Main Functions +-------------- + +.. function:: cancel_tasks() + + Cancel scheduled tasks. + + See also the :func:`get_tasks` function. + + +.. function:: clear_traces() + + Clear traces and statistics on Python memory allocations, and reset the + :func:`get_arena_size` and :func:`get_traced_memory` counters. + + +.. function:: disable() + + Stop tracing Python memory allocations and cancel scheduled tasks. + + See also :func:`cancel_tasks`, :func:`enable` and :func:`is_enabled` + functions. + + +.. function:: enable() + + Start tracing Python memory allocations. + + At fork, the module is automatically disabled in the child process. + + See also :func:`disable` and :func:`is_enabled` functions. + + +.. function:: get_stats() + + Get statistics on traced Python memory blocks as a dictionary ``{filename + (str): {line_number (int): stats}}`` where *stats* in a + ``(size: int, count: int)`` tuple, *filename* and *line_number* can + be ``None``. + + Return an empty dictionary if the :mod:`tracemalloc` module is disabled. + + See also the :func:`get_traces` function. + + +.. function:: get_tasks() + + Get the list of scheduled tasks, list of :class:`Task` instances. + + +.. function:: is_enabled() + + ``True`` if the :mod:`tracemalloc` module is tracing Python memory + allocations, ``False`` otherwise. + + See also :func:`enable` and :func:`disable` functions. + + +Trace Functions +--------------- + +.. function:: get_traceback_limit() + + Get the maximum number of frames stored in the traceback of a trace of a + memory block. + + Use the :func:`set_traceback_limit` function to change the limit. + + +.. function:: get_object_address(obj) + + Get the address of the main memory block of the specified Python object. + + A Python object can be composed by multiple memory blocks, the function only + returns the address of the main memory block. + + See also :func:`get_object_trace` and :func:`gc.get_referrers` functions. + + +.. function:: get_object_trace(obj) + + Get the trace of a Python object *obj* as a ``(size: int, traceback)`` tuple + where *traceback* is a tuple of ``(filename: str, lineno: int)`` tuples, + *filename* and *lineno* can be ``None``. + + The function only returns the trace of the main memory block of the object. + The *size* of the trace is smaller than the total size of the object if the + object is composed by more than one memory block. + + Return ``None`` if the :mod:`tracemalloc` module did not trace the + allocation of the object. + + See also :func:`get_object_address`, :func:`get_trace`, :func:`get_traces`, + :func:`gc.get_referrers` and :func:`sys.getsizeof` functions. + + +.. function:: get_trace(address) + + Get the trace of a memory block as a ``(size: int, traceback)`` tuple where + *traceback* is a tuple of ``(filename: str, lineno: int)`` tuples, + *filename* and *lineno* can be ``None``. + + Return ``None`` if the :mod:`tracemalloc` module did not trace the + allocation of the memory block. + + See also :func:`get_object_trace`, :func:`get_stats` and :func:`get_traces` + functions. + + +.. function:: get_traces() + + Get traces of Python memory allocations as a dictionary ``{address + (int): trace}`` where *trace* is a + ``(size: int, traceback)`` and *traceback* is a list of + ``(filename: str, lineno: int)``. + *traceback* can be empty, *filename* and *lineno* can be None. + + Return an empty dictionary if the :mod:`tracemalloc` module is disabled. + + See also :func:`get_object_trace`, :func:`get_stats` and :func:`get_trace` + functions. + + +.. function:: set_traceback_limit(nframe: int) + + Set the maximum number of frames stored in the traceback of a trace of a + memory block. + + Storing the traceback of each memory allocation has an important overhead on + the memory usage. Use the :func:`get_tracemalloc_memory` function to measure + the overhead and the :func:`add_filter` function to select which memory + allocations are traced. + + Use the :func:`get_traceback_limit` function to get the current limit. + + +Filter Functions +---------------- + +.. function:: add_filter(filter) + + Add a new filter on Python memory allocations, *filter* is a :class:`Filter` + instance. + + All inclusive filters are applied at once, a memory allocation is only + ignored if no inclusive filter match its trace. A memory allocation is + ignored if at least one exclusive filter matchs its trace. + + The new filter is not applied on already collected traces. Use the + :func:`clear_traces` function to ensure that all traces match the new + filter. + +.. function:: add_include_filter(filename: str, lineno: int=None, traceback: bool=False) + + Add an inclusive filter: helper for the :meth:`add_filter` method creating a + :class:`Filter` instance with the :attr:`~Filter.include` attribute set to + ``True``. + + Example: ``tracemalloc.add_include_filter(tracemalloc.__file__)`` only + includes memory blocks allocated by the :mod:`tracemalloc` module. + + +.. function:: add_exclude_filter(filename: str, lineno: int=None, traceback: bool=False) + + Add an exclusive filter: helper for the :meth:`add_filter` method creating a + :class:`Filter` instance with the :attr:`~Filter.include` attribute set to + ``False``. + + Example: ``tracemalloc.add_exclude_filter(tracemalloc.__file__)`` ignores + memory blocks allocated by the :mod:`tracemalloc` module. + + +.. function:: clear_filters() + + Reset the filter list. + + See also the :func:`get_filters` function. + + +.. function:: get_filters() + + Get the filters on Python memory allocations as list of :class:`Filter` + instances. + + See also the :func:`clear_filters` function. + + +Metric Functions +---------------- + +The following functions can be used to add metrics to a snapshot, see +the :meth:`Snapshot.add_metric` method. + +.. function:: get_allocated_blocks() + + Get the current number of allocated memory blocks. + + +.. function:: get_arena_size() + + Get the size in bytes of traced arenas. + + See also the :func:`get_pymalloc_stats` function. + + +.. function:: get_process_memory() + + Get the memory usage of the current process as a ``(rss: int, vms: int)`` + tuple, *rss* is the "Resident Set Size" in bytes and *vms* is the size of + the virtual memory in bytes + + Return ``None`` if the platform is not supported. + + +.. function:: get_pymalloc_stats() + + Get statistics on the ``pymalloc`` allocator as a dictionary. + + +---------------------+-------------------------------------------------------+ + | Key | Description | + +=====================+=======================================================+ + | ``alignment`` | Alignment of addresses returned to the user. | + +---------------------+-------------------------------------------------------+ + | ``threshold`` | Small block threshold in bytes: pymalloc uses | + | | PyMem_RawMalloc() for allocation greater than | + | | threshold. | + +---------------------+-------------------------------------------------------+ + | ``nalloc`` | Number of times object malloc called | + +---------------------+-------------------------------------------------------+ + | ``arena_size`` | Arena size in bytes | + +---------------------+-------------------------------------------------------+ + | ``total_arenas`` | Number of calls to new_arena(): total number of | + | | allocated arenas, including released arenas | + +---------------------+-------------------------------------------------------+ + | ``max_arenas`` | Maximum number of arenas | + +---------------------+-------------------------------------------------------+ + | ``arenas`` | Number of arenas currently allocated | + +---------------------+-------------------------------------------------------+ + | ``allocated_bytes`` | Number of bytes in allocated blocks | + +---------------------+-------------------------------------------------------+ + | ``available_bytes`` | Number of bytes in available blocks in used pools | + +---------------------+-------------------------------------------------------+ + | ``pool_size`` | Pool size in bytes | + +---------------------+-------------------------------------------------------+ + | ``free_pools`` | Number of unused pools | + +---------------------+-------------------------------------------------------+ + | ``pool_headers`` | Number of bytes wasted in pool headers | + +---------------------+-------------------------------------------------------+ + | ``quantization`` | Number of bytes in used and full pools wasted due to | + | | quantization, i.e. the necessarily leftover space at | + | | the ends of used and full pools. | + +---------------------+-------------------------------------------------------+ + | ``arena_alignment`` | Number of bytes for arena alignment padding | + +---------------------+-------------------------------------------------------+ + + The function is not available if Python is compiled without ``pymalloc``. + + See also :func:`get_arena_size` and :func:`sys._debugmallocstats` functions. + + +.. function:: get_traced_memory() + + Get the current size and maximum size of memory blocks traced by the + :mod:`tracemalloc` module as a tuple: ``(size: int, max_size: int)``. + + +.. function:: get_tracemalloc_memory() + + Get the memory usage in bytes of the :mod:`tracemalloc` module as a + tuple: ``(size: int, free: int)``. + + * *size*: total size of bytes allocated by the module, + including *free* bytes + * *free*: number of free bytes available to store data + + +.. function:: get_unicode_interned() + + Get the size in bytes and the length of the dictionary of Unicode interned + strings as a ``(size: int, length: int)`` tuple. + + The size is the size of the dictionary, excluding the size of strings. + + +DisplayTop +---------- + +.. class:: DisplayTop() + + Display the top of allocated memory blocks. + + .. method:: display(count=10, group_by="line", cumulative=False, file=None, callback=None) + + Take a snapshot and display the top *count* biggest allocated memory + blocks grouped by *group_by*. + + *callback* is an optional callable object which can be used to add + metrics to a snapshot. It is called with only one parameter: the newly + created snapshot instance. Use the :meth:`Snapshot.add_metric` method to + add new metric. + + Return the snapshot, a :class:`Snapshot` instance. + + .. method:: display_snapshot(snapshot, count=10, group_by="line", cumulative=False, file=None) + + Display a snapshot of memory blocks allocated by Python, *snapshot* is a + :class:`Snapshot` instance. + + .. method:: display_top_diff(top_diff, count=10, file=None) + + Display differences between two :class:`GroupedStats` instances, + *top_diff* is a :class:`StatsDiff` instance. + + .. method:: display_top_stats(top_stats, count=10, file=None) + + Display the top of allocated memory blocks grouped by the + :attr:`~GroupedStats.group_by` attribute of *top_stats*, *top_stats* is a + :class:`GroupedStats` instance. + + .. attribute:: average + + If ``True`` (default value), display the average size of memory blocks. + + .. attribute:: color + + If ``True``, always use colors. If ``False``, never use colors. The + default value is ``None``: use colors if the *file* parameter is a TTY + device. + + .. attribute:: compare_to_previous + + If ``True`` (default value), compare to the previous snapshot. If + ``False``, compare to the first snapshot. + + .. attribute:: filename_parts + + Number of displayed filename parts (int, default: ``3``). Extra parts + are replaced with ``'...'``. + + .. attribute:: metrics + + If ``True`` (default value), display metrics: see + :attr:`Snapshot.metrics`. + + .. attribute:: previous_top_stats + + Previous :class:`GroupedStats` instance, or first :class:`GroupedStats` + instance if :attr:`compare_to_previous` is ``False``, used to display the + differences between two snapshots. + + .. attribute:: size + + If ``True`` (default value), display the size of memory blocks. + + +DisplayTopTask +-------------- + +.. class:: DisplayTopTask(count=10, group_by="line", cumulative=False, file=sys.stdout, callback=None) + + Task taking temporary snapshots and displaying the top *count* memory + allocations grouped by *group_by*. + + :class:`DisplayTopTask` is based on the :class:`Task` class and so inherit + all attributes and methods, especially: + + * :meth:`~Task.cancel` + * :meth:`~Task.schedule` + * :meth:`~Task.set_delay` + * :meth:`~Task.set_memory_threshold` + + Modify the :attr:`display_top` attribute to customize the display. + + .. method:: display() + + Take a snapshot and display the top :attr:`count` biggest allocated + memory blocks grouped by :attr:`group_by` using the :attr:`display_top` + attribute. + + Return the snapshot, a :class:`Snapshot` instance. + + .. attribute:: callback + + *callback* is an optional callable object which can be used to add + metrics to a snapshot. It is called with only one parameter: the newly + created snapshot instance. Use the :meth:`Snapshot.add_metric` method to + add new metric. + + .. attribute:: count + + Maximum number of displayed memory blocks. + + .. attribute:: cumulative + + If ``True``, cumulate size and count of memory blocks of all frames of + each trace, not only the most recent frame. The default value is + ``False``. + + The option is ignored if the traceback limit is less than ``2``, see + the :func:`get_traceback_limit` function. + + .. attribute:: display_top + + Instance of :class:`DisplayTop`. + + .. attribute:: file + + The top is written into *file*. + + .. attribute:: group_by + + Determine how memory allocations are grouped: see :attr:`Snapshot.top_by` + for the available values. + + +Filter +------ + +.. class:: Filter(include: bool, pattern: str, lineno: int=None, traceback: bool=False) + + Filter to select which memory allocations are traced. Filters can be used to + reduce the memory usage of the :mod:`tracemalloc` module, which can be read + using the :func:`get_tracemalloc_memory` function. + + .. method:: match(filename: str, lineno: int) + + Return ``True`` if the filter matchs the filename and line number, + ``False`` otherwise. + + .. method:: match_filename(filename: str) + + Return ``True`` if the filter matchs the filename, ``False`` otherwise. + + .. method:: match_lineno(lineno: int) + + Return ``True`` if the filter matchs the line number, ``False`` + otherwise. + + .. method:: match_traceback(traceback) + + Return ``True`` if the filter matchs the *traceback*, ``False`` + otherwise. + + *traceback* is a tuple of ``(filename: str, lineno: int)`` tuples. + + .. attribute:: include + + If *include* is ``True``, only trace memory blocks allocated in a file + with a name matching filename :attr:`pattern` at line number + :attr:`lineno`. + + If *include* is ``False``, ignore memory blocks allocated in a file with + a name matching filename :attr`pattern` at line number :attr:`lineno`. + + .. attribute:: lineno + + Line number (``int``). If is is ``None`` or less than ``1``, it matches + any line number. + + .. attribute:: pattern + + The filename *pattern* can contain one or many ``*`` joker characters + which match any substring, including an empty string. The ``.pyc`` and + ``.pyo`` file extensions are replaced with ``.py``. On Windows, the + comparison is case insensitive and the alternative separator ``/`` is + replaced with the standard separator ``\``. + + .. attribute:: traceback + + If *traceback* is ``True``, all frames of the traceback are checked. If + *traceback* is ``False``, only the most recent frame is checked. + + This attribute is ignored if the traceback limit is less than ``2``. + See the :func:`get_traceback_limit` function. + + +GroupedStats +------------ + +.. class:: GroupedStats(timestamp: datetime.datetime, stats: dict, group_by: str, cumulative=False, metrics: dict=None) + + Top of allocated memory blocks grouped by *group_by* as a dictionary. + + The :meth:`Snapshot.top_by` method creates a :class:`GroupedStats` instance. + + .. method:: compare_to(old_stats: GroupedStats=None) + + Compare to an older :class:`GroupedStats` instance. + Return a :class:`StatsDiff` instance. + + The :attr:`StatsDiff.differences` list is not sorted: call + the :meth:`StatsDiff.sort` method to sort the list. + + ``None`` values are replaced with an empty string for filenames or zero + for line numbers, because :class:`str` and :class:`int` cannot be + compared to ``None``. + + .. attribute:: cumulative + + If ``True``, cumulate size and count of memory blocks of all frames of + the traceback of a trace, not only the most recent frame. + + .. attribute:: metrics + + Dictionary storing metrics read when the snapshot was created: + ``{name (str): metric}`` where *metric* type is :class:`Metric`. + + .. attribute:: group_by + + Determine how memory allocations were grouped: see + :attr:`Snapshot.top_by` for the available values. + + .. attribute:: stats + + Dictionary ``{key: stats}`` where the *key* type depends on the + :attr:`group_by` attribute and *stats* is a ``(size: int, count: int)`` + tuple. + + See the :meth:`Snapshot.top_by` method. + + .. attribute:: timestamp + + Creation date and time of the snapshot, :class:`datetime.datetime` + instance. + + +Metric +------ + +.. class:: Metric(name: str, value: int, format: str) + + Value of a metric when a snapshot is created. + + .. attribute:: name + + Name of the metric. + + .. attribute:: value + + Value of the metric. + + .. attribute:: format + + Format of the metric: + + * ``'int'``: a number + * ``'percent'``: percentage, ``1.0`` means ``100%`` + * ``'size'``: a size in bytes + + +Snapshot +-------- + +.. class:: Snapshot(timestamp: datetime.datetime, pid: int, traces: dict=None, stats: dict=None, metrics: dict=None) + + Snapshot of traces and statistics on memory blocks allocated by Python. + + Use :class:`TakeSnapshotTask` to take regulary snapshots. + + .. method:: add_gc_metrics() + + Add a metric on the garbage collector: + + * ``gc.objects``: total number of Python objects + + See the :mod:`gc` module. + + + .. method:: add_metric(name: str, value: int, format: str) + + Helper to add a :class:`Metric` instance to :attr:`Snapshot.metrics`. + Return the newly created :class:`Metric` instance. + + Raise an exception if the name is already present in + :attr:`Snapshot.metrics`. + + + .. method:: add_process_memory_metrics() + + Add metrics on the process memory: + + * ``process_memory.rss``: Resident Set Size + * ``process_memory.vms``: Virtual Memory Size + + These metrics are only available if the :func:`get_process_memory` + function is available on the platform. + + + .. method:: add_pymalloc_metrics() + + Add metrics on the Python memory allocator (``pymalloc``): + + * ``pymalloc.blocks``: number of allocated memory blocks + * ``pymalloc.size``: size of ``pymalloc`` arenas + * ``pymalloc.max_size``: maximum size of ``pymalloc`` arenas + * ``pymalloc.allocated``: number of allocated bytes + * ``pymalloc.free``: number of free bytes + * ``pymalloc.fragmentation``: fragmentation percentage of the arenas + + These metrics are only available if Python is compiled in debug mode, + except ``pymalloc.blocks`` which is always available. + + + .. method:: add_tracemalloc_metrics() + + Add metrics on the :mod:`tracemalloc` module: + + * ``tracemalloc.traced.size``: size of memory blocks traced by the + :mod:`tracemalloc` module + * ``tracemalloc.traced.max_size``: maximum size of memory blocks traced + by the :mod:`tracemalloc` module + * ``tracemalloc.traces``: number of traces of Python memory blocks + * ``tracemalloc.module.size``: total size of bytes allocated by the + :mod:`tracemalloc` module, including free bytes + * ``tracemalloc.module.free``: number of free bytes available for + the :mod:`tracemalloc` module + * ``tracemalloc.module.fragmentation``: percentage of fragmentation of + the memory allocated by the :mod:`tracemalloc` module + * ``tracemalloc.arena_size``: size of traced arenas + + ``tracemalloc.traces`` metric is only present if the snapshot was created + with traces. + + + .. method:: add_unicode_metrics() + + Add metrics on the Unicode interned strings: + + * ``unicode_interned.size``: size of the dictionary, excluding size + of strings + * ``unicode_interned.len``: length of the dictionary + + + .. method:: apply_filters(filters) + + Apply filters on the :attr:`traces` and :attr:`stats` dictionaries, + *filters* is a list of :class:`Filter` instances. + + + .. classmethod:: create(traces=False, metrics=True) + + Take a snapshot of traces and/or statistics of allocated memory blocks. + + If *traces* is ``True``, :func:`get_traces` is called and its result + is stored in the :attr:`Snapshot.traces` attribute. This attribute + contains more information than :attr:`Snapshot.stats` and uses more + memory and more disk space. If *traces* is ``False``, + :attr:`Snapshot.traces` is set to ``None``. + + If *metrics* is ``True``, fill :attr:`Snapshot.metrics` with metrics + using the following methods: + + * :meth:`add_gc_metrics` + * :meth:`add_process_memory_metrics` + * :meth:`add_pymalloc_metrics` + * :meth:`add_tracemalloc_metrics` + * :meth:`add_unicode_metrics` + + If *metrics* is ``False``, :attr:`Snapshot.metrics` is set to an empty + dictionary. + + Tracebacks of traces are limited to :attr:`traceback_limit` frames. Call + :func:`set_traceback_limit` before calling :meth:`~Snapshot.create` to + store more frames. + + The :mod:`tracemalloc` module must be enabled to take a snapshot. See the + the :func:`enable` function. + + .. method:: get_metric(name, default=None) + + Get the value of the metric called *name*. Return *default* if the metric + does not exist. + + + .. classmethod:: load(filename, traces=True) + + Load a snapshot from a file. + + If *traces* is ``False``, don't load traces. + + + .. method:: top_by(group_by: str, cumulative: bool=False) + + Compute top statistics grouped by *group_by* as a :class:`GroupedStats` + instance: + + ===================== ======================== ============== + group_by description key type + ===================== ======================== ============== + ``'filename'`` filename ``str`` + ``'line'`` filename and line number ``(str, int)`` + ``'address'`` memory block address ``int`` + ===================== ======================== ============== + + If *cumulative* is ``True``, cumulate size and count of memory blocks of + all frames of the traceback of a trace, not only the most recent frame. + The *cumulative* parameter is ignored if *group_by* is ``'address'`` or + if the traceback limit is less than ``2``. + + + .. method:: write(filename) + + Write the snapshot into a file. + + + .. attribute:: metrics + + Dictionary storing metrics read when the snapshot was created: + ``{name (str): metric}`` where *metric* type is :class:`Metric`. + + .. attribute:: pid + + Identifier of the process which created the snapshot, result of + :func:`os.getpid`. + + .. attribute:: stats + + Statistics on traced Python memory, result of the :func:`get_stats` + function. + + .. attribute:: traceback_limit + + Maximum number of frames stored in a trace of a memory block allocated by + Python. + + .. attribute:: traces + + Traces of Python memory allocations, result of the :func:`get_traces` + function, can be ``None``. + + .. attribute:: timestamp + + Creation date and time of the snapshot, :class:`datetime.datetime` + instance. + + +StatsDiff +--------- + +.. class:: StatsDiff(differences, old_stats, new_stats) + + Differences between two :class:`GroupedStats` instances. + + The :meth:`GroupedStats.compare_to` method creates a :class:`StatsDiff` + instance. + + .. method:: sort() + + Sort the :attr:`differences` list from the biggest difference to the + smallest difference. Sort by ``abs(size_diff)``, *size*, + ``abs(count_diff)``, *count* and then by *key*. + + .. attribute:: differences + + Differences between :attr:`old_stats` and :attr:`new_stats` as a list of + ``(size_diff, size, count_diff, count, key)`` tuples. *size_diff*, + *size*, *count_diff* and *count* are ``int``. The key type depends on the + :attr:`~GroupedStats.group_by` attribute of :attr:`new_stats`: see the + :meth:`Snapshot.top_by` method. + + .. attribute:: old_stats + + Old :class:`GroupedStats` instance, can be ``None``. + + .. attribute:: new_stats + + New :class:`GroupedStats` instance. + + +Task +---- + +.. class:: Task(func, \*args, \*\*kw) + + Task calling ``func(*args, **kw)``. When scheduled, the task is called when + the traced memory is increased or decreased by more than *threshold* bytes, + or after *delay* seconds. + + .. method:: call() + + Call ``func(*args, **kw)`` and return the result. + + + .. method:: cancel() + + Cancel the task. + + Do nothing if the task is not scheduled. + + + .. method:: get_delay() + + Get the delay in seconds. If the delay is ``None``, the timer is + disabled. + + + .. method:: get_memory_threshold() + + Get the threshold of the traced memory. When scheduled, the task is + called when the traced memory is increased or decreased by more than + *threshold* bytes. The memory threshold is disabled if *threshold* is + ``None``. + + See also the :meth:`set_memory_threshold` method and the + :func:`get_traced_memory` function. + + + .. method:: schedule(repeat: int=None) + + Schedule the task *repeat* times. If *repeat* is ``None``, the task is + rescheduled after each call until it is cancelled. + + If the method is called twice, the task is rescheduled with the new + *repeat* parameter. + + The task must have a memory threshold or a delay: see :meth:`set_delay` + and :meth:`set_memory_threshold` methods. The :mod:`tracemalloc` must be + enabled to schedule a task: see the :func:`enable` function. + + The task is cancelled if the :meth:`call` method raises an exception. + The task can be cancelled using the :meth:`cancel` method or the + :func:`cancel_tasks` function. + + + .. method:: set_delay(seconds: int) + + Set the delay in seconds before the task will be called. Set the delay to + ``None`` to disable the timer. + + The timer is based on the Python memory allocator, it is not real time. + The task is called after at least *delay* seconds, it is not called + exactly after *delay* seconds if no Python memory allocation occurred. + The timer has a resolution of 1 second. + + The task is rescheduled if it was scheduled. + + + .. method:: set_memory_threshold(size: int) + + Set the threshold of the traced memory. When scheduled, the task is + called when the traced memory is increased or decreased by more than + *threshold* bytes. Set the threshold to ``None`` to disable it. + + The task is rescheduled if it was scheduled. + + See also the :meth:`get_memory_threshold` method and the + :func:`get_traced_memory` function. + + + .. attribute:: func + + Function, callable object. + + .. attribute:: func_args + + Function arguments, :class:`tuple`. + + .. attribute:: func_kwargs + + Function keyword arguments, :class:`dict`. It can be ``None``. + + +TakeSnapshotTask +---------------- + +.. class:: TakeSnapshotTask(filename_template: str="tracemalloc-$counter.pickle", traces: bool=False, metrics: bool=True, callback: callable=None) + + Task taking snapshots of Python memory allocations and writing them into + files. + + :class:`TakeSnapshotTask` is based on the :class:`Task` class and so inherit + all attributes and methods, especially: + + * :meth:`~Task.cancel` + * :meth:`~Task.schedule` + * :meth:`~Task.set_delay` + * :meth:`~Task.set_memory_threshold` + + .. method:: take_snapshot() + + Take a snapshot and write it into a file. + Return ``(snapshot, filename)`` where *snapshot* is a :class:`Snapshot` + instance and filename type is :class:`str`. + + .. attribute:: callback + + *callback* is an optional callable object which can be used to add + metrics to a snapshot. It is called with only one parameter: the newly + created snapshot instance. Use the :meth:`Snapshot.add_metric` method to + add new metric. + + .. attribute:: filename_template + + Template to create a filename. The template supports the following + variables: + + * ``$pid``: identifier of the current process + * ``$timestamp``: current date and time + * ``$counter``: counter starting at 1 and incremented at each snapshot, + formatted as 4 decimal digits + + The default template is ``'tracemalloc-$counter.pickle'``. + + .. attribute:: metrics + + Parameter passed to the :meth:`Snapshot.create` function. + + .. attribute:: traces + + Parameter passed to the :meth:`Snapshot.create` function. + + +Command line options +==================== + +The ``python -m tracemalloc`` command can be used to display, analyze and +compare snapshot files. + +The command has the following options. + +``-a``, ``--address`` option: + + Group memory allocations by address, instead of grouping by line number. + +``-f``, ``--file`` option: + + Group memory allocations per filename, instead of grouping by line number. + +``-n NUMBER``, ``--number NUMBER`` option: + + Number of traces displayed per top (default: 10): set the *count* parameter + of the :meth:`DisplayTop.display_snapshot` method. + +``--first`` option: + + Compare with the first snapshot, instead of comparing with the previous + snapshot: set the :attr:`DisplayTop.compare_to_previous` attribute to + ``False``. + +``-c``, ``--cumulative`` option: + + Cumulate size and count of allocated memory blocks using all frames, not + only the most recent frame: set *cumulative* parameter of the + :meth:`DisplayTop.display_snapshot` method to ``True``. + + The option has only an effect if the snapshot + contains traces and if the traceback limit was greater than ``1``. + +``-b ADDRESS``, ``--block=ADDRESS`` option: + + Get the memory block at address *ADDRESS*, display its size and the + traceback where it was allocated. + + The option can only be used on snapshots created with traces. + +``-t``, ``--traceback`` option: + + Group memory allocations by address, display the size and the traceback + of the *NUMBER* biggest allocated memory blocks. + + The option can only be used on snapshots created with traces. By default, + the traceback limit is ``1`` frame: set a greater limit with the + :func:`set_traceback_limit` function before taking snapshots to get more + frames. + + See the ``--number`` option for *NUMBER*. + +``-i FILENAME[:LINENO]``, ``--include FILENAME[:LINENO]`` option: + + Only include traces of files with a name matching *FILENAME* pattern at + line number *LINENO*. Only check the most recent frame. The option can be + specified multiple times. + + See the :func:`add_include_filter` function for the syntax of a filter. + +``-I FILENAME[:LINENO]``, ``--include-traceback FILENAME[:LINENO]`` option: + + Similar to ``--include`` option, but check all frames of the traceback. + +``-x FILENAME[:LINENO]``, ``--exclude FILENAME[:LINENO]`` option: + + Exclude traces of files with a name matching *FILENAME* pattern at line + number *LINENO*. Only check the most recent frame. The option can be + specified multiple times. + + See the :func:`add_exclude_filter` method for the syntax of a filter. + +``-X FILENAME[:LINENO]``, ``--exclude-traceback FILENAME[:LINENO]`` option: + + Similar to ``--exclude`` option, but check all frames of the traceback. + +``-S``, ``--hide-size`` option: + + Hide the size of allocations: set :attr:`DisplayTop.size` attribute to + ``False``. + +``-C``, ``--hide-count`` option: + + Hide the number of allocations: set :attr:`DisplayTop.count` attribute + to ``False``. + +``-A``, ``--hide-average`` option: + + Hide the average size of allocations: set :attr:`DisplayTop.average` + attribute to ``False``. + +``-M``, ``--hide-metrics`` option: + + Hide metrics, see :attr:`DisplayTop.metrics`. + +``-P PARTS``, ``--filename-parts=PARTS`` option: + + Number of displayed filename parts (default: 3): set + :attr:`DisplayTop.filename_parts` attribute. + +``--color`` option: + + Always use colors, even if :data:`sys.stdout` is not a TTY device: set the + :attr:`DisplayTop.color` attribute to ``True``. + +``--no-color`` option: + + Never use colors, even if :data:`sys.stdout` is a TTY device: set the + :attr:`DisplayTop.color` attribute to ``False``. + diff -r 50e4d0c62c2b -r ec121a72e848 Doc/license.rst --- a/Doc/license.rst Sun Oct 06 13:24:52 2013 +0200 +++ b/Doc/license.rst Sun Oct 06 17:53:23 2013 +0200 @@ -864,3 +864,44 @@ used for the build:: Jean-loup Gailly Mark Adler jloup@gzip.org madler@alumni.caltech.edu + +cfuhash +------- + +The implementtation of the hash table used by the :mod:`tracemalloc` is based +on the cfuhash project:: + + Copyright (c) 2005 Don Owens + All rights reserved. + + This code is released under the BSD license: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + diff -r 50e4d0c62c2b -r ec121a72e848 Doc/using/cmdline.rst --- a/Doc/using/cmdline.rst Sun Oct 06 13:24:52 2013 +0200 +++ b/Doc/using/cmdline.rst Sun Oct 06 17:53:23 2013 +0200 @@ -381,6 +381,7 @@ Miscellaneous options * ``-X faulthandler`` to enable :mod:`faulthandler`; * ``-X showrefcount`` to enable the output of the total reference count and memory blocks (only works on debug builds); + * ``-X tracemalloc`` to enable :mod:`tracemalloc`. It also allows to pass arbitrary values and retrieve them through the :data:`sys._xoptions` dictionary. @@ -392,7 +393,7 @@ Miscellaneous options The ``-X faulthandler`` option. .. versionadded:: 3.4 - The ``-X showrefcount`` option. + The ``-X showrefcount`` and ``-X tracemalloc`` options. Options you shouldn't use @@ -594,6 +595,14 @@ conflict. .. versionadded:: 3.3 +.. envvar:: PYTHONTRACEMALLOC + + If this environment variable is set to a non-empty string, all memory + allocations made by Python are traced by the :mod:`tracemalloc` module. + + .. versionadded:: 3.4 + + Debug-mode variables ~~~~~~~~~~~~~~~~~~~~ diff -r 50e4d0c62c2b -r ec121a72e848 Include/pymem.h --- a/Include/pymem.h Sun Oct 06 13:24:52 2013 +0200 +++ b/Include/pymem.h Sun Oct 06 17:53:23 2013 +0200 @@ -172,6 +172,60 @@ PyAPI_FUNC(void) PyMem_SetAllocator(PyMe PyAPI_FUNC(void) PyMem_SetupDebugHooks(void); #endif +#if defined(WITH_PYMALLOC) && !defined(Py_LIMITED_API) +typedef struct { + /* Alignment of addresses returned to the user */ + size_t alignment; + + /* Small block threshold in bytes: pymalloc uses PyMem_RawMalloc() for + allocation greater than threshold */ + size_t threshold; + + /* Number of times object malloc called */ + size_t nalloc; + + /* Arena size in bytes */ + size_t arena_size; + + /* Number of calls to new_arena(): total number of allocated arenas, + including released arenas */ + size_t total_arenas; + + /* Maximum number of arenas */ + size_t max_arenas; + + /* Number of arenas currently allocated */ + size_t arenas; + + /* Number of bytes in allocated blocks */ + size_t allocated_bytes; + + /* Number of bytes in available blocks in used pools */ + size_t available_bytes; + + /* Pool size in bytes */ + size_t pool_size; + + /* Number of unused pools */ + /* # of free pools + pools not yet carved out of current arena */ + unsigned int free_pools; + + /* Number of bytes wasted in pool headers */ + size_t pool_headers; + + /* Number of bytes in used and full pools wasted due to quantization, + * i.e. the necessarily leftover space at the ends of used and + * full pools. + */ + size_t quantization; + + /* Number of bytes for arena alignment padding */ + size_t arena_alignment; +} pymalloc_stats_t; + +void _PyObject_GetMallocStats(pymalloc_stats_t *stats); +#endif + #ifdef __cplusplus } #endif diff -r 50e4d0c62c2b -r ec121a72e848 Lib/test/regrtest.py --- a/Lib/test/regrtest.py Sun Oct 06 13:24:52 2013 +0200 +++ b/Lib/test/regrtest.py Sun Oct 06 17:53:23 2013 +0200 @@ -479,6 +479,51 @@ def main(tests=None, **kwargs): directly to set the values that would normally be set by flags on the command line. """ + import tracemalloc + if tracemalloc.is_enabled(): + import gc + import time + + fast = False + traces = True + + tracemalloc.add_exclude_filter(tracemalloc.__file__) + if traces: + tracemalloc.set_traceback_limit(25) + #tracemalloc.clear_traces() + + top = tracemalloc.DisplayTop() + take = tracemalloc.TakeSnapshotTask(traces=traces) + + stream = sys.__stderr__ + def taskfunc(take, top, stream, monotonic, count): + try: + start = monotonic() + snapshot, filename = take.take_snapshot() + dt = monotonic() - start + stream.write("%s: Write a snapshot of memory allocations into %s (%.1f sec)\n" + % (snapshot.timestamp, filename, dt)) + stream.flush() + + top.display_snapshot(snapshot, count, file=stream) + except Exception as err: + stream.write("Failed to take a snapshot: %s\n" % err) + stream.flush() + + take.filename_template = "/tmp/tracemalloc-$pid-$counter.pickle" + if fast: + delay = 10 + top_count = 5 + threshold = 5 * 1024 * 1024 + else: + delay = 30 + top_count = 5 + threshold = 10 * 1024 * 1024 + task = tracemalloc.Task(taskfunc, take, top, stream, time.monotonic, top_count) + task.set_memory_threshold(threshold) + task.set_delay(delay) + task.schedule() + # Display the Python traceback on fatal errors (e.g. segfault) faulthandler.enable(all_threads=True) diff -r 50e4d0c62c2b -r ec121a72e848 Lib/test/support/__init__.py --- a/Lib/test/support/__init__.py Sun Oct 06 13:24:52 2013 +0200 +++ b/Lib/test/support/__init__.py Sun Oct 06 17:53:23 2013 +0200 @@ -2030,10 +2030,16 @@ if sys.platform.startswith('win'): finally: k32.SetErrorMode(old_error_mode) else: + import resource # this is a no-op for other platforms @contextlib.contextmanager def suppress_crash_popup(): - yield + limit = resource.getrlimit(resource.RLIMIT_CORE) + try: + resource.setrlimit(resource.RLIMIT_CORE, (0, limit[1])) + yield + finally: + resource.setrlimit(resource.RLIMIT_CORE, limit) def patch(test_instance, object_to_patch, attr_name, new_value): @@ -2069,6 +2075,22 @@ def patch(test_instance, object_to_patch # actually override the attribute setattr(object_to_patch, attr_name, new_value) +def run_in_subinterp(code): + """ + Run code in a subinterpreter. Raise unittest.SkipTest if the tracemalloc + module is enabled. + """ + # Issue #10915, #15751: PyGILState_*() functions don't work with + # sub-interpreters, the tracemalloc module uses these functions internally + try: + import tracemalloc + except ImportError: + pass + else: + if tracemalloc.is_enabled(): + raise unittest.SkipTest("run_in_subinterp() cannot be used " + "if tracemalloc module is enabled") + return _testcapi.run_in_subinterp(code) class SuppressCoreFiles: diff -r 50e4d0c62c2b -r ec121a72e848 Lib/test/test_atexit.py --- a/Lib/test/test_atexit.py Sun Oct 06 13:24:52 2013 +0200 +++ b/Lib/test/test_atexit.py Sun Oct 06 17:53:23 2013 +0200 @@ -139,7 +139,7 @@ class SubinterpreterTest(unittest.TestCa atexit.register(f) del atexit """ - ret = _testcapi.run_in_subinterp(code) + ret = support.run_in_subinterp(code) self.assertEqual(ret, 0) self.assertEqual(atexit._ncallbacks(), n) @@ -154,7 +154,7 @@ class SubinterpreterTest(unittest.TestCa atexit.register(f) atexit.__atexit = atexit """ - ret = _testcapi.run_in_subinterp(code) + ret = support.run_in_subinterp(code) self.assertEqual(ret, 0) self.assertEqual(atexit._ncallbacks(), n) diff -r 50e4d0c62c2b -r ec121a72e848 Lib/test/test_capi.py --- a/Lib/test/test_capi.py Sun Oct 06 13:24:52 2013 +0200 +++ b/Lib/test/test_capi.py Sun Oct 06 17:53:23 2013 +0200 @@ -206,7 +206,7 @@ class SubinterpreterTest(unittest.TestCa pickle.dump(id(builtins), f) """.format(w) with open(r, "rb") as f: - ret = _testcapi.run_in_subinterp(code) + ret = support.run_in_subinterp(code) self.assertEqual(ret, 0) self.assertNotEqual(pickle.load(f), id(sys.modules)) self.assertNotEqual(pickle.load(f), id(builtins)) diff -r 50e4d0c62c2b -r ec121a72e848 Lib/test/test_threading.py --- a/Lib/test/test_threading.py Sun Oct 06 13:24:52 2013 +0200 +++ b/Lib/test/test_threading.py Sun Oct 06 17:53:23 2013 +0200 @@ -785,7 +785,7 @@ class SubinterpThreadingTests(BaseTestCa os.write(%d, b"x") threading.Thread(target=f).start() """ % (w,) - ret = _testcapi.run_in_subinterp(code) + ret = test.support.run_in_subinterp(code) self.assertEqual(ret, 0) # The thread was joined properly. self.assertEqual(os.read(r, 1), b"x") diff -r 50e4d0c62c2b -r ec121a72e848 Lib/test/test_tracemalloc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/test/test_tracemalloc.py Sun Oct 06 17:53:23 2013 +0200 @@ -0,0 +1,1295 @@ +import contextlib +import datetime +import io +import os +import sys +import time +import tracemalloc +import unittest +from unittest.mock import patch +from test.script_helper import assert_python_ok +from test import support + +EMPTY_STRING_SIZE = sys.getsizeof(b'') + +def noop(*args, **kw): + pass + +def get_frames(nframe, lineno_delta): + frames = [] + frame = sys._getframe(1) + for index in range(nframe): + code = frame.f_code + lineno = frame.f_lineno + lineno_delta + frames.append((code.co_filename, lineno)) + lineno_delta = 0 + frame = frame.f_back + if frame is None: + break + return tuple(frames) + +def allocate_bytes(size): + nframe = tracemalloc.get_traceback_limit() + bytes_len = (size - EMPTY_STRING_SIZE) + frames = get_frames(nframe, 1) + data = b'x' * bytes_len + return data, frames + +def create_snapshots(): + pid = 123 + traceback_limit = 2 + + traceback_a_2 = [('a.py', 2), + ('b.py', 4)] + traceback_a_5 = [('a.py', 5), + ('b.py', 4)] + traceback_b_1 = [('b.py', 1)] + traceback_c_578 = [('c.py', 30)] + traceback_none_none = [(None, None)] + + timestamp = datetime.datetime(2013, 9, 12, 15, 16, 17) + stats = { + 'a.py': {2: (30, 3), + 5: (2, 1)}, + 'b.py': {1: (66, 1)}, + None: {None: (7, 1)}, + } + traces = { + 0x10001: (10, traceback_a_2), + 0x10002: (10, traceback_a_2), + 0x10003: (10, traceback_a_2), + + 0x20001: (2, traceback_a_5), + + 0x30001: (66, traceback_b_1), + + 0x40001: (7, traceback_none_none), + } + snapshot = tracemalloc.Snapshot(timestamp, pid, traceback_limit, + stats, traces) + snapshot.add_metric('process_memory.rss', 1024, 'size') + snapshot.add_metric('tracemalloc.size', 100, 'size') + snapshot.add_metric('my_data', 8, 'int') + + timestamp2 = datetime.datetime(2013, 9, 12, 15, 16, 50) + stats2 = { + 'a.py': {2: (30, 3), + 5: (5002, 2)}, + 'c.py': {578: (400, 1)}, + } + traces2 = { + 0x10001: (10, traceback_a_2), + 0x10002: (10, traceback_a_2), + 0x10003: (10, traceback_a_2), + + 0x20001: (2, traceback_a_5), + 0x20002: (5000, traceback_a_5), + + 0x30001: (400, traceback_c_578), + } + snapshot2 = tracemalloc.Snapshot(timestamp2, pid, traceback_limit, + stats2, traces2) + snapshot2.add_metric('process_memory.rss', 1500, 'size') + snapshot2.add_metric('tracemalloc.size', 200, 'size') + snapshot2.add_metric('my_data', 10, 'int') + + return (snapshot, snapshot2) + + +class TestTracemallocEnabled(unittest.TestCase): + def setUp(self): + if tracemalloc.is_enabled(): + self.skipTest("tracemalloc must be disabled before the test") + + tracemalloc.clear_filters() + tracemalloc.add_exclude_filter(tracemalloc.__file__) + tracemalloc.set_traceback_limit(1) + tracemalloc.enable() + + def tearDown(self): + tracemalloc.disable() + tracemalloc.clear_filters() + + def test_get_tracemalloc_memory(self): + data = [allocate_bytes(123) for count in range(1000)] + size, free = tracemalloc.get_tracemalloc_memory() + self.assertGreaterEqual(size, 0) + self.assertGreaterEqual(free, 0) + self.assertGreater(size, free) + + tracemalloc.clear_traces() + size2, free2 = tracemalloc.get_tracemalloc_memory() + self.assertLessEqual(size2, size) + + def test_get_trace(self): + tracemalloc.clear_traces() + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) + address = tracemalloc.get_object_address(obj) + trace = tracemalloc.get_trace(address) + self.assertIsInstance(trace, tuple) + size, traceback = trace + self.assertEqual(size, obj_size) + self.assertEqual(traceback, obj_frames) + + def test_get_object_trace(self): + tracemalloc.clear_traces() + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) + trace = tracemalloc.get_object_trace(obj) + self.assertIsInstance(trace, tuple) + size, traceback = trace + self.assertEqual(size, obj_size) + self.assertEqual(traceback, obj_frames) + + def test_set_traceback_limit(self): + obj_size = 10 + + nframe = tracemalloc.get_traceback_limit() + self.addCleanup(tracemalloc.set_traceback_limit, nframe) + + self.assertRaises(ValueError, tracemalloc.set_traceback_limit, -1) + + tracemalloc.clear_traces() + tracemalloc.set_traceback_limit(0) + obj, obj_frames = allocate_bytes(obj_size) + trace = tracemalloc.get_object_trace(obj) + size, traceback = trace + self.assertEqual(len(traceback), 0) + self.assertEqual(traceback, obj_frames) + + tracemalloc.clear_traces() + tracemalloc.set_traceback_limit(1) + obj, obj_frames = allocate_bytes(obj_size) + trace = tracemalloc.get_object_trace(obj) + size, traceback = trace + self.assertEqual(len(traceback), 1) + self.assertEqual(traceback, obj_frames) + + tracemalloc.clear_traces() + tracemalloc.set_traceback_limit(10) + obj2, obj2_frames = allocate_bytes(obj_size) + trace = tracemalloc.get_object_trace(obj2) + size, traceback = trace + self.assertEqual(len(traceback), 10) + self.assertEqual(traceback, obj2_frames) + + def test_get_traces(self): + tracemalloc.clear_traces() + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) + + traces = tracemalloc.get_traces() + address = tracemalloc.get_object_address(obj) + self.assertIn(address, traces) + trace = traces[address] + + self.assertIsInstance(trace, tuple) + size, traceback = trace + self.assertEqual(size, obj_size) + self.assertEqual(traceback, obj_frames) + + def test_get_traces_intern_traceback(self): + # dummy wrappers to get more useful and identical frames in the traceback + def allocate_bytes2(size): + return allocate_bytes(size) + def allocate_bytes3(size): + return allocate_bytes2(size) + def allocate_bytes4(size): + return allocate_bytes3(size) + + # Ensure that two identical tracebacks are not duplicated + tracemalloc.clear_traces() + tracemalloc.set_traceback_limit(4) + obj_size = 123 + obj1, obj1_frames = allocate_bytes4(obj_size) + obj2, obj2_frames = allocate_bytes4(obj_size) + + traces = tracemalloc.get_traces() + + address1 = tracemalloc.get_object_address(obj1) + address2 = tracemalloc.get_object_address(obj2) + trace1 = traces[address1] + trace2 = traces[address2] + size1, traceback1 = trace1 + size2, traceback2 = trace2 + self.assertEqual(traceback2, traceback1) + self.assertIs(traceback2, traceback1) + + def test_get_process_memory(self): + tracemalloc.clear_traces() + obj_size = 5 * 1024 * 1024 + + orig = tracemalloc.get_process_memory() + if orig is None: + self.skipTest("get_process_memory is not supported") + self.assertGreater(orig[0], 0) + self.assertGreater(orig[1], 0) + + obj, obj_frames = allocate_bytes(obj_size) + curr = tracemalloc.get_process_memory() + self.assertGreaterEqual(curr[0], orig[0]) + self.assertGreaterEqual(curr[1], orig[1]) + + def test_get_traced_memory(self): + # get the allocation location to filter allocations + size = 12345 + obj, frames = allocate_bytes(size) + filename, lineno = frames[0] + tracemalloc.add_include_filter(filename, lineno) + + # allocate one object + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + self.assertEqual(tracemalloc.get_traced_memory(), (size, size)) + + # destroy the object + obj = None + self.assertEqual(tracemalloc.get_traced_memory(), (0, size)) + + # clear_traces() must reset traced memory counters + tracemalloc.clear_traces() + self.assertEqual(tracemalloc.get_traced_memory(), (0, 0)) + + # allocate another object + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + self.assertEqual(tracemalloc.get_traced_memory(), (size, size)) + + # disable() rests also traced memory counters + tracemalloc.disable() + self.assertEqual(tracemalloc.get_traced_memory(), (0, 0)) + + def test_get_arena_size(self): + size = tracemalloc.get_arena_size() + self.assertGreaterEqual(size, 0) + + def test_get_stats(self): + tracemalloc.clear_traces() + total_size = 0 + total_count = 0 + objs = [] + for index in range(5): + size = 1234 + obj, obj_frames = allocate_bytes(size) + objs.append(obj) + total_size += size + total_count += 1 + + stats = tracemalloc.get_stats() + for filename, line_stats in stats.items(): + for lineno, line_stat in line_stats.items(): + # stats can be huge, one test per file should be enough + self.assertIsInstance(line_stat, tuple) + size, count = line_stat + self.assertIsInstance(size, int) + self.assertIsInstance(count, int) + self.assertGreaterEqual(size, 0) + self.assertGreaterEqual(count, 1) + break + + filename, lineno = obj_frames[0] + self.assertIn(filename, stats) + line_stats = stats[filename] + self.assertIn(lineno, line_stats) + size, count = line_stats[lineno] + self.assertEqual(size, total_size) + self.assertEqual(count, total_count) + + def test_clear_traces(self): + tracemalloc.clear_traces() + obj_size = 1234 + obj, obj_frames = allocate_bytes(obj_size) + + stats = tracemalloc.get_stats() + filename, lineno = obj_frames[0] + line_stats = stats[filename][lineno] + size, count = line_stats + self.assertEqual(size, obj_size) + self.assertEqual(count, 1) + + tracemalloc.clear_traces() + stats2 = tracemalloc.get_stats() + self.assertNotIn(lineno, stats2[filename]) + + def test_task_is_scheduled(self): + task = tracemalloc.Task(noop) + task.set_delay(60) + + task.schedule() + self.assertTrue(task.is_scheduled()) + scheduled = tracemalloc.get_tasks() + self.assertIn(task, scheduled) + + task.cancel() + self.assertFalse(task.is_scheduled()) + scheduled = tracemalloc.get_tasks() + self.assertNotIn(task, scheduled) + + # second cancel() should not fail + task.cancel() + + # reschedule + task.schedule() + self.assertTrue(task.is_scheduled()) + + # schedule twice should not fail + task.schedule() + + # cannot schedule if the tracemalloc module is disabled + tracemalloc.disable() + with self.assertRaises(RuntimeError) as cm: + task.schedule() + self.assertEqual(str(cm.exception), + "the tracemalloc module must be enabled " + "to schedule a task") + + def test_cancel_tasks(self): + task = tracemalloc.Task(noop) + task.set_delay(60) + + task.schedule() + self.assertTrue(task.is_scheduled()) + + tracemalloc.cancel_tasks() + self.assertFalse(task.is_scheduled()) + + def test_task_delay(self): + calls = [] + def log_func(*args, **kw): + calls.append(log_func) + + args = (1, 2, 3) + kwargs = {'arg': 4} + task = tracemalloc.Task(log_func) + task.set_delay(1) + task.schedule() + time.sleep(1) + obj, source = allocate_bytes(123) + self.assertEqual(len(calls), 1) + self.assertIs(calls[0], log_func) + + def test_task_memory_threshold(self): + diff = None + def log_func(*args, **kw): + nonlocal diff + size, max_size = tracemalloc.get_traced_memory() + diff = (size - old_size) + + obj_size = 1024 * 1024 + threshold = int(obj_size * 0.75) + args = (1, 2, 3) + kwargs = {'arg': 4} + + old_size, max_size = tracemalloc.get_traced_memory() + task = tracemalloc.Task(log_func, args, kwargs) + task.set_memory_threshold(threshold) + task.schedule() + + # allocate + obj, source = allocate_bytes(obj_size) + self.assertIsNotNone(diff) + self.assertGreaterEqual(diff, threshold) + + # release + diff = None + old_size, max_size = tracemalloc.get_traced_memory() + obj = None + size, max_size = tracemalloc.get_traced_memory() + self.assertIsNotNone(diff) + self.assertLessEqual(diff, threshold) + + def test_task_repeat(self): + calls = [] + def log_func(): + calls.append(log_func) + + task = tracemalloc.Task(log_func) + task.set_memory_threshold(1) + + # allocate at least 100 memory blocks, but the task should only be + # called 3 times + task.schedule(3) + objects = [object() for n in range(100)] + self.assertEqual(len(calls), 3) + + def test_disable_scheduled_tasks(self): + task = tracemalloc.Task(noop) + task.set_delay(60) + task.schedule() + self.assertTrue(task.is_scheduled()) + tracemalloc.disable() + self.assertFalse(task.is_scheduled()) + + def test_task_callback_error(self): + calls = [] + def failing_func(): + calls.append((failing_func,)) + raise ValueError("oops") + + task = tracemalloc.Task(failing_func) + task.set_memory_threshold(1) + + # If the task raises an exception, the exception should be logged to + # sys.stderr (don't raise an exception at a random place) + with support.captured_stderr() as stderr: + task.schedule() + + obj, source = allocate_bytes(123) + obj2, source = allocate_bytes(456) + # the timer should not be rescheduler on error + self.assertEqual(len(calls), 1) + self.assertEqual(calls[0], (failing_func,)) + output = stderr.getvalue() + self.assertIn('ValueError: oops', output) + self.assertEqual(output.count('Traceback'), 1) + + self.assertFalse(task.is_scheduled()) + + def test_is_enabled(self): + tracemalloc.clear_traces() + tracemalloc.disable() + self.assertFalse(tracemalloc.is_enabled()) + + tracemalloc.enable() + self.assertTrue(tracemalloc.is_enabled()) + + def test_snapshot(self): + def compute_nstats(stats): + return sum(len(line_stats) + for filename, line_stats in stats.items()) + + tracemalloc.clear_traces() + obj, source = allocate_bytes(123) + + stats1 = tracemalloc.get_stats() + traces = tracemalloc.get_stats() + nstat1 = compute_nstats(stats1) + + # take a snapshot with traces + snapshot = tracemalloc.Snapshot.create(traces=True) + nstat2 = compute_nstats(snapshot.stats) + self.assertGreaterEqual(nstat2, nstat2) + self.assertEqual(snapshot.pid, os.getpid()) + process_rss = snapshot.get_metric('process_rss') + if process_rss is not None: + self.assertGreater(process_rss, 0) + process_vms = snapshot.get_metric('process_vms') + if process_rss is not None: + self.assertGreater(process_vms, 0) + + self.assertIsInstance(snapshot.metrics, dict) + for key in ('tracemalloc.arena_size', + 'tracemalloc.module.free', + 'tracemalloc.module.size', + 'tracemalloc.module.fragmentation', + 'tracemalloc.traced.max_size', + 'tracemalloc.traced.size', + 'unicode_interned.len', + 'unicode_interned.size'): + self.assertIn(key, snapshot.metrics) + + # write on disk + snapshot.write(support.TESTFN) + self.addCleanup(support.unlink, support.TESTFN) + + # load with traces + snapshot2 = tracemalloc.Snapshot.load(support.TESTFN) + self.assertEqual(snapshot2.timestamp, snapshot.timestamp) + self.assertEqual(snapshot2.pid, snapshot.pid) + self.assertEqual(snapshot2.traces, snapshot.traces) + self.assertEqual(snapshot2.stats, snapshot.stats) + self.assertEqual(snapshot2.metrics, snapshot.metrics) + + # load without traces + snapshot2 = tracemalloc.Snapshot.create() + self.assertIsNone(snapshot2.traces) + + # tracemalloc must be enabled to take a snapshot + tracemalloc.disable() + with self.assertRaises(RuntimeError) as cm: + tracemalloc.Snapshot.create() + self.assertEqual(str(cm.exception), + "the tracemalloc module must be enabled " + "to take a snapshot") + + def test_snapshot_metrics(self): + now = datetime.datetime.now() + snapshot = tracemalloc.Snapshot(now, 123, 1, {}) + + metric = snapshot.add_metric('key', 3, 'size') + self.assertRaises(ValueError, snapshot.add_metric, 'key', 4, 'size') + self.assertEqual(snapshot.get_metric('key'), 3) + self.assertIn('key', snapshot.metrics) + self.assertIs(metric, snapshot.metrics['key']) + + def test_take_snapshot(self): + def callback(snapshot): + snapshot.add_metric('callback', 5, 'size') + + with support.temp_cwd() as temp_dir: + task = tracemalloc.TakeSnapshotTask(callback=callback) + for index in range(1, 4): + snapshot, filename = task.take_snapshot() + self.assertEqual(snapshot.get_metric('callback'), 5) + self.assertEqual(filename, + 'tracemalloc-%04d.pickle' % index) + self.assertTrue(os.path.exists(filename)) + + def test_filters(self): + tracemalloc.clear_filters() + tracemalloc.add_exclude_filter(tracemalloc.__file__) + # test multiple inclusive filters + tracemalloc.add_include_filter('should never match 1') + tracemalloc.add_include_filter('should never match 2') + tracemalloc.add_include_filter(__file__) + tracemalloc.clear_traces() + size = 1000 + obj, obj_frames = allocate_bytes(size) + trace = tracemalloc.get_object_trace(obj) + self.assertIsNotNone(trace) + + # test exclusive filter, based on previous filters + filename, lineno = obj_frames[0] + tracemalloc.add_exclude_filter(filename, lineno) + tracemalloc.clear_traces() + obj, obj_frames = allocate_bytes(size) + trace = tracemalloc.get_object_trace(obj) + self.assertIsNone(trace) + + def fork_child(self): + # tracemalloc must be disabled after fork + enabled = tracemalloc.is_enabled() + if enabled: + return 2 + + # ensure that tracemalloc can be reenabled after fork + tracemalloc.enable() + + # check that tracemalloc is still working + obj_size = 12345 + obj, obj_frames = allocate_bytes(obj_size) + trace = tracemalloc.get_object_trace(obj) + if trace is None: + return 3 + + # everything is fine + return 0 + + @unittest.skipUnless(hasattr(os, 'fork'), 'need os.fork()') + def test_fork(self): + pid = os.fork() + if not pid: + # child + exitcode = 1 + try: + exitcode = self.fork_child() + finally: + os._exit(exitcode) + else: + pid2, status = os.waitpid(pid, 0) + self.assertTrue(os.WIFEXITED(status)) + exitcode = os.WEXITSTATUS(status) + self.assertEqual(exitcode, 0) + + +class TestSnapshot(unittest.TestCase): + def test_create_snapshot(self): + stats = {'a.py': {1: (5, 1)}} + traces = {0x123: (5, ('a.py', 1))} + + with contextlib.ExitStack() as stack: + stack.enter_context(patch.object(tracemalloc, 'is_enabled', return_value=True)) + stack.enter_context(patch.object(os, 'getpid', return_value=77)) + stack.enter_context(patch.object(tracemalloc, 'get_traceback_limit', return_value=5)) + stack.enter_context(patch.object(tracemalloc, 'get_stats', return_value=stats)) + stack.enter_context(patch.object(tracemalloc, 'get_traces', return_value=traces)) + + snapshot = tracemalloc.Snapshot.create(traces=True, metrics=False) + self.assertIsInstance(snapshot.timestamp, datetime.datetime) + self.assertEqual(snapshot.pid, 77) + self.assertEqual(snapshot.traceback_limit, 5) + self.assertEqual(snapshot.stats, stats) + self.assertEqual(snapshot.traces, traces) + self.assertEqual(snapshot.metrics, {}) + +class TestFilters(unittest.TestCase): + maxDiff = 2048 + def test_add_clear_filter(self): + old_filters = tracemalloc.get_filters() + try: + # test add_filter() + tracemalloc.clear_filters() + tracemalloc.add_filter(tracemalloc.Filter(True, "abc", 3)) + tracemalloc.add_filter(tracemalloc.Filter(False, "12345", 0)) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abc', 3, False), + tracemalloc.Filter(False, '12345', None, False)]) + + # test add_include_filter(), add_exclude_filter() + tracemalloc.clear_filters() + tracemalloc.add_include_filter("abc", 3) + tracemalloc.add_exclude_filter("12345", 0) + tracemalloc.add_exclude_filter("6789", None) + tracemalloc.add_exclude_filter("def#", 55) + tracemalloc.add_exclude_filter("trace", 123, True) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abc', 3, False), + tracemalloc.Filter(False, '12345', None, False), + tracemalloc.Filter(False, '6789', None, False), + tracemalloc.Filter(False, "def#", 55, False), + tracemalloc.Filter(False, "trace", 123, True)]) + + # test filename normalization (.pyc/.pyo) + tracemalloc.clear_filters() + tracemalloc.add_include_filter("abc.pyc") + tracemalloc.add_include_filter("name.pyo") + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abc.py', None, False), + tracemalloc.Filter(True, 'name.py', None, False) ]) + + # test filename normalization ('*' joker character) + tracemalloc.clear_filters() + tracemalloc.add_include_filter('a****b') + tracemalloc.add_include_filter('***x****') + tracemalloc.add_include_filter('1*2**3***4') + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'a*b', None, False), + tracemalloc.Filter(True, '*x*', None, False), + tracemalloc.Filter(True, '1*2*3*4', None, False)]) + + # ignore duplicated filters + tracemalloc.clear_filters() + tracemalloc.add_include_filter('a.py') + tracemalloc.add_include_filter('a.py', 5) + tracemalloc.add_include_filter('a.py') + tracemalloc.add_include_filter('a.py', 5) + tracemalloc.add_exclude_filter('b.py') + tracemalloc.add_exclude_filter('b.py', 10) + tracemalloc.add_exclude_filter('b.py') + tracemalloc.add_exclude_filter('b.py', 10, True) + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'a.py', None, False), + tracemalloc.Filter(True, 'a.py', 5, False), + tracemalloc.Filter(False, 'b.py', None, False), + tracemalloc.Filter(False, 'b.py', 10, False), + tracemalloc.Filter(False, 'b.py', 10, True)]) + + # Windows: test filename normalization (lower case, slash) + if os.name == "nt": + tracemalloc.clear_filters() + tracemalloc.add_include_filter("aBcD\xC9") + tracemalloc.add_include_filter("MODule.PYc") + tracemalloc.add_include_filter(r"path/to\file") + self.assertEqual(tracemalloc.get_filters(), + [tracemalloc.Filter(True, 'abcd\xe9', None, False), + tracemalloc.Filter(True, 'module.py', None, False), + tracemalloc.Filter(True, r'path\to\file', None, False)]) + + # test clear_filters() + tracemalloc.clear_filters() + self.assertEqual(tracemalloc.get_filters(), []) + finally: + tracemalloc.clear_filters() + for trace_filter in old_filters: + tracemalloc.add_filter(trace_filter) + + def test_filter_attributes(self): + # test default values + f = tracemalloc.Filter(True, "abc") + self.assertEqual(f.include, True) + self.assertEqual(f.pattern, "abc") + self.assertIsNone(f.lineno) + self.assertEqual(f.traceback, False) + + # test custom values + f = tracemalloc.Filter(False, "test.py", 123, True) + self.assertEqual(f.include, False) + self.assertEqual(f.pattern, "test.py") + self.assertEqual(f.lineno, 123) + self.assertEqual(f.traceback, True) + + # attributes are read-only + self.assertRaises(AttributeError, setattr, f, "include", True) + self.assertRaises(AttributeError, setattr, f, "pattern", True) + self.assertRaises(AttributeError, setattr, f, "lineno", True) + self.assertRaises(AttributeError, setattr, f, "traceback", True) + + def test_filter_match(self): + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f.match("abc", 5)) + self.assertTrue(f.match("abc", None)) + self.assertFalse(f.match("12356", 5)) + self.assertFalse(f.match("12356", None)) + self.assertFalse(f.match(None, 5)) + self.assertFalse(f.match(None, None)) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f.match("abc", 5)) + self.assertFalse(f.match("abc", None)) + self.assertTrue(f.match("12356", 5)) + self.assertTrue(f.match("12356", None)) + self.assertTrue(f.match(None, 5)) + self.assertTrue(f.match(None, None)) + + f = tracemalloc.Filter(True, "abc", 5) + self.assertTrue(f.match("abc", 5)) + self.assertFalse(f.match("abc", 10)) + self.assertFalse(f.match("abc", None)) + self.assertFalse(f.match("12356", 5)) + self.assertFalse(f.match("12356", 10)) + self.assertFalse(f.match("12356", None)) + self.assertFalse(f.match(None, 5)) + self.assertFalse(f.match(None, 10)) + self.assertFalse(f.match(None, None)) + + f = tracemalloc.Filter(False, "abc", 5) + self.assertFalse(f.match("abc", 5)) + self.assertTrue(f.match("abc", 10)) + self.assertTrue(f.match("abc", None)) + self.assertTrue(f.match("12356", 5)) + self.assertTrue(f.match("12356", 10)) + self.assertTrue(f.match("12356", None)) + self.assertTrue(f.match(None, 5)) + self.assertTrue(f.match(None, 10)) + self.assertTrue(f.match(None, None)) + + def test_filter_match_lineno(self): + f = tracemalloc.Filter(True, "unused") + self.assertTrue(f.match_lineno(5)) + self.assertTrue(f.match_lineno(10)) + self.assertTrue(f.match_lineno(None)) + + f = tracemalloc.Filter(True, "unused", 5) + self.assertTrue(f.match_lineno(5)) + self.assertFalse(f.match_lineno(10)) + self.assertFalse(f.match_lineno(None)) + + f = tracemalloc.Filter(False, "unused") + self.assertTrue(f.match_lineno(5)) + self.assertTrue(f.match_lineno(10)) + self.assertTrue(f.match_lineno(None)) + + f = tracemalloc.Filter(False, "unused", 5) + self.assertFalse(f.match_lineno(5)) + self.assertTrue(f.match_lineno(10)) + self.assertTrue(f.match_lineno(None)) + + def test_filter_match_filename(self): + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f.match_filename("abc")) + self.assertFalse(f.match_filename("12356")) + self.assertFalse(f.match_filename(None)) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f.match_filename("abc")) + self.assertTrue(f.match_filename("12356")) + self.assertTrue(f.match_filename(None)) + + f = tracemalloc.Filter(True, "abc") + self.assertTrue(f.match_filename("abc")) + self.assertFalse(f.match_filename("12356")) + self.assertFalse(f.match_filename(None)) + + f = tracemalloc.Filter(False, "abc") + self.assertFalse(f.match_filename("abc")) + self.assertTrue(f.match_filename("12356")) + self.assertTrue(f.match_filename(None)) + + def test_filter_match_filename_joker(self): + def fnmatch(filename, pattern): + filter = tracemalloc.Filter(True, pattern) + return filter.match_filename(filename) + + # no * + self.assertTrue(fnmatch('abc', 'abc')) + self.assertFalse(fnmatch('abc', 'abcd')) + self.assertFalse(fnmatch('abc', 'def')) + + # a* + self.assertTrue(fnmatch('abc', 'a*')) + self.assertTrue(fnmatch('abc', 'abc*')) + self.assertFalse(fnmatch('abc', 'b*')) + self.assertFalse(fnmatch('abc', 'abcd*')) + + # a*b + self.assertTrue(fnmatch('abc', 'a*c')) + self.assertTrue(fnmatch('abcdcx', 'a*cx')) + self.assertFalse(fnmatch('abb', 'a*c')) + self.assertFalse(fnmatch('abcdce', 'a*cx')) + + # a*b*c + self.assertTrue(fnmatch('abcde', 'a*c*e')) + self.assertTrue(fnmatch('abcbdefeg', 'a*bd*eg')) + self.assertFalse(fnmatch('abcdd', 'a*c*e')) + self.assertFalse(fnmatch('abcbdefef', 'a*bd*eg')) + + # replace .pyc and .pyo suffix with .py + self.assertTrue(fnmatch('a.pyc', 'a.py')) + self.assertTrue(fnmatch('a.pyo', 'a.py')) + self.assertTrue(fnmatch('a.py', 'a.pyc')) + self.assertTrue(fnmatch('a.py', 'a.pyo')) + + if os.name == 'nt': + # case insensitive + self.assertTrue(fnmatch('aBC', 'ABc')) + self.assertTrue(fnmatch('aBcDe', 'Ab*dE')) + + self.assertTrue(fnmatch('a.pyc', 'a.PY')) + self.assertTrue(fnmatch('a.PYO', 'a.py')) + self.assertTrue(fnmatch('a.py', 'a.PYC')) + self.assertTrue(fnmatch('a.PY', 'a.pyo')) + else: + # case sensitive + self.assertFalse(fnmatch('aBC', 'ABc')) + self.assertFalse(fnmatch('aBcDe', 'Ab*dE')) + + self.assertFalse(fnmatch('a.pyc', 'a.PY')) + self.assertFalse(fnmatch('a.PYO', 'a.py')) + self.assertFalse(fnmatch('a.py', 'a.PYC')) + self.assertFalse(fnmatch('a.PY', 'a.pyo')) + + if os.name == 'nt': + # normalize alternate separator "/" to the standard separator "\" + self.assertTrue(fnmatch(r'a/b', r'a\b')) + self.assertTrue(fnmatch(r'a\b', r'a/b')) + self.assertTrue(fnmatch(r'a/b\c', r'a\b/c')) + self.assertTrue(fnmatch(r'a/b/c', r'a\b\c')) + else: + # there is no alternate separator + self.assertFalse(fnmatch(r'a/b', r'a\b')) + self.assertFalse(fnmatch(r'a\b', r'a/b')) + self.assertFalse(fnmatch(r'a/b\c', r'a\b/c')) + self.assertFalse(fnmatch(r'a/b/c', r'a\b\c')) + + # a******b + N = 10 ** 6 + self.assertTrue (fnmatch('a' * N, '*' * N)) + self.assertTrue (fnmatch('a' * N + 'c', '*' * N)) + self.assertTrue (fnmatch('a' * N, 'a' + '*' * N + 'a')) + self.assertTrue (fnmatch('a' * N + 'b', 'a' + '*' * N + 'b')) + self.assertFalse(fnmatch('a' * N + 'b', 'a' + '*' * N + 'c')) + + # a*a*a*a* + self.assertTrue(fnmatch('a' * 10, 'a*' * 10)) + self.assertFalse(fnmatch('a' * 10, 'a*' * 10 + 'b')) + with self.assertRaises(ValueError) as cm: + fnmatch('abc', 'a*' * 101) + self.assertEqual(str(cm.exception), + "too many joker characters in the filename pattern") + + def test_filter_match_trace(self): + t1 = (("a.py", 2), ("b.py", 3)) + t2 = (("b.py", 4), ("b.py", 5)) + + f = tracemalloc.Filter(True, "b.py", traceback=True) + self.assertTrue(f.match_traceback(t1)) + self.assertTrue(f.match_traceback(t2)) + + f = tracemalloc.Filter(True, "b.py", traceback=False) + self.assertFalse(f.match_traceback(t1)) + self.assertTrue(f.match_traceback(t2)) + + f = tracemalloc.Filter(False, "b.py", traceback=True) + self.assertFalse(f.match_traceback(t1)) + self.assertFalse(f.match_traceback(t2)) + + f = tracemalloc.Filter(False, "b.py", traceback=False) + self.assertTrue(f.match_traceback(t1)) + self.assertFalse(f.match_traceback(t2)) + + +class TestCommandLine(unittest.TestCase): + def test_env_var(self): + # disabled by default + code = 'import tracemalloc; print(tracemalloc.is_enabled())' + ok, stdout, stderr = assert_python_ok('-c', code) + stdout = stdout.rstrip() + self.assertEqual(stdout, b'False') + + # PYTHON* environment varibles must be ignored when -E option is + # present + code = 'import tracemalloc; print(tracemalloc.is_enabled())' + ok, stdout, stderr = assert_python_ok('-E', '-c', code, PYTHONTRACEMALLOC='1') + stdout = stdout.rstrip() + self.assertEqual(stdout, b'False') + + # enabled by default + code = 'import tracemalloc; print(tracemalloc.is_enabled())' + ok, stdout, stderr = assert_python_ok('-c', code, PYTHONTRACEMALLOC='1') + stdout = stdout.rstrip() + self.assertEqual(stdout, b'True') + + def test_sys_xoptions(self): + # -X tracemalloc + code = 'import tracemalloc; print(tracemalloc.is_enabled())' + ok, stdout, stderr = assert_python_ok('-X', 'tracemalloc', '-c', code) + stdout = stdout.rstrip() + self.assertEqual(stdout, b'True') + + +class TestTop(unittest.TestCase): + maxDiff = 2048 + + def test_snapshot_top_by_line(self): + snapshot, snapshot2 = create_snapshots() + + # stats per file and line + top_stats = snapshot.top_by('line') + self.assertEqual(top_stats.stats, { + ('a.py', 2): (30, 3), + ('a.py', 5): (2, 1), + ('b.py', 1): (66, 1), + (None, None): (7, 1), + }) + self.assertEqual(top_stats.group_by, 'line') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, False) + self.assertEqual(top_stats.metrics, snapshot.metrics) + + # stats per file and line (2) + top_stats2 = snapshot2.top_by('line') + self.assertEqual(top_stats2.stats, { + ('a.py', 2): (30, 3), + ('a.py', 5): (5002, 2), + ('c.py', 578): (400, 1), + }) + self.assertEqual(top_stats2.group_by, 'line') + self.assertEqual(top_stats2.timestamp, snapshot2.timestamp) + self.assertEqual(top_stats2.cumulative, False) + self.assertEqual(top_stats2.metrics, snapshot2.metrics) + + # stats diff per file and line + top_diff = top_stats2.compare_to(top_stats) + self.assertIsInstance(top_diff, tracemalloc.StatsDiff) + top_diff.sort() + self.assertEqual(top_diff.differences, [ + (5000, 5002, 1, 2, ('a.py', 5)), + (400, 400, 1, 1, ('c.py', 578)), + (-66, 0, -1, 0, ('b.py', 1)), + (-7, 0, -1, 0, ('', 0)), + (0, 30, 0, 3, ('a.py', 2)), + ]) + + def test_snapshot_top_by_file(self): + snapshot, snapshot2 = create_snapshots() + + # stats per file + top_stats = snapshot.top_by('filename') + self.assertEqual(top_stats.stats, { + 'a.py': (32, 4), + 'b.py': (66, 1), + None: (7, 1), + }) + self.assertEqual(top_stats.group_by, 'filename') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, False) + self.assertEqual(top_stats.metrics, snapshot.metrics) + + # stats per file (2) + top_stats2 = snapshot2.top_by('filename') + self.assertEqual(top_stats2.stats, { + 'a.py': (5032, 5), + 'c.py': (400, 1), + }) + self.assertEqual(top_stats2.group_by, 'filename') + self.assertEqual(top_stats2.timestamp, snapshot2.timestamp) + self.assertEqual(top_stats2.cumulative, False) + self.assertEqual(top_stats2.metrics, snapshot2.metrics) + + # stats diff per file + top_diff = top_stats2.compare_to(top_stats) + self.assertIsInstance(top_diff, tracemalloc.StatsDiff) + top_diff.sort() + self.assertEqual(top_diff.differences, [ + (5000, 5032, 1, 5, 'a.py'), + (400, 400, 1, 1, 'c.py'), + (-66, 0, -1, 0, 'b.py'), + (-7, 0, -1, 0, ''), + ]) + + def test_snapshot_top_by_address(self): + snapshot, snapshot2 = create_snapshots() + + # stats per address + top_stats = snapshot.top_by('address') + self.assertEqual(top_stats.stats, { + 0x10001: (10, 1), + 0x10002: (10, 1), + 0x10003: (10, 1), + 0x20001: (2, 1), + 0x30001: (66, 1), + 0x40001: (7, 1), + }) + self.assertEqual(top_stats.group_by, 'address') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, False) + self.assertEqual(top_stats.metrics, snapshot.metrics) + + # stats per address (2) + top_stats2 = snapshot2.top_by('address') + self.assertEqual(top_stats2.stats, { + 0x10001: (10, 1), + 0x10002: (10, 1), + 0x10003: (10, 1), + 0x20001: (2, 1), + 0x20002: (5000, 1), + 0x30001: (400, 1), + }) + self.assertEqual(top_stats2.group_by, 'address') + self.assertEqual(top_stats2.timestamp, snapshot2.timestamp) + self.assertEqual(top_stats2.cumulative, False) + self.assertEqual(top_stats2.metrics, snapshot2.metrics) + + # diff + top_diff = top_stats2.compare_to(top_stats) + self.assertIsInstance(top_diff, tracemalloc.StatsDiff) + top_diff.sort() + self.assertEqual(top_diff.differences, [ + (5000, 5000, 1, 1, 0x20002), + (334, 400, 0, 1, 0x30001), + (-7, 0, -1, 0, 0x40001), + (0, 10, 0, 1, 0x10003), + (0, 10, 0, 1, 0x10002), + (0, 10, 0, 1, 0x10001), + (0, 2, 0, 1, 0x20001), + ]) + + with self.assertRaises(ValueError) as cm: + snapshot.traces = None + snapshot.top_by('address') + self.assertEqual(str(cm.exception), "need traces") + + def test_snapshot_top_cumulative(self): + snapshot, snapshot2 = create_snapshots() + + # per file + top_stats = snapshot.top_by('filename', True) + self.assertEqual(top_stats.stats, { + 'a.py': (32, 4), + 'b.py': (98, 5), + None: (7, 1), + }) + self.assertEqual(top_stats.group_by, 'filename') + self.assertEqual(top_stats.timestamp, snapshot.timestamp) + self.assertEqual(top_stats.cumulative, True) + self.assertEqual(top_stats.metrics, snapshot.metrics) + + # per line + top_stats2 = snapshot.top_by('line', True) + self.assertEqual(top_stats2.stats, { + ('a.py', 2): (30, 3), + ('a.py', 5): (2, 1), + ('b.py', 1): (66, 1), + ('b.py', 4): (32, 4), + (None, None): (7, 1), + }) + self.assertEqual(top_stats2.group_by, 'line') + self.assertEqual(top_stats2.timestamp, snapshot.timestamp) + self.assertEqual(top_stats2.cumulative, True) + self.assertEqual(top_stats2.metrics, snapshot.metrics) + + with self.assertRaises(ValueError) as cm: + snapshot.traces = None + snapshot.top_by('filename', True) + self.assertEqual(str(cm.exception), "need traces") + + def test_display_top_by_line(self): + snapshot, snapshot2 = create_snapshots() + + # top per line + output = io.StringIO() + top = tracemalloc.DisplayTop() + top.display_snapshot(snapshot, file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:17: Top 4 allocations per filename and line number +#1: b.py:1: size=66 B, count=1 +#2: a.py:2: size=30 B, count=3, average=10 B +#3: ???:?: size=7 B, count=1 +#4: a.py:5: size=2 B, count=1 +Traced Python memory: size=105 B, count=6, average=17 B + +my_data: 8 +process_memory.rss: 1024 B +tracemalloc.size: 100 B + '''.strip() + '\n\n') + + # diff per line + output = io.StringIO() + top.display_snapshot(snapshot2, count=3, file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:50: Top 3 allocations per filename and line number (compared to 2013-09-12 15:16:17) +#1: a.py:5: size=5002 B (+5000 B), count=2 (+1), average=2501 B +#2: c.py:578: size=400 B (+400 B), count=1 (+1) +#3: b.py:1: size=0 B (-66 B), count=0 (-1) +2 more: size=30 B (-7 B), count=3 (-1), average=10 B +Traced Python memory: size=5 KiB (+5 KiB), count=6 (+0), average=905 B + +my_data: 10 (+2) +process_memory.rss: 1500 B (+476 B) +tracemalloc.size: 200 B (+100 B) + '''.strip() + '\n\n') + + def test_display_top_by_file(self): + snapshot, snapshot2 = create_snapshots() + + # group per file + output = io.StringIO() + top = tracemalloc.DisplayTop() + top.display_snapshot(snapshot, group_by='filename', file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:17: Top 3 allocations per filename +#1: b.py: size=66 B, count=1 +#2: a.py: size=32 B, count=4, average=8 B +#3: ???: size=7 B, count=1 +Traced Python memory: size=105 B, count=6, average=17 B + +my_data: 8 +process_memory.rss: 1024 B +tracemalloc.size: 100 B + '''.strip() + '\n\n') + + # diff per file + output = io.StringIO() + top.display_snapshot(snapshot2, group_by='filename', file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:50: Top 4 allocations per filename (compared to 2013-09-12 15:16:17) +#1: a.py: size=5032 B (+5000 B), count=5 (+1), average=1006 B +#2: c.py: size=400 B (+400 B), count=1 (+1) +#3: b.py: size=0 B (-66 B), count=0 (-1) +#4: ???: size=0 B (-7 B), count=0 (-1) +Traced Python memory: size=5 KiB (+5 KiB), count=6 (+0), average=905 B + +my_data: 10 (+2) +process_memory.rss: 1500 B (+476 B) +tracemalloc.size: 200 B (+100 B) + '''.strip() + '\n\n') + + def test_display_top_options(self): + snapshot, snapshot2 = create_snapshots() + output = io.StringIO() + top = tracemalloc.DisplayTop() + top.metrics = False + top.average = False + top.count = False + top.display_snapshot(snapshot, file=output) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:17: Top 4 allocations per filename and line number +#1: b.py:1: 66 B +#2: a.py:2: 30 B +#3: ???:?: 7 B +#4: a.py:5: 2 B +Traced Python memory: 105 B + '''.strip() + '\n\n') + + def test_display_top_task(self): + def callback(snapshot): + snapshot.add_metric('task', 700, 'size') + + snapshot, snapshot2 = create_snapshots() + + # top per file (default options) + output = io.StringIO() + top = tracemalloc.DisplayTop() + + with patch.object(tracemalloc.Snapshot, + 'create', return_value=snapshot): + top.display(2, group_by='filename', file=output, callback=callback) + text = output.getvalue() + self.assertEqual(text, ''' +2013-09-12 15:16:17: Top 2 allocations per filename +#1: b.py: size=66 B, count=1 +#2: a.py: size=32 B, count=4, average=8 B +1 more: size=7 B, count=1 +Traced Python memory: size=105 B, count=6, average=17 B + +my_data: 8 +process_memory.rss: 1024 B +task: 700 B +tracemalloc.size: 100 B + '''.strip() + '\n\n') + + +class TestTask(unittest.TestCase): + def test_func_args(self): + def func2(*args, **kw): + pass + + # constructor + task = tracemalloc.Task(noop, 1, 2, 3, key='value') + self.assertIs(task.func, noop) + self.assertEqual(task.func_args, (1, 2, 3)) + self.assertEqual(task.func_kwargs, {'key': 'value'}) + + # func + task.func = print + self.assertIs(task.func, print) + self.assertRaises(TypeError, setattr, task, 'func', 5) + + # func_args + task.func_args = ("Hello", "World!") + self.assertEqual(task.func_args, ("Hello", "World!")) + self.assertRaises(TypeError, task.func_args, 5) + + # func_kwargs + task.func_kwargs = {'flush': True} + self.assertEqual(task.func_kwargs, {'flush': True}) + task.func_kwargs = None + self.assertIsNone(task.func_kwargs) + self.assertRaises(TypeError, task.func_kwargs, 5) + + def test_memory_threshold(self): + task = tracemalloc.Task(noop) + self.assertIsNone(task.get_memory_threshold()) + + task.set_memory_threshold(1024 * 1024) + self.assertEqual(task.get_memory_threshold(), 1024 * 1024) + + self.assertRaises(ValueError, task.set_memory_threshold, 0) + self.assertRaises(ValueError, task.set_memory_threshold, -1) + self.assertRaises(TypeError, task.set_memory_threshold, 99.9) + self.assertRaises(TypeError, task.set_memory_threshold, "str") + + def test_delay(self): + task = tracemalloc.Task(noop) + self.assertIsNone(task.get_delay()) + + task.set_delay(60) + self.assertEqual(task.get_delay(), 60) + + task.set_delay(9.9) + self.assertEqual(task.get_delay(), 9) + self.assertRaises(ValueError, task.set_delay, -1) + self.assertRaises(ValueError, task.set_delay, 0) + self.assertRaises(TypeError, task.set_delay, "str") + + def test_call(self): + calls = [] + def log_func(*args, **kwargs): + calls.append((args, kwargs)) + + task = tracemalloc.Task(log_func, 1, 2, 3, key='value') + task.call() + self.assertEqual(len(calls), 1) + args, kwargs = calls[0] + self.assertEqual(args, (1, 2, 3)) + self.assertEqual(kwargs, {'key': 'value'}) + + +def test_main(): + support.run_unittest( + TestTracemallocEnabled, + TestSnapshot, + TestFilters, + TestCommandLine, + TestTop, + TestTask, + ) + +if __name__ == "__main__": + test_main() diff -r 50e4d0c62c2b -r ec121a72e848 Lib/tracemalloc.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Lib/tracemalloc.py Sun Oct 06 17:53:23 2013 +0200 @@ -0,0 +1,1073 @@ +from __future__ import with_statement +import _tracemalloc +import collections +import datetime +import functools +import gc +import linecache +import operator +import os +import pickle +import sys +import types +try: + from time import monotonic as _time_monotonic +except ImportError: + from time import time as _time_monotonic + +# Import types and functions implemented in C +from _tracemalloc import * + +def _format_timestamp(timestamp): + return str(timestamp).split(".", 1)[0] + +def _format_size(size, sign=False): + for unit in ('B', 'KiB', 'MiB', 'GiB'): + if abs(size) < 5 * 1024: + if sign: + return "%+i %s" % (size, unit) + else: + return "%i %s" % (size, unit) + size /= 1024 + + if sign: + return "%+i TiB" % size + else: + return "%i TiB" % size + +if os.name != "nt": + _FORMAT_YELLOW = '\x1b[1;33m%s\x1b[0m' + _FORMAT_BOLD = '\x1b[1m%s\x1b[0m' + _FORMAT_CYAN = '\x1b[36m%s\x1b[0m' +else: + _FORMAT_YELLOW = _FORMAT_BOLD = _FORMAT_CYAN = "%s" + +def _format_size_color(size, color): + text = _format_size(size) + if color: + text = _FORMAT_YELLOW % text + return text + +def _format_size_diff(size, diff, color): + text = _format_size(size) + if diff is not None: + if color: + text = _FORMAT_BOLD % text + textdiff = _format_size(diff, sign=True) + if color: + textdiff = _FORMAT_YELLOW % textdiff + text += " (%s)" % textdiff + else: + if color: + text = _FORMAT_YELLOW % text + return text + +def _format_address(address, color): + address = '0x%x' % address + if color: + address = _FORMAT_BOLD % address + return address + +def _format_int(value, color): + text = '%i' % value + if color: + text = _FORMAT_YELLOW % text + return text + +def _colorize_filename(filename): + path, basename = os.path.split(filename) + if path: + path += os.path.sep + return _FORMAT_CYAN % path + basename + +def _format_filename(filename, max_parts, color): + if filename: + parts = filename.split(os.path.sep) + if max_parts < len(parts): + parts = ['...'] + parts[-max_parts:] + filename = os.path.sep.join(parts) + else: + # filename is None or an empty string + filename = '???' + if color: + filename = _colorize_filename(filename) + return filename + +def _format_lineno(lineno): + if lineno: + return str(lineno) + else: + # lineno is None or an empty string + return '?' + +def _format_traceback(traceback, filename_parts, color): + if traceback is None: + return ('(empty traceback)',) + lines = ['Traceback (most recent call first):'] + if traceback is not None: + for frame in traceback: + filename, lineno = frame + if filename and lineno: + line = linecache.getline(filename, lineno) + line = line.strip() + else: + line = None + + filename = _format_filename(filename, filename_parts, color) + lineno = _format_lineno(lineno) + lines.append(' File "%s", line %s' % (filename, lineno)) + if line: + lines.append(' ' + line) + else: + filename = _format_filename(None, filename_parts, color) + lineno = _format_lineno(None) + lines.append(' File "%s", line %s' % (filename, lineno)) + return lines + +# On Windows, get_process_memory() is implemented in _tracemalloc +if os.name != "nt": + def get_process_memory(): + """ + Get the memory usage of the current process as a (rss: int, vms: int) + tuple, rss is the Resident Set Size in bytes and vms is the size of the + virtual memory in bytes + + Return None if the platform is not supported. + """ + if get_process_memory.support_proc == False: + return None + + try: + fp = open("/proc/self/statm", "rb") + except IOError: + get_process_memory.support_proc = False + return None + + try: + page_size = os.sysconf("SC_PAGE_SIZE") + except AttributeError: + get_process_memory.support_proc = False + return None + + get_process_memory.support_proc = True + with fp: + statm = fp.readline().split() + vms = int(statm[0]) * page_size + rss = int(statm[1]) * page_size + return (rss, vms) + + get_process_memory.support_proc = None + +def _stat_key(stats): + return (abs(stats[0]), stats[1], abs(stats[2]), stats[3], stats[4]) + +class StatsDiff: + __slots__ = ('differences', 'old_stats', 'new_stats') + + def __init__(self, differences, old_stats, new_stats): + self.differences = differences + self.old_stats = old_stats + self.new_stats = new_stats + + def sort(self): + self.differences.sort(reverse=True, key=_stat_key) + + +class GroupedStats: + __slots__ = ('timestamp', 'stats', 'group_by', 'cumulative', 'metrics') + + def __init__(self, timestamp, stats, group_by, + cumulative=False, metrics=None): + if group_by not in ('filename', 'line', 'address'): + raise ValueError("invalid group_by value") + # dictionary {key: stats} where stats is + # a (size: int, count: int) tuple + self.stats = stats + self.group_by = group_by + self.cumulative = cumulative + self.timestamp = timestamp + self.metrics = metrics + + def _create_key(self, key): + if self.group_by == 'filename': + if key is None: + return '' + elif self.group_by == 'line': + filename, lineno = key + if filename is None: + filename = '' + if lineno is None: + lineno = 0 + return (filename, lineno) + return key + + def compare_to(self, old_stats=None): + if old_stats is not None: + previous_dict = old_stats.stats.copy() + + differences = [] + for key, stats in self.stats.items(): + size, count = stats + previous = previous_dict.pop(key, None) + key = self._create_key(key) + if previous is not None: + diff = (size - previous[0], size, + count - previous[1], count, + key) + else: + diff = (size, size, count, count, key) + differences.append(diff) + + for key, stats in previous_dict.items(): + key = self._create_key(key) + diff = (-stats[0], 0, -stats[1], 0, key) + differences.append(diff) + else: + differences = [ + (0, stats[0], 0, stats[1], self._create_key(key)) + for key, stats in self.stats.items()] + + return StatsDiff(differences, old_stats, self) + + +class DisplayTop: + def __init__(self): + self.size = True + self.count = True + self.average = True + self.metrics = True + self.filename_parts = 3 + self.color = None + self.compare_to_previous = True + self.previous_top_stats = None + + def _format_diff(self, diff, show_diff, show_count, color): + if not show_count and not self.average: + if show_diff: + return _format_size_diff(diff[1], diff[0], color) + else: + return _format_size_color(diff[1], color) + + parts = [] + if self.size: + if show_diff: + text = _format_size_diff(diff[1], diff[0], color) + else: + text = _format_size_color(diff[1], color) + parts.append("size=%s" % text) + if show_count and (diff[3] or diff[2]): + text = "count=%s" % diff[3] + if show_diff: + text += " (%+i)" % diff[2] + parts.append(text) + if (self.average + and diff[3] > 1): + parts.append('average=%s' % _format_size_color(diff[1] // diff[3], False)) + return ', '.join(parts) + + def _format_filename(self, key, color): + return _format_filename(key, self.filename_parts, color) + + def _format_address(self, key, color): + return 'memory block %s' % _format_address(key, color) + + def _format_filename_lineno(self, key, color): + filename, lineno = key + filename = _format_filename(filename, self.filename_parts, color) + lineno = _format_lineno(lineno) + return "%s:%s" % (filename, lineno) + + def _format_metric(self, value, format, sign=False): + if format == 'size': + return _format_size(value, sign=sign) + elif format == 'percent': + if sign: + return "%+.1f%%" % (value * 100) + else: + return "%.1f%%" % (value * 100) + else: + if sign: + return "%+i" % value + else: + return "%i" % value + + def _display_metrics(self, log, previous_top_stats, top_stats, color): + if top_stats.metrics is None and previous_top_stats is None: + return + + if previous_top_stats is not None: + old_metrics = previous_top_stats.metrics + else: + old_metrics = {} + if top_stats is not None: + new_metrics = top_stats.metrics + else: + new_metrics = {} + + names = list(old_metrics.keys() | new_metrics.keys()) + names.sort() + if not names: + return + + log("\n") + for name in names: + old_metric = old_metrics.get(name) + if old_metric is not None: + old_value = old_metric.value + name = old_metric.name + format = old_metric.format + else: + old_value = None + + new_metric = new_metrics.get(name) + if new_metric is not None: + new_value = new_metric.value + name = new_metric.name + format = new_metric.format + else: + new_value = 0 + + if format == 'size': + formatter = _format_size_color + else: + text = _format_int + + text = self._format_metric(new_value, format) + if color: + text = _FORMAT_BOLD % text + if old_value is not None: + diff = self._format_metric(new_value - old_value, format, sign=True) + if color: + diff = _FORMAT_YELLOW % diff + text = '%s (%s)' % (text, diff) + log("%s: %s\n" % (name, text)) + + def display_top_diff(self, top_diff, count=10, file=None): + if file is None: + file = sys.stdout + log = file.write + if self.color is None: + color = file.isatty() + else: + color = self.color + diff_list = top_diff.differences + top_stats = top_diff.new_stats + previous_top_stats = top_diff.old_stats + has_previous = (top_diff.old_stats is not None) + if top_stats.group_by == 'address': + show_count = False + else: + show_count = self.count + + if top_stats.group_by == 'filename': + format_key = self._format_filename + per_text = "filename" + elif top_stats.group_by == 'address': + format_key = self._format_address + per_text = "address" + else: + format_key = self._format_filename_lineno + per_text = "filename and line number" + + # Write the header + nother = max(len(diff_list) - count, 0) + count = min(count, len(diff_list)) + if top_stats.cumulative: + text = "Cumulative top %s allocations per %s" % (count, per_text) + else: + text = "Top %s allocations per %s" % (count, per_text) + if color: + text = _FORMAT_CYAN % text + if previous_top_stats is not None: + text += ' (compared to %s)' % _format_timestamp(previous_top_stats.timestamp) + name = _format_timestamp(top_stats.timestamp) + if color: + name = _FORMAT_BOLD % name + file.write("%s: %s\n" % (name, text)) + + # Sort differences by size and then by count + top_diff.sort() + + # Display items + total = [0, 0, 0, 0] + for index in range(0, count): + diff = diff_list[index] + key_text = format_key(diff[4], color) + diff_text = self._format_diff(diff, has_previous, show_count, color) + log("#%s: %s: %s\n" % (1 + index, key_text, diff_text)) + total[0] += diff[0] + total[1] += diff[1] + total[2] += diff[2] + total[3] += diff[3] + + other = tuple(total) + for index in range(count, len(diff_list)): + diff = diff_list[index] + total[0] += diff[0] + total[1] += diff[1] + total[2] += diff[2] + total[3] += diff[3] + + # Display "xxx more" + if nother > 0: + other = [ + total[0] - other[0], + total[1] - other[1], + total[2] - other[2], + total[3] - other[3], + ] + other = self._format_diff(other, has_previous, show_count, color) + text = "%s more" % nother + if color: + text = _FORMAT_CYAN % text + log("%s: %s\n" % (text, other)) + + if not top_stats.cumulative: + text = self._format_diff(total, has_previous, show_count, color) + log("Traced Python memory: %s\n" % text) + + if self.metrics: + self._display_metrics(log, previous_top_stats, top_stats, color) + + log("\n") + file.flush() + + # store the current top stats as the previous top stats for later + # comparison with a newer top stats + if self.compare_to_previous: + self.previous_top_stats = top_stats + else: + if self.previous_top_stats is None: + self.previous_top_stats = top_stats + + def display_top_stats(self, top_stats, count=10, file=None): + top_diff = top_stats.compare_to(self.previous_top_stats) + self.display_top_diff(top_diff, count=count, file=file) + + def display_snapshot(self, snapshot, count=10, group_by="line", + cumulative=False, file=None): + top_stats = snapshot.top_by(group_by, cumulative) + self.display_top_stats(top_stats, count=count, file=file) + + def display(self, count=10, group_by="line", cumulative=False, file=None, + callback=None): + if group_by == 'address': + traces = True + elif cumulative: + traces = (tracemalloc.get_traceback_limit() > 1) + else: + traces = False + snapshot = Snapshot.create(traces=traces, + metrics=self.metrics) + if callback is not None: + callback(snapshot) + + self.display_snapshot(snapshot, + count=count, + group_by=group_by, + cumulative=cumulative, + file=file) + return snapshot + + +class DisplayTopTask(Task): + def __init__(self, count, group_by="line", cumulative=False, + file=None, callback=None): + Task.__init__(self, self.display) + self.display_top = DisplayTop() + self.count = count + self.group_by = group_by + self.cumulative = cumulative + self.file = file + self.callback = callback + self._task = None + + def display(self): + return self.display_top.display(self.count, self.group_by, + self.cumulative, self.file, + self.callback) + + +def _compute_stats_frame(stats, group_per_file, size, frame): + if not group_per_file: + if frame is not None: + key = frame + else: + key = (None, None) + else: + if frame is not None: + key = frame[0] + else: + key = None + if key in stats: + stat_size, count = stats[key] + size += stat_size + count = count + 1 + else: + count = 1 + stats[key] = (size, count) + + +class Metric: + __slots__ = ('name', 'value', 'format') + + def __init__(self, name, value, format): + self.name = name + self.value = value + self.format = format + + def __eq__(self, other): + return (self.name == other.name and self.value == other.value) + + def __repr__(self): + return ('' + % (self.name, self.value, self.format)) + + +class Snapshot: + FORMAT_VERSION = (3, 4) + __slots__ = ('timestamp', 'pid', 'traceback_limit', + 'stats', 'traces', 'metrics') + + def __init__(self, timestamp, pid, traceback_limit, + stats=None, traces=None, metrics=None): + if traces is None and stats is None: + raise ValueError("traces and stats cannot be None at the same time") + self.timestamp = timestamp + self.pid = pid + self.traceback_limit = traceback_limit + self.stats = stats + self.traces = traces + if metrics: + self.metrics = metrics + else: + self.metrics = {} + + def add_metric(self, name, value, format): + if name in self.metrics: + raise ValueError("name already present: %r" % (name,)) + metric = Metric(name, value, format) + self.metrics[metric.name] = metric + return metric + + def add_tracemalloc_metrics(self): + size, max_size = get_traced_memory() + self.add_metric('tracemalloc.traced.size', size, 'size') + self.add_metric('tracemalloc.traced.max_size', max_size, 'size') + + if self.traces: + self.add_metric('tracemalloc.traces', len(self.traces), 'int') + + size, free = get_tracemalloc_memory() + self.add_metric('tracemalloc.module.size', size, 'size') + self.add_metric('tracemalloc.module.free', free, 'size') + if size: + frag = free / size + self.add_metric('tracemalloc.module.fragmentation', frag, 'percent') + + self.add_metric('tracemalloc.arena_size', get_arena_size(), 'size') + + def add_unicode_metrics(self): + unicode_interned = get_unicode_interned() + self.add_metric('unicode_interned.size', unicode_interned[0], 'size') + self.add_metric('unicode_interned.len', unicode_interned[1], 'int') + + def add_pymalloc_metrics(self): + self.add_metric('pymalloc.blocks', get_allocated_blocks(), 'int') + if hasattr(_tracemalloc, 'get_pymalloc_stats'): + pymalloc = _tracemalloc.get_pymalloc_stats() + + size = pymalloc['arenas'] * pymalloc['arena_size'] + self.add_metric('pymalloc.size', size, 'size') + + size = pymalloc['max_arenas'] * pymalloc['arena_size'] + self.add_metric('pymalloc.max_size', size, 'size') + + data = pymalloc['allocated_bytes'] + self.add_metric('pymalloc.allocated', data, 'size') + + free = pymalloc['available_bytes'] + free += pymalloc['free_pools'] * pymalloc['pool_size'] + self.add_metric('pymalloc.free', free, 'size') + + size = (data + free) + if size: + fragmentation = free / size + self.add_metric('pymalloc.fragmentation', fragmentation, 'percent') + + def add_process_memory_metrics(self): + process_memory = get_process_memory() + if process_memory is not None: + self.add_metric('process_memory.rss', process_memory[0], 'size') + self.add_metric('process_memory.vms', process_memory[1], 'size') + + def add_gc_metrics(self): + self.add_metric('gc.objects', len(gc.get_objects()), 'int') + + def get_metric(self, name, default=None): + if name in self.metrics: + return self.metrics[name].value + else: + return default + + @classmethod + def create(cls, traces=False, metrics=True): + if not is_enabled(): + raise RuntimeError("the tracemalloc module must be enabled " + "to take a snapshot") + timestamp = datetime.datetime.now() + pid = os.getpid() + traceback_limit = get_traceback_limit() + if traces: + traces = get_traces() + else: + traces = None + stats = get_stats() + + snapshot = cls(timestamp, pid, traceback_limit, stats, traces) + if metrics: + snapshot.add_tracemalloc_metrics() + snapshot.add_unicode_metrics() + snapshot.add_pymalloc_metrics() + snapshot.add_gc_metrics() + snapshot.add_process_memory_metrics() + return snapshot + + @classmethod + def load(cls, filename, traces=True): + with open(filename, "rb") as fp: + data = pickle.load(fp) + + try: + if data['format_version'] != cls.FORMAT_VERSION: + raise TypeError("unknown format version") + + timestamp = data['timestamp'] + pid = data['pid'] + stats = data['stats'] + traceback_limit = data['traceback_limit'] + metrics = data.get('metrics') + except KeyError: + raise TypeError("invalid file format") + + if traces: + traces = pickle.load(fp) + else: + traces = None + + return cls(timestamp, pid, traceback_limit, stats, traces, metrics) + + def write(self, filename): + data = { + 'format_version': self.FORMAT_VERSION, + 'timestamp': self.timestamp, + 'pid': self.pid, + 'traceback_limit': self.traceback_limit, + 'stats': self.stats, + } + if self.metrics: + data['metrics'] = self.metrics + + try: + with open(filename, "wb") as fp: + pickle.dump(data, fp, pickle.HIGHEST_PROTOCOL) + pickle.dump(self.traces, fp, pickle.HIGHEST_PROTOCOL) + except: + # Remove corrupted pickle file + if os.path.exists(filename): + os.unlink(filename) + raise + + def _filter_traces(self, include, filters): + new_traces = {} + for address, trace in self.traces.items(): + if include: + match = any(trace_filter.match_traceback(trace[1]) + for trace_filter in filters) + else: + match = all(trace_filter.match_traceback(trace[1]) + for trace_filter in filters) + if match: + new_traces[address] = trace + return new_traces + + def _filter_stats(self, include, filters): + file_stats = {} + for filename, line_stats in self.stats.items(): + if include: + match = any(trace_filter.match_filename(filename) + for trace_filter in filters) + else: + match = all(trace_filter.match_filename(filename) + for trace_filter in filters) + if not match: + continue + + new_line_stats = {} + for lineno, line_stat in line_stats.items(): + if include: + match = any(trace_filter.match(filename, lineno) + for trace_filter in filters) + else: + match = all(trace_filter.match(filename, lineno) + for trace_filter in filters) + if match: + new_line_stats[lineno] = line_stat + + file_stats[filename] = new_line_stats + return file_stats + + def _apply_filters(self, include, filters): + if not filters: + return + self.stats = self._filter_stats(include, filters) + if self.traces is not None: + self.traces = self._filter_traces(include, filters) + + def apply_filters(self, filters): + include_filters = [] + exclude_filters = [] + for trace_filter in filters: + if trace_filter.include: + include_filters.append(trace_filter) + else: + exclude_filters.append(trace_filter) + self._apply_filters(True, include_filters) + self._apply_filters(False, exclude_filters) + + def top_by(self, group_by, cumulative=False): + if cumulative and self.traceback_limit < 2: + cumulative = False + + stats = {} + if group_by == 'address': + cumulative = False + + if self.traces is None: + raise ValueError("need traces") + + for address, trace in self.traces.items(): + stats[address] = (trace[0], 1) + else: + if group_by == 'filename': + group_per_file = True + elif group_by == 'line': + group_per_file = False + else: + raise ValueError("unknown group_by value: %r" % (group_by,)) + + if not cumulative: + for filename, line_dict in self.stats.items(): + if not group_per_file: + for lineno, line_stats in line_dict.items(): + key = (filename, lineno) + stats[key] = line_stats + else: + key = filename + total_size = total_count = 0 + for size, count in line_dict.values(): + total_size += size + total_count += count + stats[key] = (total_size, total_count) + else: + if self.traces is None: + raise ValueError("need traces") + + for trace in self.traces.values(): + size, traceback = trace + if traceback: + for frame in traceback: + _compute_stats_frame(stats, group_per_file, size, frame) + else: + _compute_stats_frame(stats, group_per_file, size, None) + + return GroupedStats(self.timestamp, stats, group_by, + cumulative, self.metrics) + + +class TakeSnapshotTask(Task): + def __init__(self, filename_template="tracemalloc-$counter.pickle", + traces=False, metrics=True, + callback=None): + Task.__init__(self, self.take_snapshot) + self.filename_template = filename_template + self.traces = traces + self.metrics = metrics + self.callback = callback + self.counter = 1 + + def create_filename(self, snapshot): + filename = self.filename_template + filename = filename.replace("$pid", str(snapshot.pid)) + + timestamp = _format_timestamp(snapshot.timestamp) + timestamp = timestamp.replace(" ", "-") + filename = filename.replace("$timestamp", timestamp) + + filename = filename.replace("$counter", "%04i" % self.counter) + self.counter += 1 + return filename + + def take_snapshot(self): + snapshot = Snapshot.create(traces=self.traces, + metrics=self.metrics) + if self.callback is not None: + self.callback(snapshot) + + filename = self.create_filename(snapshot) + snapshot.write(filename) + return snapshot, filename + + +def main(): + from optparse import OptionParser + + parser = OptionParser(usage="%prog trace1.pickle [trace2.pickle trace3.pickle ...]") + parser.add_option("-a", "--address", + help="Group memory allocations by address, " + "instead of grouping by line number", + action="store_true", default=False) + parser.add_option("-f", "--file", + help="Group memory allocations per filename, " + "instead of grouping by line number", + action="store_true", default=False) + parser.add_option("-n", "--number", + help="Number of traces displayed per top (default: 10)", + type="int", action="store", default=10) + parser.add_option("--first", + help="Compare with the first trace, instead of with the previous trace", + action="store_true", default=False) + parser.add_option("-c", "--cumulative", + help="Cumulate size and count of memory blocks using " + "all frames, not only the most recent frame. The option has only " + "an effect if the snapshot contains traces and the traceback limit" + "was greater than 1", + action="store_true", default=False) + parser.add_option("-b", "--block", metavar="ADDRESS", + help="Get the memory block at address ADDRESS, display its size and " + "the traceback where it was allocated.", + action="store", type="int", default=None) + parser.add_option("-t", "--traceback", + help="Group memmory allocations by address and display the size and " + "the traceback of the NUMBER biggest allocated memory blocks", + action="store_true", default=False) + parser.add_option("-i", "--include", metavar="FILENAME[:LINENO]", + help="Only show memory block allocated in a file with a name matching " + "FILENAME pattern at line number LINENO. Ony check the most " + "recent frame. The option can be specified multiple times.", + action="append", type=str, default=[]) + parser.add_option("-I", "--include-traceback", metavar="FILENAME[:LINENO]", + help="Similar to --include, but check all frames of the traceback.", + action="append", type=str, default=[]) + parser.add_option("-x", "--exclude", metavar="FILENAME[:LINENO]", + help="Exclude filenames matching FILENAME pattern at line number " + "LINENO. Only check the most recent frame. The option can be " + "specified multiple times.", + action="append", type=str, default=[]) + parser.add_option("-X", "--exclude-traceback", metavar="FILENAME[:LINENO]", + help="Similar to --exclude, but check all frames of the traceback.", + action="append", type=str, default=[]) + parser.add_option("-S", "--hide-size", + help="Hide the size of allocations", + action="store_true", default=False) + parser.add_option("-C", "--hide-count", + help="Hide the number of allocations", + action="store_true", default=False) + parser.add_option("-A", "--hide-average", + help="Hide the average size of allocations", + action="store_true", default=False) + parser.add_option("-M", "--hide-metrics", + help="Hide metrics", + action="store_true", default=False) + parser.add_option("-P", "--filename-parts", + help="Number of displayed filename parts (default: 3)", + type="int", action="store", default=3) + parser.add_option("--color", + help="Always use colors", + action="store_true", default=False) + parser.add_option("--no-color", + help="Never use colors", + action="store_true", default=False) + + options, filenames = parser.parse_args() + if not filenames: + parser.print_help() + sys.exit(1) + + if options.address or options.traceback: + group_by = "address" + elif options.file: + group_by = "filename" + else: + group_by = "line" + + # use set() to delete duplicate filters + filters = set() + for include, values, traceback in ( + (True, options.include, False), + (True, options.include_traceback, True), + (False, options.exclude, False), + (False, options.exclude_traceback, True), + ): + for value in values: + if ':' in value: + pattern, lineno = value.rsplit(':', 1) + lineno = int(lineno) + else: + pattern = value + lineno = None + filters.add(Filter(include, pattern, lineno, traceback)) + + def log(message, *args): + if args: + message = message % args + sys.stderr.write(message + "\n") + sys.stderr.flush() + + snapshots = [] + for filename in filenames: + load_traces = (options.block or options.address + or options.traceback or options.cumulative) + + start = _time_monotonic() + if load_traces: + load_text = "Load snapshot %s" % filename + else: + load_text = "Load snapshot %s without traces" % filename + log(load_text) + try: + snapshot = Snapshot.load(filename, load_traces) + except Exception: + err = sys.exc_info()[1] + print("ERROR: Failed to load %s: [%s] %s" + % (filename, type(err).__name__, err)) + sys.exit(1) + + info = [] + if snapshot.stats is not None: + info.append('%s files' % len(snapshot.stats)) + if snapshot.traces is not None: + info.append('%s traces (limit=%s frames)' + % (len(snapshot.traces), snapshot.traceback_limit)) + dt = _time_monotonic() - start + log("Load snapshot %s: %s (%.1f sec)", + filename, ', '.join(info), dt) + + if options.block is not None or options.traceback: + if snapshot.traces is None: + print("ERROR: The snapshot %s does not contain traces, " + "only stats" % filename) + sys.exit(1) + + if filters: + start = _time_monotonic() + text = ("Apply %s filter%s on snapshot %s..." + % (len(filters), + 's' if not filters or len(filters) > 1 else '', + _format_timestamp(snapshot.timestamp))) + log(text) + snapshot.apply_filters(filters) + dt = _time_monotonic() - start + log(text + " done (%.1f sec)" % dt) + + snapshots.append(snapshot) + snapshots.sort(key=lambda snapshot: snapshot.timestamp) + + pids = set(snapshot.pid for snapshot in snapshots) + if len(pids) > 1: + pids = ', '.join(map(str, sorted(pids))) + print("WARNING: Traces generated by different processes: %s" % pids) + print("") + + stream = sys.stdout + if options.color: + color = True + elif options.no_color: + color = False + else: + color = stream.isatty() + + log("") + if options.block is not None: + address = options.block + + for snapshot in snapshots: + log("") + trace = snapshot.traces.get(address) + timestamp = _format_timestamp(snapshot.timestamp) + address = _format_address(address, color) + if color: + timestamp = _FORMAT_CYAN % timestamp + if trace is not None: + size = _format_size_color(trace[0], color) + else: + size = '(not found)' + if color: + size = _FORMAT_YELLOW % size + print("%s, memory block %s: %s" + % (timestamp, address, size)) + if trace is not None: + for line in _format_traceback(trace[1], options.filename_parts, color): + print(line) + + elif options.traceback: + for snapshot in snapshots: + log("Sort traces of snapshot %s", + _format_timestamp(snapshot.timestamp)) + traces = [(trace[0], address, trace[1]) + for address, trace in snapshot.traces.items()] + traces.sort(reverse=True) + displayed_traces = traces[:options.number] + + timestamp = _format_timestamp(snapshot.timestamp) + number = len(displayed_traces) + if color: + number = _FORMAT_BOLD % number + log("") + print("%s: Traceback of the top %s biggest memory blocks" + % (timestamp, number)) + print() + + for size, address, traceback in displayed_traces: + address = _format_address(address, color) + size = _format_size_color(size, color) + print("Memory block %s: %s" % (address, size)) + for line in _format_traceback(traceback, options.filename_parts, color): + print(line) + print() + + ignored = len(traces) - len(displayed_traces) + if ignored: + ignored_size = sum(size for size, address, traceback in traces[options.number:]) + size = _format_size_color(ignored_size, color) + print("%s more memory blocks: size=%s" % (ignored, size)) + + else: + top = DisplayTop() + top.filename_parts = options.filename_parts + top.average = not options.hide_average + top.count = not options.hide_count + top.size = not options.hide_size + top.metrics = not options.hide_metrics + top.compare_to_previous = not options.first + top.color = color + + for snapshot in snapshots: + log("Group stats by %s ...", group_by) + start = _time_monotonic() + top_stats = snapshot.top_by(group_by, options.cumulative) + dt = _time_monotonic() - start + if dt > 0.5: + log("Group stats by %s (%.1f sec)", group_by, dt) + top.display_top_stats(top_stats, count=options.number, file=stream) + + print("%s snapshots" % len(snapshots)) + + +if __name__ == "__main__": + if 0: + import cProfile + cProfile.run('main()', sort='tottime') + else: + main() + diff -r 50e4d0c62c2b -r ec121a72e848 Modules/Setup.dist --- a/Modules/Setup.dist Sun Oct 06 13:24:52 2013 +0200 +++ b/Modules/Setup.dist Sun Oct 06 17:53:23 2013 +0200 @@ -102,7 +102,7 @@ PYTHONPATH=$(COREPYTHONPATH) # various reasons; therefore they are listed here instead of in the # normal order. -# This only contains the minimal set of modules required to run the +# This only contains the minimal set of modules required to run the # setup.py script in the root of the Python source tree. posix posixmodule.c # posix (UNIX) system calls @@ -115,7 +115,7 @@ pwd pwdmodule.c # this is needed to fi _functools _functoolsmodule.c # Tools for working with functions and callable objects _operator _operator.c # operator.add() and similar goodies _collections _collectionsmodule.c # Container types -itertools itertoolsmodule.c # Functions creating iterators for efficient looping +itertools itertoolsmodule.c # Functions creating iterators for efficient looping atexit atexitmodule.c # Register functions to be run at interpreter-shutdown _stat _stat.c # stat.h interface @@ -132,12 +132,15 @@ zipimport zipimport.c # faulthandler module faulthandler faulthandler.c +# _tracemalloc module +_tracemalloc _tracemalloc.c + # The rest of the modules listed in this file are all commented out by # default. Usually they can be detected and built as dynamically # loaded modules by the new setup.py script added in Python 2.1. If -# you're on a platform that doesn't support dynamic loading, want to -# compile modules statically into the Python binary, or need to -# specify some odd set of compiler switches, you can uncomment the +# you're on a platform that doesn't support dynamic loading, want to +# compile modules statically into the Python binary, or need to +# specify some odd set of compiler switches, you can uncomment the # appropriate lines below. # ====================================================================== @@ -186,7 +189,7 @@ faulthandler faulthandler.c # supported...) #fcntl fcntlmodule.c # fcntl(2) and ioctl(2) -#spwd spwdmodule.c # spwd(3) +#spwd spwdmodule.c # spwd(3) #grp grpmodule.c # grp(3) #select selectmodule.c # select(2); not on ancient System V @@ -302,7 +305,7 @@ faulthandler faulthandler.c #_curses _cursesmodule.c -lcurses -ltermcap # Wrapper for the panel library that's part of ncurses and SYSV curses. -#_curses_panel _curses_panel.c -lpanel -lncurses +#_curses_panel _curses_panel.c -lpanel -lncurses # Modules that provide persistent dictionary-like semantics. You will diff -r 50e4d0c62c2b -r ec121a72e848 Modules/_tracemalloc.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Modules/_tracemalloc.c Sun Oct 06 17:53:23 2013 +0200 @@ -0,0 +1,4976 @@ +/* The implementation of the hash table (hash_t) is based on the cfuhash + project: + http://sourceforge.net/projects/libcfu/ + + Copyright of cfuhash: + ---------------------------------- + Creation date: 2005-06-24 21:22:40 + Authors: Don + Change log: + + Copyright (c) 2005 Don Owens + All rights reserved. + + This code is released under the BSD license: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + * Neither the name of the author nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + OF THE POSSIBILITY OF SUCH DAMAGE. + ---------------------------------- +*/ + +#include "Python.h" +#include "frameobject.h" +#include "pythread.h" +#include "osdefs.h" +#ifdef NT_THREADS +# include +#endif +#ifdef MS_WINDOWS +# include +#endif + +/* Page of a memory page */ +#define PAGE_SIZE 4096 +#define PAGE_SIZE_BITS 12 + +/* Trace also memory blocks allocated by PyMem_RawMalloc() */ +#define TRACE_RAW_MALLOC + +/* Use a memory pool to release the memory as soon as traces are deleted, + and to reduce the fragmentation of the heap */ +#define USE_MEMORY_POOL + +#if defined(HAVE_PTHREAD_ATFORK) && defined(WITH_THREAD) +# define TRACEMALLOC_ATFORK + +/* Forward declaration */ +static void tracemalloc_atfork(void); +#endif + +/* Forward declaration */ +static int tracemalloc_disable(void); +static void* raw_malloc(size_t size); +static void* raw_realloc(void *ptr, size_t size); +static void raw_free(void *ptr); +static void task_list_check(void); +static int task_list_clear(void); +#ifdef USE_MEMORY_POOL +static void* raw_alloc_arena(size_t size); +static void raw_free_arena(void *ptr, size_t size); +#endif + +#ifdef MS_WINDOWS +# define TRACE_CASE_INSENSITIVE +#endif + +#if defined(ALTSEP) || defined(TRACE_CASE_INSENSITIVE) +# define TRACE_NORMALIZE_FILENAME +#endif + +#ifdef TRACEMALLOC_ATFORK +# include +#endif + +#ifdef Py_DEBUG +# define TRACE_DEBUG +#endif + +#if !defined(PRINT_STATS) && (defined(DEBUG_POOL) || defined(DEBUG_HASH_TABLE)) +# define PRINT_STATS +#endif + +#define INT_TO_POINTER(value) ((void*)(Py_uintptr_t)(int)(value)) +#define POINTER_TO_INT(ptr) ((int)(Py_uintptr_t)(ptr)) + +/* number chosen to limit the number of compaction of a memory pool */ +#define MAX_ARENAS 32 + +/* Thresholds for the defragmentation of memory pool. + Factor chosen as a cpu/memory compromise */ +#define MAX_FRAG 0.50 + +/* Value chosen to limit the number of memory mappings per process + and not waste too much memory */ +#define ARENA_MIN_SIZE (128 * 1024) + +#define HASH_MIN_SIZE 32 +#define HASH_HIGH 0.50 +#define HASH_LOW 0.10 +#define HASH_REHASH_FACTOR 2.0 / (HASH_LOW + HASH_HIGH) + +typedef struct slist_item_s { + struct slist_item_s *next; +} slist_item_t; + +#define SLIST_ITEM_NEXT(ITEM) (((slist_item_t *)ITEM)->next) + +typedef struct { + slist_item_t *head; +} slist_t; + +#define SLIST_HEAD(SLIST) (((slist_t *)SLIST)->head) + +typedef struct { + /* used by hash_t.buckets to link entries */ + slist_item_t slist_item; + + const void *key; + Py_uhash_t key_hash; + + /* data folllows */ +} hash_entry_t; + +#define BUCKETS_HEAD(SLIST) ((hash_entry_t *)SLIST_HEAD(&(SLIST))) +#define TABLE_HEAD(HT, BUCKET) ((hash_entry_t *)SLIST_HEAD(&(HT)->buckets[BUCKET])) +#define ENTRY_NEXT(ENTRY) ((hash_entry_t *)SLIST_ITEM_NEXT(ENTRY)) + +#define ENTRY_DATA_PTR(ENTRY) \ + ((char *)(ENTRY) + sizeof(hash_entry_t)) +#define HASH_ENTRY_DATA_AS_VOID_P(ENTRY) \ + (*(void **)ENTRY_DATA_PTR(ENTRY)) + +#define HASH_ENTRY_READ_DATA(TABLE, DATA, DATA_SIZE, ENTRY) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + memcpy(DATA, ENTRY_DATA_PTR(ENTRY), DATA_SIZE); \ + } while (0) + +#define HASH_ENTRY_WRITE_DATA(TABLE, ENTRY, DATA, DATA_SIZE) \ + do { \ + assert((DATA_SIZE) == (TABLE)->data_size); \ + memcpy(ENTRY_DATA_PTR(ENTRY), DATA, DATA_SIZE); \ + } while (0) + +#define HASH_GET_DATA(TABLE, KEY, DATA) \ + hash_get_data(TABLE, KEY, &(DATA), sizeof(DATA)) + +#define HASH_PUT_DATA(TABLE, KEY, DATA) \ + hash_put_data(TABLE, KEY, &(DATA), sizeof(DATA)) + +typedef struct { + size_t length; + size_t used; + slist_t free_list; + char *items; +} arena_t; + +typedef struct { + size_t item_size; + size_t used; + size_t length; + unsigned int narena; + arena_t arenas[MAX_ARENAS]; + slist_t hash_tables; +#ifdef PRINT_STATS + const char *name; + int debug; +#endif +} pool_t; + +typedef struct { + size_t data; + size_t free; +} mem_stats_t; + +typedef Py_uhash_t (*key_hash_func) (const void *key); +typedef int (*key_compare_func) (const void *key, hash_entry_t *he); +typedef void* (*hash_copy_data_func)(void *data); +typedef void (*hash_free_data_func)(void *data); +typedef void (*hash_get_data_size_func)(void *data, mem_stats_t *stats); + +typedef struct { + /* used by pool_t.hash_tables to link tables */ + slist_item_t slist_item; + + size_t num_buckets; + size_t entries; /* Total number of entries in the table. */ + slist_t *buckets; + size_t data_size; + + pool_t *pool; + + key_hash_func hash_func; + key_compare_func compare_func; + hash_copy_data_func copy_data_func; + hash_free_data_func free_data_func; + hash_get_data_size_func get_data_size_func; +#ifdef PRINT_STATS + const char *name; + int debug; +#endif +} hash_t; + +/* Forward declaration */ +static void hash_rehash(hash_t *ht); +#ifdef USE_MEMORY_POOL +static int pool_defrag(pool_t *pool, size_t length); +#endif +#ifdef PRINT_STATS +static void pool_print_stats(pool_t *pool); +static void hash_print_stats(hash_t *ht); +#endif + +#ifdef TRACE_DEBUG +static void +tracemalloc_error(const char *format, ...) +{ + va_list ap; + fprintf(stderr, "tracemalloc: "); + va_start(ap, format); + vfprintf(stderr, format, ap); + va_end(ap); + fprintf(stderr, "\n"); + fflush(stderr); +} +#endif + +static void +slist_init(slist_t *list) +{ + list->head = NULL; +} + +static void +slist_prepend(slist_t *list, slist_item_t *item) +{ + item->next = list->head; + list->head = item; +} + +#ifdef USE_MEMORY_POOL +static slist_item_t * +slist_pop(slist_t *list) +{ + slist_item_t *item; + item = list->head; + assert(item != NULL); + list->head = item->next; + return item; +} +#endif + +static void +_slist_remove(slist_t *list, slist_item_t *previous, slist_item_t *item) +{ + if (previous != NULL) + previous->next = item->next; + else + list->head = item->next; +} + +/* This function has a complexity of O(n): don't use it on large lists */ +static void +slist_remove(slist_t *list, slist_item_t *item) +{ + slist_item_t *previous, *it; + + previous = NULL; + for (it = list->head; it != NULL; it = it->next) { + if (it == item) + break; + previous = it; + } + /* not found? */ + assert(it != NULL); + _slist_remove(list, previous, item); +} + +static Py_uhash_t +key_hash_int(const void *key) +{ + return (Py_uhash_t)POINTER_TO_INT(key); +} + +static Py_uhash_t +key_hash_ptr(const void *key) +{ + return (Py_uhash_t)_Py_HashPointer((void *)key); +} + +static Py_uhash_t +key_hash_arena_ptr(const void *key) +{ + Py_hash_t x; + size_t y = (size_t)key; + /* bottom 12 bits are likely to be 0 (arena pointers aligned on a page + size, usually 4096 bytes); rotate y by 12 to avoid excessive hash + collisions for dicts and sets */ + y = (y >> PAGE_SIZE_BITS) | (y << (8 * SIZEOF_VOID_P - PAGE_SIZE_BITS)); + x = (Py_hash_t)y; + if (x == -1) + x = -2; + return x; +} + +static Py_uhash_t +filename_hash(PyObject *filename) +{ + if (filename != NULL) + return PyObject_Hash(filename); + else + return 0; +} + +static int +key_cmp_unicode(const void *key, hash_entry_t *he) +{ + if (key != NULL && he->key != NULL) + return (PyUnicode_Compare((PyObject *)key, (PyObject *)he->key) == 0); + else + return key == he->key; +} + +static int +key_cmp_direct(const void *key, hash_entry_t *he) +{ + return he->key == key; +} + +/* makes sure the real size of the buckets array is a power of 2 */ +static size_t +round_size(size_t s) +{ + size_t i; + if (s < HASH_MIN_SIZE) + return HASH_MIN_SIZE; + i = 1; + while (i < s) + i <<= 1; + return i; +} + +static void +pool_init(pool_t *pool, size_t item_size) +{ + assert(item_size >= sizeof(slist_item_t)); + pool->item_size = item_size; + pool->narena = 0; + pool->used = 0; + pool->length = 0; + slist_init(&pool->hash_tables); +#ifdef PRINT_STATS + pool->name = NULL; + pool->debug = 1; +#endif +} + +#ifdef USE_MEMORY_POOL +static int +arena_init(pool_t *pool, arena_t *arena, size_t length) +{ + size_t pool_size; + char *ptr, *end; + slist_item_t *previous, *item; + + assert(pool->item_size < PY_SIZE_MAX / length); + pool_size = pool->item_size * length; + + arena->length = length; + arena->used = 0; + arena->items = raw_alloc_arena(pool_size); + if (arena->items == NULL) + return -1; + + ptr = arena->items; + item = (slist_item_t *)ptr; + end = ptr + pool_size; + + arena->free_list.head = item; + + previous = item; + ptr += pool->item_size; + while (ptr < end) { + item = (slist_item_t *)ptr; + previous->next = item; + + previous = item; + ptr += pool->item_size; + } + previous->next = NULL; + + return 0; +} + +static void +arena_free(pool_t *pool, arena_t *arena) +{ + size_t pool_size = pool->item_size * arena->length; + assert(arena->used == 0); + raw_free_arena(arena->items, pool_size); +} + +static void +pool_copy(pool_t *copy, pool_t *pool) +{ + pool_init(copy, pool->item_size); +#ifdef PRINT_STATS + copy->name = pool->name; + copy->debug = pool->debug; +#endif +} + +static int +pool_alloc_arena(pool_t *pool, size_t arena_length) +{ + assert(pool->narena < MAX_ARENAS); + + if (arena_init(pool, &pool->arenas[pool->narena], arena_length) < 0) + return -1; + + pool->length += arena_length; + pool->narena++; + return 0; +} + +static hash_entry_t* +arena_alloc_entry(pool_t *pool, arena_t *arena) +{ + if (arena->free_list.head == NULL) { + assert(arena->used == arena->length); + return NULL; + } + + arena->used++; + pool->used++; + return (hash_entry_t *)slist_pop(&arena->free_list); +} + +static int +pool_alloc(pool_t *pool, size_t items) +{ + size_t free, arena_length, min_length, new_length; + const double factor = 1.0 - (MAX_FRAG / 2.0); + unsigned int narena; + double new_frag; + + free = pool->length - pool->used; + if (items <= free) + return 0; + + if (pool->narena == MAX_ARENAS) { + /* too much arenas: compact into one unique arena */ + return pool_defrag(pool, items); + } + + arena_length = items - free; + + min_length = pool->used + items; + min_length -= (size_t)(pool->length * factor); + min_length = (size_t)((double)min_length / factor); + if (arena_length < min_length) + arena_length = min_length; + + assert(pool->narena < MAX_ARENAS); + narena = (pool->narena + 1); + + min_length = ARENA_MIN_SIZE / pool->item_size; + if (arena_length < min_length) + arena_length = min_length; + + /* don't defrag small tables */ + if (pool->narena > 0) { + new_length = pool->length + arena_length; + if (new_length / narena > ARENA_MIN_SIZE / pool->item_size + /* two checks to avoid an integer overflow */ + && (new_length * pool->item_size / narena) > ARENA_MIN_SIZE) + { + free = new_length - pool->used - items; + new_frag = (double)free / new_length; + if (new_frag > MAX_FRAG) { + /* allocating a new arena would increase the fragmentation */ + return pool_defrag(pool, arena_length); + } + } + } + + return pool_alloc_arena(pool, arena_length); +} + +static hash_entry_t* +pool_alloc_entry(pool_t *pool) +{ + int i; + hash_entry_t* entry; + + for (i=pool->narena-1; i >= 0; i--) { + entry = arena_alloc_entry(pool, &pool->arenas[i]); + if (entry != NULL) + return entry; + } + return NULL; +} + +static int +arena_free_entry(pool_t *pool, arena_t *arena, hash_entry_t *entry) +{ + size_t pool_size; + char *start, *end; + + pool_size = pool->item_size * arena->length; + start = arena->items; + end = start + pool_size; + + if ((char*)entry < start) { + return 0; + } + if ((char*)entry >= end) + return 0; + + assert(arena->used != 0); + arena->used--; + pool->used--; + slist_prepend(&arena->free_list, (slist_item_t *)entry); + return 1; +} + +static void +pool_remove_arena(pool_t *pool, unsigned int index) +{ + arena_t *arena; + size_t narena; + + /* don't delete the last arena */ + if (pool->narena == 1) + return; + + /* the arena should not be removed if all other arenas are full */ + if (pool->used == pool->length) + return; + + assert(index < pool->narena); + arena = &pool->arenas[index]; + assert(arena->used == 0); + + assert(pool->length >= arena->length); + pool->length -= arena->length; + + arena_free(pool, arena); + narena = pool->narena - index; + if (narena != 1) { + memmove(&pool->arenas[index], + &pool->arenas[index + 1], + (narena - 1) * sizeof(arena_t)); + } + assert(pool->narena >= 2); + pool->narena--; + +#ifdef DEBUG_POOL + if (pool->debug) { + printf("Pool %p remove arena: [%u arenas] %zu/%zu, frag=%.0f%%\n", + pool, pool->narena, + pool->used, pool->length, + (pool->length - pool->used) * 100.0 / pool->length); + } +#endif +} + +static void +pool_clear(pool_t *pool) +{ + unsigned int i; + assert(pool->used == 0); + if (pool->narena != 0) { + for (i=0; i < pool->narena; i++) + arena_free(pool, &pool->arenas[i]); + pool->narena = 0; + pool->length = 0; + } + else { + assert(pool->narena == 0); + assert(pool->length == 0); + } +} +#endif + +static hash_entry_t* +hash_alloc_entry(hash_t *ht) +{ +#ifdef USE_MEMORY_POOL + hash_entry_t* entry; + pool_t *pool = ht->pool; + + entry = pool_alloc_entry(pool); + if (entry != NULL) + return entry; + + assert(pool->length == pool->used); + + if (pool_alloc(pool, 1) < 0) + return NULL; + + entry = arena_alloc_entry(pool, &pool->arenas[pool->narena-1]); + assert(entry != NULL); +#ifdef DEBUG_POOL + printf("[Allocate 1 entry] "); + pool_print_stats(pool); +#endif + return entry; +#else + return malloc(ht->pool->item_size); +#endif +} + +static void +pool_free_entry(pool_t *pool, hash_entry_t *entry) +{ +#ifdef USE_MEMORY_POOL + unsigned int i; + arena_t *arena; + + for (i=0; i < pool->narena; i++) { + arena = &pool->arenas[i]; + if (!arena_free_entry(pool, arena, entry)) { + /* entry is not part of the arena */ + continue; + } + if (arena->used == 0) + pool_remove_arena(pool, i); + return; + } + /* at least one arena should be non empty */ + assert(0); +#else + free(entry); +#endif +} + +static hash_t * +hash_new_full(pool_t *pool, size_t data_size, size_t size, + key_hash_func hash_func, + key_compare_func compare_func, + hash_copy_data_func copy_data_func, + hash_free_data_func free_data_func, + hash_get_data_size_func get_data_size_func) +{ + hash_t *ht; + size_t buckets_size; + + assert(pool->item_size == sizeof(hash_entry_t) + data_size); + + ht = (hash_t *)raw_malloc(sizeof(hash_t)); + if (ht == NULL) + return ht; + + ht->num_buckets = round_size(size); + ht->entries = 0; + ht->data_size = data_size; + ht->pool = pool; + + buckets_size = ht->num_buckets * sizeof(ht->buckets[0]); + ht->buckets = raw_malloc(buckets_size); + if (ht->buckets == NULL) { + raw_free(ht); + return NULL; + } + memset(ht->buckets, 0, buckets_size); + + ht->hash_func = hash_func; + ht->compare_func = compare_func; + ht->copy_data_func = copy_data_func; + ht->free_data_func = free_data_func; + ht->get_data_size_func = get_data_size_func; +#ifdef PRINT_STATS + ht->name = NULL; + ht->debug = 1; +#endif + + slist_prepend(&ht->pool->hash_tables, (slist_item_t *)ht); + return ht; +} + +static hash_t * +hash_new(pool_t *pool, size_t data_size, + key_hash_func hash_func, + key_compare_func compare_func) +{ + return hash_new_full(pool, data_size, HASH_MIN_SIZE, + hash_func, compare_func, + NULL, NULL, NULL); +} + +static void +pool_mem_stats(pool_t *pool, mem_stats_t *stats) +{ + stats->data += sizeof(pool_t); + stats->data += pool->narena * sizeof(arena_t); + stats->data += pool->item_size * pool->used; + stats->free += pool->item_size * (pool->length - pool->used); +} + +/* Don't count memory allocated in the memory pool, only memory directly + allocated by the hash table: see pool_mem_stats() */ +static void +hash_mem_stats(hash_t *ht, mem_stats_t *stats) +{ + hash_entry_t *entry; + size_t hv; + + stats->data += sizeof(hash_t); + + /* buckets */ + stats->data += ht->num_buckets * sizeof(hash_entry_t *); + +#ifndef USE_MEMORY_POOL + stats->data += ht->entries * sizeof(hash_entry_t); + stats->data += ht->entries * ht->data_size; +#endif + + /* data linked from entries */ + if (ht->get_data_size_func) { + for (hv = 0; hv < ht->num_buckets; hv++) { + for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { + void *data = HASH_ENTRY_DATA_AS_VOID_P(entry); + ht->get_data_size_func(data, stats); + } + } + } +} + +#ifdef PRINT_STATS +static void +pool_print_stats(pool_t *pool) +{ + size_t size; + mem_stats_t stats; + double usage, fragmentation; + + memset(&stats, 0, sizeof(stats)); + pool_mem_stats(pool, &stats); + if (pool->length != 0) + usage = pool->used * 100.0 / pool->length; + else + usage = 0.0; + size = stats.data + stats.free; + if (size) + fragmentation = stats.free * 100.0 / size; + else + fragmentation = 0.0; + printf("pool %s (%p): %u arenas, " + "%zu/%zu items (%.0f%%), " + "%zu/%zu kB (frag=%.1f%%)\n", + pool->name, pool, pool->narena, + pool->used, pool->length, usage, + stats.data / 1024, size / 1024, fragmentation); +} + +static void +hash_print_stats(hash_t *ht) +{ + mem_stats_t stats; + size_t chain_len, max_chain_len, total_chain_len, nchains; + hash_entry_t *entry; + size_t hv; + double load; + + memset(&stats, 0, sizeof(stats)); + hash_mem_stats(ht, &stats); + + load = (double)ht->entries / ht->num_buckets; + + max_chain_len = 0; + total_chain_len = 0; + nchains = 0; + for (hv = 0; hv < ht->num_buckets; hv++) { + entry = TABLE_HEAD(ht, hv); + if (entry != NULL) { + chain_len = 0; + for (; entry; entry = ENTRY_NEXT(entry)) { + chain_len++; + } + if (chain_len > max_chain_len) + max_chain_len = chain_len; + total_chain_len += chain_len; + nchains++; + } + } + printf("hash table %s (%p): entries=%zu/%zu (%.0f%%), ", + ht->name, ht, ht->entries, ht->num_buckets, load * 100.0); + if (nchains) + printf("avg_chain_len=%.1f, ", (double)total_chain_len / nchains); + printf("max_chain_len=%zu, " + "%zu/%zu kB\n", + max_chain_len, + stats.data / 1024, (stats.data + stats.free) / 1024); +} +#endif + +/* + Returns one if the entry was found, zero otherwise. If found, r is + changed to point to the data in the entry. +*/ +static hash_entry_t * +hash_get_entry(hash_t *ht, const void *key) +{ + Py_uhash_t key_hash; + size_t index; + hash_entry_t *entry; + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { + if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + break; + } + + return entry; +} + +static int +_hash_pop_entry(hash_t *ht, const void *key, void *data, size_t data_size) +{ + Py_uhash_t key_hash; + size_t index; + hash_entry_t *entry, *previous; + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + previous = NULL; + for (entry = TABLE_HEAD(ht, index); entry != NULL; entry = ENTRY_NEXT(entry)) { + if (entry->key_hash == key_hash && ht->compare_func(key, entry)) + break; + previous = entry; + } + + if (entry == NULL) + return 0; + + _slist_remove(&ht->buckets[index], (slist_item_t*)previous, (slist_item_t*)entry); + ht->entries--; + + if (data != NULL) + HASH_ENTRY_READ_DATA(ht, data, data_size, entry); + pool_free_entry(ht->pool, entry); + + if ((float)ht->entries / (float)ht->num_buckets < HASH_LOW) + hash_rehash(ht); +#ifdef USE_MEMORY_POOL + pool_defrag(ht->pool, 0); +#endif + return 1; +} + +/* Add a new entry to the hash. Return 0 on success, -1 on memory error. */ +static int +hash_put_data(hash_t *ht, const void *key, void *data, size_t data_size) +{ + Py_uhash_t key_hash; + size_t index; + hash_entry_t *entry; + + assert(data != NULL || data_size == 0); +#ifndef NDEBUG + entry = hash_get_entry(ht, key); + assert(entry == NULL); +#endif + + key_hash = ht->hash_func(key); + index = key_hash & (ht->num_buckets - 1); + + entry = hash_alloc_entry(ht); + if (entry == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("memory allocation failed in hash_put_data()"); +#endif + return -1; + } + + entry->key = (void *)key; + entry->key_hash = key_hash; + HASH_ENTRY_WRITE_DATA(ht, entry, data, data_size); + + slist_prepend(&ht->buckets[index], (slist_item_t*)entry); + ht->entries++; + + if ((float)ht->entries / (float)ht->num_buckets > HASH_HIGH) + hash_rehash(ht); + return 0; +} + +/* Get data from an entry. Copy entry data into data and return 1 if the entry + exists, return 0 if the entry does not exist. */ +static int +hash_get_data(hash_t *ht, const void *key, void *data, size_t data_size) +{ + hash_entry_t *entry; + + assert(data != NULL); + + entry = hash_get_entry(ht, key); + if (entry == NULL) + return 0; + HASH_ENTRY_READ_DATA(ht, data, data_size, entry); + return 1; +} + +static int +hash_pop_data(hash_t *ht, const void *key, void *data, size_t data_size) +{ + assert(data != NULL); + assert(ht->free_data_func == NULL); + return _hash_pop_entry(ht, key, data, data_size); +} + +/* Try to delete an entry. Return 1 if the entry is deleted, 0 if the + entry was not found. */ +static int +hash_may_delete_data(hash_t *ht, const void *key) +{ + return _hash_pop_entry(ht, key, NULL, 0); +} + +static void +hash_delete_data(hash_t *ht, const void *key) +{ +#ifndef NDEBUG + int found = hash_may_delete_data(ht, key); + assert(found); +#else + (void)hash_may_delete_data(ht, key); +#endif +} + +/* Prototype for a pointer to a function to be called foreach + key/value pair in the hash by hash_foreach(). Iteration + stops if a non-zero value is returned. */ +static int +hash_foreach(hash_t *ht, + int (*fe_fn) (hash_entry_t *entry, void *arg), + void *arg) +{ + hash_entry_t *entry; + size_t hv; + + for (hv = 0; hv < ht->num_buckets; hv++) { + for (entry = TABLE_HEAD(ht, hv); entry; entry = ENTRY_NEXT(entry)) { + int res = fe_fn(entry, arg); + if (res) + return res; + } + } + return 0; +} + +static void +hash_rehash(hash_t *ht) +{ + size_t buckets_size, new_size, bucket; + slist_t *old_buckets = NULL; + size_t old_num_buckets; + + new_size = round_size(ht->entries * HASH_REHASH_FACTOR); + if (new_size == ht->num_buckets) + return; + +#ifdef DEBUG_HASH_TABLE + if (ht->debug) { + printf("[before rehash] "); + hash_print_stats(ht); + } +#endif + + old_num_buckets = ht->num_buckets; + + buckets_size = new_size * sizeof(ht->buckets[0]); + old_buckets = ht->buckets; + ht->buckets = raw_malloc(buckets_size); + if (ht->buckets == NULL) { + /* cancel rehash on memory allocation failure */ + ht->buckets = old_buckets ; +#ifdef TRACE_DEBUG + tracemalloc_error("memory allocation failed in hash_rehash()"); +#endif + return; + } + memset(ht->buckets, 0, buckets_size); + + ht->num_buckets = new_size; + + for (bucket = 0; bucket < old_num_buckets; bucket++) { + hash_entry_t *entry, *next; + for (entry = BUCKETS_HEAD(old_buckets[bucket]); entry != NULL; entry = next) { + size_t entry_index; + + assert(ht->hash_func(entry->key) == entry->key_hash); + next = ENTRY_NEXT(entry); + entry_index = entry->key_hash & (new_size - 1); + + slist_prepend(&ht->buckets[entry_index], (slist_item_t*)entry); + } + } + + raw_free(old_buckets); + +#ifdef DEBUG_HASH_TABLE + if (ht->debug) { + printf("[after rehash] "); + hash_print_stats(ht); + } +#endif +} + +#ifdef USE_MEMORY_POOL +static void +hash_move_pool(hash_t *ht, pool_t *old_pool, pool_t *new_pool) +{ + size_t i; + + for (i = 0; i < ht->num_buckets; i++) { + hash_entry_t *entry, *next, *new_entry; + + entry = TABLE_HEAD(ht, i); + slist_init(&ht->buckets[i]); + for (; entry != NULL; entry = next) { + next = ENTRY_NEXT(entry); + + new_entry = pool_alloc_entry(new_pool); + assert(new_entry != NULL); + + memcpy(new_entry, entry, ht->pool->item_size); + pool_free_entry(old_pool, entry); + + slist_prepend(&ht->buckets[i], (slist_item_t*)new_entry); + } + } +} + +static int +pool_defrag(pool_t *pool, size_t length) +{ + pool_t new_pool; + slist_item_t *it, *next; + size_t prealloc; + + if (length == 0) { + /* an item has been removed */ + size_t free; + + /* don't defrag small tables */ + if (pool->length / pool->narena <= ARENA_MIN_SIZE / pool->item_size + /* two checks to avoid an integer overflow */ + && (pool->length * pool->item_size / pool->narena) <= ARENA_MIN_SIZE) + return 0; + + /* fragmentation lower than the threshold? */ + free = pool->length - pool->used; + if ((double)free <= MAX_FRAG * pool->length) + return 0; + } + +#ifdef DEBUG_POOL + if (pool->debug) { + printf("[before defrag, length=%zu] ", length); + pool_print_stats(pool); + } +#endif + + pool_copy(&new_pool, pool); + prealloc = pool->used + length; + if (pool_alloc(&new_pool, prealloc) < 0) { +#ifdef TRACE_DEBUG + tracemalloc_error("memory allocation failed in pool_defrag()"); +#endif + return -1; + } + + /* move entries of the tables to the new pool */ + for (it=pool->hash_tables.head; it != NULL; it = next) { + next = it->next; + hash_move_pool((hash_t *)it, pool, &new_pool); + slist_prepend(&new_pool.hash_tables, it); + } + + slist_init(&pool->hash_tables); + pool_clear(pool); + *pool = new_pool; + +#ifdef DEBUG_POOL + if (pool->debug) { + printf("[after defrag, length=%zu] ", length); + pool_print_stats(pool); + } +#endif + return 0; +} +#endif + +static void +hash_clear(hash_t *ht) +{ + hash_entry_t *entry, *next; + size_t i; + + for (i=0; i < ht->num_buckets; i++) { + for (entry = TABLE_HEAD(ht, i); entry != NULL; entry = next) { + next = ENTRY_NEXT(entry); + if (ht->free_data_func) + ht->free_data_func(HASH_ENTRY_DATA_AS_VOID_P(entry)); + pool_free_entry(ht->pool, entry); + } + slist_init(&ht->buckets[i]); + } + ht->entries = 0; + hash_rehash(ht); +#ifdef USE_MEMORY_POOL + pool_defrag(ht->pool, 0); +#endif +} + +static void +hash_destroy(hash_t *ht) +{ + size_t i; + + for (i = 0; i < ht->num_buckets; i++) { + slist_item_t *entry = ht->buckets[i].head; + while (entry) { + slist_item_t *entry_next = entry->next; + if (ht->free_data_func) + ht->free_data_func(HASH_ENTRY_DATA_AS_VOID_P(entry)); + pool_free_entry(ht->pool, (hash_entry_t *)entry); + entry = entry_next; + } + } + + slist_remove(&ht->pool->hash_tables, (slist_item_t *)ht); + + raw_free(ht->buckets); + raw_free(ht); +} + +static hash_t * +hash_copy_with_pool(hash_t *src, pool_t *pool) +{ + hash_t *dst; + hash_entry_t *entry; + size_t bucket; + int err; + void *data, *new_data; + +#ifdef USE_MEMORY_POOL + if (pool_alloc(pool, src->entries) < 0) + return NULL; +#endif + + dst = hash_new_full(pool, src->data_size, src->num_buckets, + src->hash_func, src->compare_func, + src->copy_data_func, src->free_data_func, + src->get_data_size_func); + if (dst == NULL) + return NULL; + + for (bucket=0; bucket < src->num_buckets; bucket++) { + entry = TABLE_HEAD(src, bucket); + for (; entry; entry = ENTRY_NEXT(entry)) { + if (src->copy_data_func) { + data = HASH_ENTRY_DATA_AS_VOID_P(entry); + new_data = src->copy_data_func(data); + if (new_data != NULL) + err = hash_put_data(dst, entry->key, + &new_data, src->data_size); + else + err = 1; + } + else { + data = ENTRY_DATA_PTR(entry); + err = hash_put_data(dst, entry->key, data, src->data_size); + } + if (err) { + hash_destroy(dst); + return NULL; + } + } + } +#ifdef DEBUG_POOL + printf("[copy hash] "); + pool_print_stats(dst->pool); +#endif + return dst; +} + +/* Return a copy of the hash table */ +static hash_t * +hash_copy(hash_t *src) +{ + return hash_copy_with_pool(src, src->pool); +} + +/* Arbitrary limit of the number of frames in a traceback. The value was chosen + to not allocate too much memory on the stack (see TRACEBACK_STACK_SIZE + below). */ +#define MAX_NFRAME 100 + +/* arbitrary limit of the depth of the recursive + match_filename_joker() function to avoid a stack overflow */ +#define MAX_NJOKER 100 + +/* Protected by the GIL */ +static struct { + PyMemAllocator mem; + PyMemAllocator raw; + PyMemAllocator obj; +} allocators; + +/* Protected by the GIL */ +static PyObjectArenaAllocator arena_allocator; + +static struct { + /* tracemalloc_init() was already called? + Variable protected by the GIL */ + int init; + + /* is the module enabled? + Variable protected by the GIL */ + int enabled; + + /* limit of the number of frames in a traceback, 1 by default. + Variable protected by the GIL. */ + int max_nframe; +} tracemalloc_config = {0, 0, 1}; + +#if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) +#define REENTRANT_THREADLOCAL + +#ifdef NT_THREADS +static DWORD tracemalloc_reentrant_key = TLS_OUT_OF_INDEXES; + +static int +get_reentrant(void) +{ + LPVOID ptr; + int value; + + ptr = TlsGetValue(tracemalloc_reentrant_key); + if (ptr == NULL) { + assert(GetLastError() == ERROR_SUCCESS); + return 0; + } + + value = POINTER_TO_INT(ptr); + return (value == 1); +} + +static void +set_reentrant_tls(int reentrant) +{ + LPVOID ptr; +#ifndef NDEBUG + BOOL res; +#endif + + ptr = INT_TO_POINTER(reentrant); +#ifndef NDEBUG + res = TlsSetValue(tracemalloc_reentrant_key, ptr); + assert(res != 0); +#else + (void)TlsSetValue(tracemalloc_reentrant_key, ptr); +#endif +} + +#elif defined(_POSIX_THREADS) +static pthread_key_t tracemalloc_reentrant_key; + +static int +get_reentrant(void) +{ + void *ptr; + int value; + ptr = pthread_getspecific(tracemalloc_reentrant_key); + if (ptr == NULL) + return 0; + value = POINTER_TO_INT(ptr); + return (value == 1); +} + +static void +set_reentrant_tls(int reentrant) +{ +#ifndef NDEBUG + int res; +#endif + void *ptr; + + ptr = INT_TO_POINTER(reentrant); +#ifndef NDEBUG + res = pthread_setspecific(tracemalloc_reentrant_key, ptr); + assert(res == 0); +#else + (void)pthread_setspecific(tracemalloc_reentrant_key, ptr); +#endif +} +#else +# error "unsupported thread model" +#endif + +#else + +/* WITH_THREAD not defined: Python compiled without threads, + or TRACE_RAW_MALLOC not defined: variable protected by the GIL */ +static int tracemalloc_reentrant = 0; + +#define get_reentrant() tracemalloc_reentrant +#define set_reentrant_tls(value) do { tracemalloc_reentrant = value; } while (0) + +#endif + +static void +set_reentrant(int reentrant) +{ + assert(!reentrant || !get_reentrant()); + set_reentrant_tls(reentrant); +} + +typedef struct { + /* include (1) or exclude (0) matching frame? */ + int include; + Py_hash_t pattern_hash; + PyObject *pattern; +#ifndef TRACE_NORMALIZE_FILENAME + int use_joker; +#endif + /* ignore any line number if lineno < 1 */ + int lineno; + /* use the whole traceback, or only the most recent frame? */ + int traceback; +} filter_t; + +typedef struct { + size_t nfilter; + filter_t *filters; +} filter_list_t; + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) +/* This lock is needed because tracemalloc_raw_free() is called without + the GIL held. It cannot acquire the lock because it would introduce + a deadlock in PyThreadState_DeleteCurrent(). */ +static PyThread_type_lock tables_lock; +# define TABLES_LOCK() PyThread_acquire_lock(tables_lock, 1) +# define TABLES_UNLOCK() PyThread_release_lock(tables_lock) +#else + /* variables are protected by the GIL */ +# define TABLES_LOCK() +# define TABLES_UNLOCK() +#endif + +#ifdef WITH_THREAD +#endif + +/* Protected by the GIL */ +static filter_list_t tracemalloc_include_filters; +static filter_list_t tracemalloc_exclude_filters; + +#pragma pack(4) +typedef struct +#ifdef __GNUC__ +__attribute__((packed)) +#endif +{ + PyObject *filename; + int lineno; +} frame_t; + +typedef struct { + Py_uhash_t hash; + int nframe; + frame_t frames[1]; +} traceback_t; + +static traceback_t tracemalloc_empty_traceback; + +typedef struct { + size_t size; + traceback_t *traceback; +} trace_t; + +#define TRACEBACK_SIZE(NFRAME) \ + (sizeof(traceback_t) + sizeof(frame_t) * (NFRAME - 1)) +#define TRACEBACK_STACK_SIZE TRACEBACK_SIZE(MAX_NFRAME) + +typedef struct { + int enabled; + int failed; + + Py_ssize_t ncall; + + int delay; + time_t timeout; + + Py_ssize_t memory_threshold; + size_t min_traced; + size_t max_traced; + + PyObject *func; + PyObject *func_args; + PyObject *func_kwargs; +} task_t; + +/* List of tasks (PyListObject*). + Protected by the GIL */ +static PyObject* tracemalloc_tasks; + +typedef struct { + size_t size; + size_t count; +} trace_stats_t; + +/* Size of Currently traced memory. + Protected by TABLES_LOCK(). */ +static size_t tracemalloc_traced_memory = 0; + +/* Maximum size of traced memory. + Protected by TABLES_LOCK(). */ +static size_t tracemalloc_max_traced_memory = 0; + +/* Total size of the arenas memory. + Protected by the GIL. */ +static size_t tracemalloc_arena_size = 0; + +static struct { + /* tracemalloc_filenames, tracemalloc_tracebacks. + Protected by the GIL. */ + pool_t no_data; + + /* tracemalloc_traces and line_hash of tracemalloc_file_stats. + Protected by TABLES_LOCK(). */ + pool_t traces; + + /* tracemalloc_file_stats: "hash_t*" type. + Protected by TABLES_LOCK(). */ + pool_t hash_tables; +} tracemalloc_pools; + +/* Hash table used to intern filenames. + Protected by the GIL */ +static hash_t *tracemalloc_filenames = NULL; + +/* Hash table used to intern tracebacks. + Protected by the GIL */ +static hash_t *tracemalloc_tracebacks = NULL; + +/* Statistics on Python memory allocations per file and per line: + {filename: PyObject* => {lineno: int => stat: trace_stats_t}. + Protected by TABLES_LOCK(). */ +static hash_t *tracemalloc_file_stats = NULL; + +/* pointer (void*) => trace (trace_t). + Protected by TABLES_LOCK(). */ +static hash_t *tracemalloc_traces = NULL; + +/* Set of arena pointers, see tracemalloc_alloc_arena(). + Protected by the GIL */ +static hash_t *tracemalloc_arenas = NULL; + +/* Forward declaration */ +static int traceback_match_filters(traceback_t *traceback); + +static void* +raw_malloc(size_t size) +{ + return allocators.raw.malloc(allocators.raw.ctx, size); +} + +static void* +raw_realloc(void *ptr, size_t size) +{ + return allocators.raw.realloc(allocators.raw.ctx, ptr, size); +} + +static void +raw_free(void *ptr) +{ + allocators.raw.free(allocators.raw.ctx, ptr); +} + +#ifdef USE_MEMORY_POOL +static void* +raw_alloc_arena(size_t size) +{ + return arena_allocator.alloc(arena_allocator.ctx, size); +} + +static void +raw_free_arena(void *ptr, size_t size) +{ + arena_allocator.free(arena_allocator.ctx, ptr, size); +} +#endif + + +static void +task_init(task_t *task) +{ + memset(task, 0, sizeof(task_t)); + task->memory_threshold = -1; + task->delay = -1; +} + +static void +task_schedule(task_t *task) +{ + size_t traced; + + assert(!task->enabled); + assert(task->ncall != 0); + + if (task->delay > 0) { + task->timeout = time(NULL) + task->delay; + } + if (task->memory_threshold > 0) { + TABLES_LOCK(); + traced = tracemalloc_traced_memory; + TABLES_UNLOCK(); + + if (traced >= task->memory_threshold) + task->min_traced = traced - task->memory_threshold; + else + task->min_traced = 0; + if (traced <= PY_SSIZE_T_MAX - task->memory_threshold) + task->max_traced = traced + task->memory_threshold; + else + task->max_traced = PY_SSIZE_T_MAX; + } + task->failed = 0; + task->enabled = 1; +} + +static void +task_reschedule(task_t *task) +{ + if (task->failed) + return; + assert(task->enabled); + task->enabled = 0; + task_schedule(task); +} + +static void +task_cancel(task_t *task) +{ + task->enabled = 0; +} + +static PyObject* +task_call(task_t *task) +{ + return PyEval_CallObjectWithKeywords(task->func, + task->func_args, + task->func_kwargs); +} + +static int +task_call_pending(void *user_data) +{ + task_t *task = user_data; + PyObject *res; + + assert(task->ncall != 0); + if (task->ncall > 0) + task->ncall--; + + res = task_call(task); + if (res == NULL) { + /* on error, don't reschedule the task */ + task->failed = 1; + PyErr_WriteUnraisable(NULL); + return 0; + } + Py_DECREF(res); + + if (task->ncall != 0) + task_schedule(task); + + return 0; +} + +static void +task_call_later(task_t *task) +{ + int res; + + task->enabled = 0; + res = Py_AddPendingCall(task_call_pending, task); + if (res != 0) { + /* failed to add the pending call: retry later */ + task->enabled = 1; + return; + } +} + +static void +task_check(task_t *task) +{ + if (task->memory_threshold > 0 + && (tracemalloc_traced_memory <= task->min_traced + || tracemalloc_traced_memory >= task->max_traced)) + { + task_call_later(task); + return; + } + + if (task->delay > 0 && time(NULL) >= task->timeout) + { + task_call_later(task); + return; + } +} + + +static Py_uhash_t +key_hash_traceback(const void *key) +{ + const traceback_t *traceback = key; + return traceback->hash; +} + +static int +key_cmp_traceback(const traceback_t *traceback1, hash_entry_t *he) +{ + const traceback_t *traceback2 = he->key; + const frame_t *frame1, *frame2; + Py_hash_t hash1, hash2; + int i; + + if (traceback1->nframe != traceback2->nframe) + return 0; + + for (i=0; i < traceback1->nframe; i++) { + frame1 = &traceback1->frames[i]; + frame2 = &traceback2->frames[i]; + + if (frame1->lineno != frame2->lineno) + return 0; + + if (frame1->filename != frame2->filename) { + hash1 = filename_hash(frame1->filename); + hash2 = filename_hash(frame2->filename); + if (hash1 != hash2) + return 0; + + if (PyUnicode_Compare(frame1->filename, frame2->filename) != 0) + return 0; + } + } + return 1; +} + +static void +tracemalloc_get_frame(PyFrameObject *pyframe, frame_t *frame) +{ + PyCodeObject *code; + PyObject *filename; + hash_entry_t *entry; + + frame->filename = NULL; + frame->lineno = PyFrame_GetLineNumber(pyframe); + + code = pyframe->f_code; + if (code == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error( + "failed to get the code object of " + "the a frame (thread %li)", + PyThread_get_thread_ident()); +#endif + return; + } + + if (code->co_filename == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error( + "failed to get the filename of the code object " + "(thread %li)", + PyThread_get_thread_ident()); +#endif + return; + } + + filename = code->co_filename; + assert(filename != NULL); + + if (!PyUnicode_CheckExact(filename)) { +#ifdef TRACE_DEBUG + tracemalloc_error("filename is not an unicode string"); +#endif + return; + } + if (!PyUnicode_IS_READY(filename)) { + /* Don't make a Unicode string ready to avoid reentrant calls + to tracemalloc_malloc() or tracemalloc_realloc() */ +#ifdef TRACE_DEBUG + tracemalloc_error("filename is not a ready unicode string"); +#endif + return; + } + + /* intern the filename */ + entry = hash_get_entry(tracemalloc_filenames, filename); + if (entry != NULL) { + filename = (PyObject*)entry->key; + } + else { + /* tracemalloc_filenames is responsible to keep a reference + to the filename */ + Py_INCREF(filename); + if (hash_put_data(tracemalloc_filenames, filename, NULL, 0) < 0) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to intern the filename"); +#endif + return; + } + } + + /* the tracemalloc_filenames table keeps a reference to the filename */ + frame->filename = filename; +} + +static Py_uhash_t +traceback_hash(traceback_t *traceback) +{ + /* code based on tuplehash() of Objects/tupleobject.c */ + Py_uhash_t x; /* Unsigned for defined overflow behavior. */ + Py_hash_t y; + int len = traceback->nframe; + Py_uhash_t mult = _PyHASH_MULTIPLIER; + frame_t *frame; + + x = 0x345678UL; + frame = traceback->frames; + while (--len >= 0) { + y = filename_hash(frame->filename); + y ^= frame->lineno; + frame++; + + x = (x ^ y) * mult; + /* the cast might truncate len; that doesn't change hash stability */ + mult += (Py_hash_t)(82520UL + len + len); + } + x += 97531UL; + return x; +} + +static void +traceback_get_frames(traceback_t *traceback) +{ + PyThreadState *tstate; + PyFrameObject *pyframe; + + tstate = PyGILState_GetThisThreadState(); + if (tstate == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error( + "failed to get the current thread state (thread %li)", + PyThread_get_thread_ident()); +#endif + return; + } + + for (pyframe = tstate->frame; pyframe != NULL; pyframe = pyframe->f_back) { + tracemalloc_get_frame(pyframe, &traceback->frames[traceback->nframe]); + traceback->nframe++; + if (traceback->nframe == tracemalloc_config.max_nframe) + break; + } +} + +static traceback_t * +traceback_new(void) +{ + char stack_buffer[TRACEBACK_STACK_SIZE]; + traceback_t *traceback = (traceback_t *)stack_buffer; + hash_entry_t *entry; + + assert(PyGILState_Check()); + traceback->nframe = 0; + assert(tracemalloc_config.max_nframe > 0); + traceback_get_frames(traceback); + traceback->hash = traceback_hash(traceback); + + if (!traceback_match_filters(traceback)) + return NULL; + + /* intern the traceback */ + entry = hash_get_entry(tracemalloc_tracebacks, traceback); + if (entry != NULL) { + traceback = (traceback_t *)entry->key; + } + else { + traceback_t *copy; + size_t traceback_size; + + traceback_size = TRACEBACK_SIZE(traceback->nframe); + + copy = raw_malloc(traceback_size); + if (copy == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to intern the traceback: malloc failed"); +#endif + return NULL; + } + memcpy(copy, traceback, traceback_size); + + /* tracemalloc_tracebacks is responsible to keep a reference + to the traceback */ + if (hash_put_data(tracemalloc_tracebacks, copy, NULL, 0) < 0) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to intern the traceback: putdata failed"); +#endif + return NULL; + } + traceback = copy; + } + return traceback; +} + +static void +tracemalloc_update_stats(trace_t *trace, int is_alloc) +{ + trace_stats_t local_trace_stat; + trace_stats_t *trace_stats; + hash_t *line_hash; + void *line_key; + hash_entry_t *line_entry; + PyObject *filename; + int lineno; + traceback_t *traceback; + + traceback = trace->traceback; + if (traceback->nframe >= 1) { + filename = traceback->frames[0].filename; + lineno = traceback->frames[0].lineno; + } + else { + filename = NULL; + lineno = -1; + } + + if (!HASH_GET_DATA(tracemalloc_file_stats, filename, line_hash)) { + if (!is_alloc) { + /* clear_traces() was called or tracemalloc_update_stats() failed + to store the allocation */ + return; + } + + line_hash = hash_new(&tracemalloc_pools.traces, + sizeof(trace_stats_t), + key_hash_int, key_cmp_direct); + if (line_hash == NULL) { +#ifdef TRACE_DEBUG + tracemalloc_error("failed to allocate a hash table for lines " + "for a new filename"); +#endif + return; + } +#ifdef PRINT_STATS + line_hash->name = "line_stats"; +#endif + + if (HASH_PUT_DATA(tracemalloc_file_stats, filename, line_hash) < 0) { + hash_destroy(line_hash); + return; + } + } + + line_key = INT_TO_POINTER(lineno); + line_entry = hash_get_entry(line_hash, line_key); + if (line_entry != NULL) { + assert(line_hash->data_size == sizeof(trace_stats_t)); + trace_stats = (trace_stats_t *)ENTRY_DATA_PTR(line_entry); + + if (is_alloc) { + assert(trace_stats->size < PY_SIZE_MAX - trace->size); + trace_stats->size += trace->size; + assert(trace_stats->count != PY_SIZE_MAX); + trace_stats->count++; + } + else { + assert(trace_stats->size >= trace->size); + trace_stats->size -= trace->size; + assert(trace_stats->count != 0); + trace_stats->count--; + assert(trace_stats->count != 0 || trace_stats->size == 0); + + if (trace_stats->count == 0) { + hash_delete_data(line_hash, line_key); + if (line_hash->entries == 0) + hash_delete_data(tracemalloc_file_stats, filename); + } + } + } + else { + if (!is_alloc) { + /* clear_traces() was called or tracemalloc_update_stats() failed + to store the allocation */ + return; + } + + local_trace_stat.size = trace->size; + local_trace_stat.count = 1; + HASH_PUT_DATA(line_hash, line_key, local_trace_stat); + } +} + +#ifdef TRACE_NORMALIZE_FILENAME +static Py_UCS4 +tracemalloc_normalize_filename(Py_UCS4 ch) +{ +#ifdef ALTSEP + if (ch == ALTSEP) + return SEP; +#endif +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + return ch; +} +#endif + +typedef struct { + PyObject *filename, *pattern; + int file_kind, pat_kind; + void *file_data, *pat_data; + Py_ssize_t file_len, pat_len; +} match_t; + +static int +match_filename_joker(match_t *match, Py_ssize_t file_pos, Py_ssize_t pat_pos) +{ + Py_UCS4 ch1, ch2; + + while (file_pos < match->file_len && pat_pos < match->pat_len) { + ch1 = PyUnicode_READ(match->file_kind, match->file_data, file_pos); + ch2 = PyUnicode_READ(match->pat_kind, match->pat_data, pat_pos); + if (ch2 == '*') { + int found; + + do { + pat_pos++; + if (pat_pos >= match->pat_len) { + /* 'abc' always match '*' */ + return 1; + } + ch2 = PyUnicode_READ(match->pat_kind, match->pat_data, + pat_pos); + } while (ch2 == '*'); + + do { + found = match_filename_joker(match, file_pos, pat_pos); + if (found) + break; + file_pos++; + } while (file_pos < match->file_len); + + return found; + } + +#ifdef TRACE_NORMALIZE_FILENAME + ch1 = tracemalloc_normalize_filename(ch1); +#endif + if (ch1 != ch2) + return 0; + + file_pos++; + pat_pos++; + } + + if (pat_pos != match->pat_len) { + if (pat_pos == (match->pat_len - 1)) { + ch2 = PyUnicode_READ(match->pat_kind, match->pat_data, pat_pos); + if (ch2 == '*') { + /* 'abc' matchs 'abc*' */ + return 1; + } + } + return 0; + } + return 1; +} + +static int +filename_endswith_pyc_pyo(PyObject *filename) +{ + void* data; + int kind; + Py_UCS4 ch; + Py_ssize_t len; + + len = PyUnicode_GetLength(filename); + if (len < 4) + return 0; + + data = PyUnicode_DATA(filename); + kind = PyUnicode_KIND(filename); + + if (PyUnicode_READ(kind, data, len-4) != '.') + return 0; + ch = PyUnicode_READ(kind, data, len-3); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if (ch != 'p') + return 0; + + ch = PyUnicode_READ(kind, data, len-2); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if (ch != 'y') + return 0; + + ch = PyUnicode_READ(kind, data, len-1); +#ifdef TRACE_CASE_INSENSITIVE + ch = _PyUnicode_ToLowercase(ch); +#endif + if ((ch != 'c' && ch != 'o')) + return 0; + return 1; +} + +static int +match_filename(filter_t *filter, PyObject *filename) +{ + Py_ssize_t len; + match_t match; + Py_hash_t hash; + + assert(PyUnicode_IS_READY(filename)); + + if (filename == filter->pattern) + return 1; + + hash = PyObject_Hash(filename); + if (hash == filter->pattern_hash) { + if (PyUnicode_Compare(filename, filter->pattern) == 0) + return 1; + } + +#ifndef TRACE_NORMALIZE_FILENAME + if (!filter->use_joker) { + if (!filename_endswith_pyc_pyo(filename)) { + /* hash is different: strings are different */ + return 0; + } + else { + len = PyUnicode_GetLength(filename); + + /* don't compare last character */ + return PyUnicode_Tailmatch(filename, filter->pattern, + 0, len - 1, 1); + } + } +#endif + + len = PyUnicode_GetLength(filename); + + /* replace "a.pyc" and "a.pyo" with "a.py" */ + if (filename_endswith_pyc_pyo(filename)) + len--; + + match.filename = filename; + match.file_kind = PyUnicode_KIND(match.filename); + match.file_data = PyUnicode_DATA(match.filename); + match.file_len = len; + + match.pattern = filter->pattern; + match.pat_kind = PyUnicode_KIND(match.pattern); + match.pat_data = PyUnicode_DATA(match.pattern); + match.pat_len = PyUnicode_GET_LENGTH(match.pattern); + return match_filename_joker(&match, 0, 0); +} + +static int +filter_match_filename(filter_t *filter, PyObject *filename) +{ + int match; + + if (filename == NULL) + return !filter->include; + + match = match_filename(filter, filename); + return match ^ !filter->include; +} + +static int +filter_match_lineno(filter_t *filter, int lineno) +{ + int match; + + if (filter->lineno < 1) + return 1; + + if (lineno < 1) + return !filter->include; + + match = (lineno == filter->lineno); + return match ^ !filter->include; +} + +static int +filter_match(filter_t *filter, PyObject *filename, int lineno) +{ + int match; + + if (filename == NULL) + return !filter->include; + + match = match_filename(filter, filename); + if (filter->include) { + if (!match) + return 0; + } + else { + /* exclude */ + if (!match) + return 1; + else if (filter->lineno < 1) + return 0; + } + + return filter_match_lineno(filter, lineno); +} + +static int +filter_match_traceback(filter_t *filter, traceback_t *traceback) +{ + int i; + PyObject *filename; + int lineno; + int nframe; + int match; + + nframe = traceback->nframe; + if (nframe == 0) { + return filter_match(filter, NULL, -1); + } + else if (!filter->traceback) { + filename = traceback->frames[0].filename; + lineno = traceback->frames[0].lineno; + return filter_match(filter, filename, lineno); + } + else { + for (i = 0; i < nframe; i++) { + filename = traceback->frames[i].filename; + lineno = traceback->frames[i].lineno; + + match = filter_match(filter, filename, lineno); + if (match) { + if (filter->include) + return 1; + } + else { + if (!filter->include) + return 0; + } + } + return !filter->include; + } +} + +static int +filter_list_match(filter_list_t *filters, int include, traceback_t *traceback) +{ + size_t i; + filter_t *filter; + int match; + + if (filters->nfilter == 0) + return 1; + + for (i = 0; i < filters->nfilter; i++) { + filter = &filters->filters[i]; + + match = filter_match_traceback(filter, traceback); + if (include) { + if (match) + return 1; + } + else { + if (!match) + return 0; + } + } + return !include; +} + +static int +traceback_match_filters(traceback_t *traceback) +{ + if (!filter_list_match(&tracemalloc_include_filters, 1, traceback)) + return 0; + if (!filter_list_match(&tracemalloc_exclude_filters, 0, traceback)) + return 0; + return 1; +} + +static void +tracemalloc_log_alloc(void *ptr, size_t size) +{ + traceback_t *traceback; + trace_t trace; + + assert(PyGILState_Check()); + + if (tracemalloc_config.max_nframe > 0) { + traceback = traceback_new(); + if (traceback == NULL) + return; + } + else { + traceback = &tracemalloc_empty_traceback; + if (!traceback_match_filters(traceback)) + return; + } + + trace.size = size; + trace.traceback = traceback; + + TABLES_LOCK(); + assert(tracemalloc_traced_memory <= PY_SIZE_MAX - size); + tracemalloc_traced_memory += size; + if (tracemalloc_traced_memory > tracemalloc_max_traced_memory) + tracemalloc_max_traced_memory = tracemalloc_traced_memory; + + tracemalloc_update_stats(&trace, 1); + HASH_PUT_DATA(tracemalloc_traces, ptr, trace); + TABLES_UNLOCK(); +} + +static void +tracemalloc_log_free(void *ptr) +{ + trace_t trace; + + TABLES_LOCK(); + if (hash_pop_data(tracemalloc_traces, ptr, &trace, sizeof(trace))) { + assert(tracemalloc_traced_memory >= trace.size); + tracemalloc_traced_memory -= trace.size; + tracemalloc_update_stats(&trace, 0); + } + TABLES_UNLOCK(); +} + +static void* +tracemalloc_malloc(void *ctx, size_t size, int gil_held) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; +#ifdef TRACE_RAW_MALLOC + PyGILState_STATE gil_state; +#endif + void *ptr; + + if (get_reentrant()) + return alloc->malloc(alloc->ctx, size); + +#ifdef TRACE_RAW_MALLOC + if (!gil_held) { + /* PyGILState_Ensure() may call PyMem_RawMalloc() indirectly which + would call PyGILState_Ensure() if reentrant are not disabled. */ + set_reentrant(1); + gil_state = PyGILState_Ensure(); + + ptr = alloc->malloc(alloc->ctx, size); + set_reentrant(0); + } + else +#endif + { + assert(gil_held); + + /* Ignore reentrant call: PyObjet_Malloc() calls PyMem_Malloc() for + allocations larger than 512 bytes */ + set_reentrant(1); + ptr = alloc->malloc(alloc->ctx, size); + set_reentrant(0); + } + + if (ptr != NULL) + tracemalloc_log_alloc(ptr, size); + + task_list_check(); + +#ifdef TRACE_RAW_MALLOC + if (!gil_held) + PyGILState_Release(gil_state); +#endif + + return ptr; +} + +static void* +tracemalloc_realloc(void *ctx, void *ptr, size_t new_size, int gil_held) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; +#ifdef TRACE_RAW_MALLOC + PyGILState_STATE gil_state; +#endif + void *ptr2; + + if (get_reentrant()) { + /* Reentrant call to PyMem_Realloc() and PyMem_RawRealloc(). + Example: PyMem_RawRealloc() is called internally by pymalloc + (_PyObject_Malloc() and _PyObject_Realloc()) to allocate a new + arena (new_arena()). */ + ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); + + + if (ptr2 != NULL && ptr != NULL) { +#ifdef TRACE_RAW_MALLOC + if (!gil_held) { + gil_state = PyGILState_Ensure(); + tracemalloc_log_free(ptr); + PyGILState_Release(gil_state); + } + else +#endif + { + assert(gil_held); + tracemalloc_log_free(ptr); + } + } + + return ptr2; + } + +#ifdef TRACE_RAW_MALLOC + if (!gil_held) { + /* PyGILState_Ensure() may call PyMem_RawMalloc() indirectly which + would call PyGILState_Ensure() if reentrant are not disabled. */ + set_reentrant(1); + gil_state = PyGILState_Ensure(); + + ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); + set_reentrant(0); + } + else +#endif + { + assert(gil_held); + + /* PyObjet_Realloc() calls PyMem_Realloc() for allocations + larger than 512 bytes */ + set_reentrant(1); + ptr2 = alloc->realloc(alloc->ctx, ptr, new_size); + set_reentrant(0); + } + + if (ptr2 != NULL) { + if (ptr != NULL) + tracemalloc_log_free(ptr); + + tracemalloc_log_alloc(ptr2, new_size); + } + + task_list_check(); + +#ifdef TRACE_RAW_MALLOC + if (!gil_held) + PyGILState_Release(gil_state); +#endif + + return ptr2; +} + +static void +tracemalloc_free(void *ctx, void *ptr, int gil_held) +{ + PyMemAllocator *alloc = (PyMemAllocator *)ctx; + + /* Cannot lock the GIL in PyMem_RawFree() because it would introduce + a deadlock in PyThreadState_DeleteCurrent(). */ + + if (ptr != NULL) { + alloc->free(alloc->ctx, ptr); + tracemalloc_log_free(ptr); + } + + if (gil_held) { + /* tracemalloc_task is protected by the GIL */ + task_list_check(); + } +} + +static void +tracemalloc_free_gil(void *ctx, void *ptr) +{ + tracemalloc_free(ctx, ptr, 1); +} + +static void +tracemalloc_raw_free(void *ctx, void *ptr) +{ + tracemalloc_free(ctx, ptr, 0); +} + +static void* +tracemalloc_malloc_gil(void *ctx, size_t size) +{ + return tracemalloc_malloc(ctx, size, 1); +} + +static void* +tracemalloc_realloc_gil(void *ctx, void *ptr, size_t new_size) +{ + return tracemalloc_realloc(ctx, ptr, new_size, 1); +} + +#ifdef TRACE_RAW_MALLOC +static void* +tracemalloc_raw_malloc(void *ctx, size_t size) +{ + return tracemalloc_malloc(ctx, size, 0); +} + +static void* +tracemalloc_raw_realloc(void *ctx, void *ptr, size_t new_size) +{ + return tracemalloc_realloc(ctx, ptr, new_size, 0); +} +#endif + +static void* +tracemalloc_alloc_arena(void *ctx, size_t size) +{ + void *ptr; + + ptr = arena_allocator.alloc(ctx, size); + if (ptr == NULL) + return NULL; + + if (hash_put_data(tracemalloc_arenas, ptr, NULL, 0) == 0) { + assert(tracemalloc_arena_size <= PY_SIZE_MAX - size); + tracemalloc_arena_size += size; + } + else { +#ifdef TRACE_DEBUG + tracemalloc_error("alloc_arena: put_data() failed"); +#endif + } + return ptr; +} + +static void +tracemalloc_free_arena(void *ctx, void *ptr, size_t size) +{ + arena_allocator.free(ctx, ptr, size); + if (hash_may_delete_data(tracemalloc_arenas, ptr)) { + assert(tracemalloc_arena_size >= size); + tracemalloc_arena_size -= size; + } +} + +static int +filter_init(filter_t *filter, + int include, PyObject *pattern, int lineno, + int traceback) +{ + Py_ssize_t len, len2; + Py_ssize_t i, j; + PyObject *new_pattern; + Py_UCS4 maxchar, ch; + int kind, kind2; + void *data, *data2; + int previous_joker; + size_t njoker; + Py_hash_t pattern_hash; + + if (PyUnicode_READY(pattern) < 0) + return -1; + + len = PyUnicode_GetLength(pattern); + kind = PyUnicode_KIND(pattern); + data = PyUnicode_DATA(pattern); + + if (filename_endswith_pyc_pyo(pattern)) + len--; + + maxchar = 0; + len2 = 0; + njoker = 0; + previous_joker = 0; + for (i=0; i < len; i++) { + ch = PyUnicode_READ(kind, data, i); +#ifdef TRACE_NORMALIZE_FILENAME + ch = tracemalloc_normalize_filename(ch); +#endif + if (!previous_joker || ch != '*') { + previous_joker = (ch == '*'); + if (previous_joker) + njoker++; + maxchar = Py_MAX(maxchar, ch); + len2++; + } + else { + /* skip consecutive joker character */ + } + } + + if (njoker > MAX_NJOKER) { + PyErr_SetString(PyExc_ValueError, + "too many joker characters in the filename pattern"); + return -1; + } + + new_pattern = PyUnicode_New(len2, maxchar); + if (new_pattern == NULL) + return -1; + kind2 = PyUnicode_KIND(new_pattern); + data2 = PyUnicode_DATA(new_pattern); + + j = 0; + previous_joker = 0; + for (i=0; i < len; i++) { + ch = PyUnicode_READ(kind, data, i); +#ifdef TRACE_NORMALIZE_FILENAME + ch = tracemalloc_normalize_filename(ch); +#endif + if (!previous_joker || ch != '*') { + previous_joker = (ch == '*'); + PyUnicode_WRITE(kind2, data2, j, ch); + j++; + } + else { + /* skip consecutive joker character */ + } + } + assert(j == len2); + + assert(_PyUnicode_CheckConsistency(new_pattern, 1)); + + pattern = new_pattern; + + pattern_hash = PyObject_Hash(pattern); + + filter->include = include; + filter->pattern_hash = pattern_hash; + filter->pattern = pattern; +#ifndef TRACE_NORMALIZE_FILENAME + filter->use_joker = (njoker != 0); +#endif + if (lineno >= 1) + filter->lineno = lineno; + else + filter->lineno = -1; + filter->traceback = traceback; + return 0; +} + +static void +filter_deinit(filter_t *filter) +{ + Py_CLEAR(filter->pattern); +} + +static void +filter_list_init(filter_list_t *filters) +{ + filters->nfilter = 0; + filters->filters = NULL; +} + +static void +filter_list_clear(filter_list_t *filters) +{ + size_t i; + + if (filters->nfilter == 0) { + assert(filters->filters == NULL); + return; + } + + for (i=0; infilter; i++) + filter_deinit(&filters->filters[i]); + + filters->nfilter = 0; + raw_free(filters->filters); + filters->filters = NULL; +} + +static void +tracemalloc_clear_filters(void) +{ + filter_list_clear(&tracemalloc_include_filters); + filter_list_clear(&tracemalloc_exclude_filters); +} + +static int +tracemalloc_clear_filename(hash_entry_t *entry, void *user_data) +{ + PyObject *filename = (PyObject *)entry->key; + Py_DECREF(filename); + return 0; +} + +static int +traceback_free_cb(hash_entry_t *entry, void *user_data) +{ + traceback_t *traceback = (traceback_t *)entry->key; + raw_free(traceback); + return 0; +} + +/* reentrant flag must be set to call this function and GIL must be held */ +static void +tracemalloc_clear_traces(void) +{ + /* The GIL protects variables againt concurrent access */ + assert(PyGILState_Check()); + + /* Disable also reentrant calls to tracemalloc_malloc() to not add a new + trace while we are clearing traces */ + assert(get_reentrant()); + + TABLES_LOCK(); + hash_clear(tracemalloc_file_stats); + hash_clear(tracemalloc_traces); + tracemalloc_traced_memory = 0; + tracemalloc_max_traced_memory = 0; + TABLES_UNLOCK(); + + hash_foreach(tracemalloc_tracebacks, traceback_free_cb, NULL); + hash_clear(tracemalloc_tracebacks); + + hash_foreach(tracemalloc_filenames, tracemalloc_clear_filename, NULL); + hash_clear(tracemalloc_filenames); + + hash_clear(tracemalloc_arenas); + tracemalloc_arena_size = 0; +} + +static int +tracemalloc_init(void) +{ + size_t entry_size; +#ifdef TRACEMALLOC_ATFORK + int res; +#endif + + /* ensure that the frame_t structure is packed */ + assert(sizeof(frame_t) == (sizeof(PyObject*) + sizeof(int))); + + if (tracemalloc_config.init) + return 0; + + tracemalloc_tasks = PyList_New(0); + if (tracemalloc_tasks == NULL) + return -1; + + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); + PyObject_GetArenaAllocator(&arena_allocator); + +#ifdef REENTRANT_THREADLOCAL +#ifdef NT_THREADS + tracemalloc_reentrant_key = TlsAlloc(); + if (tracemalloc_reentrant_key == TLS_OUT_OF_INDEXES) { + PyErr_SetFromWindowsErr(0); + return -1; + } +#else + if (pthread_key_create(&tracemalloc_reentrant_key, NULL)) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } +#endif +#endif + + filter_list_init(&tracemalloc_include_filters); + filter_list_init(&tracemalloc_exclude_filters); + +#if defined(TRACE_RAW_MALLOC) && defined(WITH_THREAD) + if (tables_lock == NULL) { + tables_lock = PyThread_allocate_lock(); + if (tables_lock == NULL) { + PyErr_SetString(PyExc_RuntimeError, "cannot allocate lock"); + return -1; + } + } +#endif + + entry_size = sizeof(hash_entry_t); + pool_init(&tracemalloc_pools.no_data, entry_size + 0); + assert(sizeof(trace_t) == sizeof(trace_stats_t)); + pool_init(&tracemalloc_pools.traces, entry_size + sizeof(trace_t)); + pool_init(&tracemalloc_pools.hash_tables, entry_size + sizeof(hash_t*)); + + tracemalloc_filenames = hash_new(&tracemalloc_pools.no_data, + 0, + (key_hash_func)filename_hash, + key_cmp_unicode); + + tracemalloc_tracebacks = hash_new(&tracemalloc_pools.no_data, + 0, + (key_hash_func)key_hash_traceback, + (key_compare_func)key_cmp_traceback); + + tracemalloc_file_stats = hash_new_full(&tracemalloc_pools.hash_tables, + sizeof(hash_t *), + 0, + key_hash_ptr, + key_cmp_direct, + (hash_copy_data_func)hash_copy, + (hash_free_data_func)hash_destroy, + (hash_get_data_size_func)hash_mem_stats); + + tracemalloc_traces = hash_new(&tracemalloc_pools.traces, + sizeof(trace_t), + key_hash_ptr, key_cmp_direct); + + tracemalloc_arenas = hash_new(&tracemalloc_pools.no_data, + 0, + key_hash_arena_ptr, key_cmp_direct); + + if (tracemalloc_filenames == NULL || tracemalloc_tracebacks == NULL + || tracemalloc_file_stats == NULL || tracemalloc_traces == NULL + || tracemalloc_arenas == NULL ) + { + PyErr_NoMemory(); + return -1; + } +#ifdef PRINT_STATS + tracemalloc_pools.no_data.name = "no data"; + tracemalloc_pools.traces.name = "traces"; + tracemalloc_pools.hash_tables.name = "hash_tables"; + + tracemalloc_filenames->name = "filenames"; + tracemalloc_tracebacks->name = "tracebacks"; + tracemalloc_file_stats->name = "file_stats"; + tracemalloc_traces->name = "traces"; + tracemalloc_arenas->name = "arenas"; +#endif + +#ifdef TRACEMALLOC_ATFORK + res = pthread_atfork(NULL, NULL, tracemalloc_atfork); + if (res != 0) { + PyErr_SetFromErrno(PyExc_OSError); + return -1; + } +#endif + + tracemalloc_empty_traceback.nframe = 0; + tracemalloc_empty_traceback.hash = traceback_hash(&tracemalloc_empty_traceback); + + /* disable tracing allocations until tracemalloc is fully initialized */ + set_reentrant(1); + + tracemalloc_config.init = 1; + return 0; +} + +static void +tracemalloc_deinit(void) +{ + int err; +#if defined(REENTRANT_THREADLOCAL) && !defined(NDEBUG) +#ifdef NT_THREADS + BOOL res; +#else + int res; +#endif +#endif + + if (!tracemalloc_config.init) + return; + tracemalloc_config.init = 0; + + err = tracemalloc_disable(); + assert(err == 0); + + tracemalloc_clear_filters(); + + Py_CLEAR(tracemalloc_tasks); + + /* destroy hash tables */ + hash_destroy(tracemalloc_file_stats); + hash_destroy(tracemalloc_traces); + hash_destroy(tracemalloc_tracebacks); + hash_destroy(tracemalloc_filenames); + hash_destroy(tracemalloc_arenas); + +#ifdef USE_MEMORY_POOL + pool_clear(&tracemalloc_pools.no_data); + pool_clear(&tracemalloc_pools.traces); + pool_clear(&tracemalloc_pools.hash_tables); +#endif + +#if defined(WITH_THREAD) && defined(TRACE_RAW_MALLOC) + if (tables_lock == NULL) { + PyThread_free_lock(tables_lock); + tables_lock = NULL; + } +#endif + +#ifdef REENTRANT_THREADLOCAL +#ifdef NT_THREADS + /* Windows */ + if (tracemalloc_reentrant_key != TLS_OUT_OF_INDEXES) { +#ifndef NDEBUG + res = TlsFree(tracemalloc_reentrant_key); + assert(res); +#else + (void)TlsFree(tracemalloc_reentrant_key); +#endif + } +#else + /* pthread */ +#ifndef NDEBUG + res = pthread_key_delete(tracemalloc_reentrant_key); + assert(res == 0); +#else + (void)pthread_key_delete(tracemalloc_reentrant_key); +#endif +#endif +#endif +} + +static int +tracemalloc_enable(void) +{ + PyMemAllocator alloc; + PyObjectArenaAllocator arena_hook; + + if (tracemalloc_init() < 0) + return -1; + + if (tracemalloc_config.enabled) { + /* hook already installed: do nothing */ + return 0; + } + + tracemalloc_traced_memory = 0; + tracemalloc_max_traced_memory = 0; + tracemalloc_arena_size = 0; + +#ifdef TRACE_RAW_MALLOC + alloc.malloc = tracemalloc_raw_malloc; + alloc.realloc = tracemalloc_raw_realloc; + alloc.free = tracemalloc_raw_free; + + alloc.ctx = &allocators.raw; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); +#endif + + alloc.malloc = tracemalloc_malloc_gil; + alloc.realloc = tracemalloc_realloc_gil; + alloc.free = tracemalloc_free_gil; + + alloc.ctx = &allocators.mem; + PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); + + alloc.ctx = &allocators.obj; + PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); + + arena_hook.ctx = &arena_allocator; + arena_hook.alloc = tracemalloc_alloc_arena; + arena_hook.free = tracemalloc_free_arena; + PyObject_SetArenaAllocator(&arena_hook); + + /* every is ready: start tracing Python memory allocations */ + set_reentrant(0); + + tracemalloc_config.enabled = 1; + return 0; +} + +static int +tracemalloc_disable(void) +{ + int err; + + if (!tracemalloc_config.enabled) + return 0; + + /* cancel all scheduled tasks */ + err = task_list_clear(); + + /* stop tracing Python memory allocations */ + set_reentrant(1); + + tracemalloc_config.enabled = 0; + + /* unregister the hook on memory allocators */ +#ifdef TRACE_RAW_MALLOC + PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &allocators.raw); +#endif + PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &allocators.mem); + PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &allocators.obj); + + PyObject_SetArenaAllocator(&arena_allocator); + + /* release memory */ + tracemalloc_clear_traces(); + return err; +} + +#ifdef TRACEMALLOC_ATFORK +static void +tracemalloc_atfork(void) +{ + int err; + + PyGILState_STATE gil_state; + + if (!tracemalloc_config.enabled) + return; + + /* fork() can be called with the GIL released */ + set_reentrant(1); + gil_state = PyGILState_Ensure(); + set_reentrant(0); + + err = tracemalloc_disable(); + assert(err == 0); + PyGILState_Release(gil_state); +} +#endif + +typedef struct { + pool_t pool; + hash_t *file_stats; + hash_t *line_hash; + PyObject *file_dict; + PyObject *line_dict; +} get_stats_t; + +static PyObject* +tracemalloc_lineno_as_obj(int lineno) +{ + if (lineno > 0) + return PyLong_FromLong(lineno); + else + Py_RETURN_NONE; +} + +static PyObject* +trace_stats_to_pyobject(trace_stats_t *trace_stats) +{ + PyObject *size, *count, *line_obj; + + line_obj = PyTuple_New(2); + if (line_obj == NULL) + return NULL; + + size = PyLong_FromSize_t(trace_stats->size); + if (size == NULL) { + Py_DECREF(line_obj); + return NULL; + } + PyTuple_SET_ITEM(line_obj, 0, size); + + count = PyLong_FromSize_t(trace_stats->count); + if (count == NULL) { + Py_DECREF(line_obj); + return NULL; + } + PyTuple_SET_ITEM(line_obj, 1, count); + + return line_obj; +} + +static int +tracemalloc_get_stats_fill_line(hash_entry_t *entry, void *user_data) +{ + int lineno; + trace_stats_t trace_stats; + get_stats_t *get_stats = user_data; + PyObject *key, *line_obj; + int err; + + lineno = POINTER_TO_INT(entry->key); + + HASH_ENTRY_READ_DATA(get_stats->line_hash, + &trace_stats, sizeof(trace_stats), entry); + + key = tracemalloc_lineno_as_obj(lineno); + if (key == NULL) + return 1; + + line_obj = trace_stats_to_pyobject(&trace_stats); + if (line_obj == NULL) { + Py_DECREF(key); + return 1; + } + + err = PyDict_SetItem(get_stats->line_dict, key, line_obj); + Py_DECREF(key); + Py_DECREF(line_obj); + return err; +} + +static int +tracemalloc_get_stats_fill_file(hash_entry_t *entry, void *user_data) +{ + PyObject *filename; + get_stats_t *get_stats = user_data; + int res; + + filename = (PyObject *)entry->key; + if (filename == NULL) + filename = Py_None; + + get_stats->line_hash = HASH_ENTRY_DATA_AS_VOID_P(entry); + + get_stats->line_dict = PyDict_New(); + if (get_stats->line_dict == NULL) + return 1; + + res = hash_foreach(get_stats->line_hash, + tracemalloc_get_stats_fill_line, user_data); + if (res) { + Py_DECREF(get_stats->line_dict); + return 1; + } + + res = PyDict_SetItem(get_stats->file_dict, filename, get_stats->line_dict); + Py_CLEAR(get_stats->line_dict); + if (res < 0) + return 1; + + return 0; +} + +typedef struct { + PyObject_HEAD + filter_t filter; +} FilterObject; + +/* Converter for PyArg_ParseTuple() to parse a filename, accepting None */ +static int +tracemalloc_parse_filename(PyObject* arg, void* addr) +{ + PyObject *filename; + + if (arg == Py_None) { + filename = NULL; + } + else if (PyUnicode_Check(arg)) { + filename = arg; + } + else { + PyErr_SetString(PyExc_TypeError, "filename must be a str or None"); + return 0; + } + *(PyObject **)addr = filename; + return 1; +} + +/* Converter for PyArg_ParseTuple() to parse a line number, accepting None */ +static int +tracemalloc_parse_lineno(PyObject* arg, void* addr) +{ + int lineno; + + if (arg == Py_None) { + lineno = -1; + } + else { + lineno = _PyLong_AsInt(arg); + if (lineno == -1 && PyErr_Occurred()) + return 0; + } + *(int *)addr = lineno; + return 1; +} + +static int +tracemalloc_pyfilter_init(FilterObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"include", "filename", "lineno", "traceback", 0}; + int include; + PyObject *filename; + int lineno = -1; + int traceback = 0; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "iO|O&i:str", kwlist, + &include, &filename, + tracemalloc_parse_lineno, &lineno, + &traceback)) + return -1; + + filter_deinit(&self->filter); + + if (filter_init(&self->filter, include, filename, lineno, traceback) < 0) + return -1; + + return 0; +} + +static void +tracemalloc_pyfilter_dealloc(FilterObject *self) +{ + filter_deinit(&self->filter); + PyObject_FREE(self); +} + +static PyObject* +tracemalloc_pyfilter_match(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *filename; + int lineno; + int match; + + if (!PyArg_ParseTuple(args, "O&O&:match", + tracemalloc_parse_filename, &filename, + tracemalloc_parse_lineno, &lineno)) + return NULL; + + match = filter_match(&pyfilter->filter, filename, lineno); + return PyBool_FromLong(match); +} + +static int +parse_traceback(PyObject *pytraceback, + traceback_t *traceback, size_t buffer_size) +{ + PyObject *pyframe, *filename, *pylineno; + Py_ssize_t nframe, i; + int lineno; + + if (!PyTuple_Check(pytraceback)) { + PyErr_SetString(PyExc_TypeError, "traceback must be a tuple"); + return -1; + } + + nframe = PyTuple_GET_SIZE(pytraceback); + if (nframe > MAX_NFRAME) { + PyErr_SetString(PyExc_TypeError, "too many frames"); + return -1; + } + assert(TRACEBACK_SIZE(nframe) <= buffer_size); + + traceback->nframe = nframe; + for (i=0; i < nframe; i++) { + pyframe = PyTuple_GET_ITEM(pytraceback, i); + assert(pyframe != NULL); + if (!PyTuple_Check(pyframe) || Py_SIZE(pyframe) != 2) { + PyErr_SetString(PyExc_TypeError, "frames must be 2-tuples"); + return -1; + } + + filename = PyTuple_GET_ITEM(pyframe, 0); + assert(filename != NULL); + pylineno = PyTuple_GET_ITEM(pyframe, 1); + assert(pylineno != NULL); + if (tracemalloc_parse_lineno(pylineno, &lineno) == 0) + return -1; + + /* borrowed reference to filename */ + traceback->frames[i].filename = filename; + traceback->frames[i].lineno = lineno; + } + + return 0; +} + +static PyObject* +tracemalloc_pyfilter_match_traceback(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *pytraceback; + int match; + char stack_buffer[TRACEBACK_STACK_SIZE]; + traceback_t *traceback = (traceback_t *)stack_buffer; + + if (!PyArg_ParseTuple(args, "O:match_traceback", &pytraceback)) + return NULL; + + if (parse_traceback(pytraceback, traceback, sizeof(stack_buffer)) < 0) + return NULL; + + match = filter_match_traceback(&pyfilter->filter, traceback); + return PyBool_FromLong(match); +} + +static PyObject* +tracemalloc_pyfilter_match_filename(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + PyObject *filename; + int match; + + if (!PyArg_ParseTuple(args, "O&:match_filename", + tracemalloc_parse_filename, &filename)) + return NULL; + + match = filter_match_filename(&pyfilter->filter, filename); + return PyBool_FromLong(match); +} + +static PyObject* +tracemalloc_pyfilter_match_lineno(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter = (FilterObject *)self; + int lineno; + int match; + + if (!PyArg_ParseTuple(args, "O&:match_lineno", + tracemalloc_parse_lineno, &lineno)) + return NULL; + + match = filter_match_lineno(&pyfilter->filter, lineno); + return PyBool_FromLong(match); +} + +static PyObject * +tracemalloc_pyfilter_get_include(FilterObject *self, void *closure) +{ + return PyBool_FromLong(self->filter.include); +} + +static PyObject * +tracemalloc_pyfilter_get_pattern(FilterObject *self, void *closure) +{ + Py_INCREF(self->filter.pattern); + return self->filter.pattern; +} + +static PyObject * +tracemalloc_pyfilter_get_lineno(FilterObject *self, void *closure) +{ + return tracemalloc_lineno_as_obj(self->filter.lineno); +} + +static PyObject * +tracemalloc_pyfilter_get_traceback(FilterObject *self, void *closure) +{ + return PyBool_FromLong(self->filter.traceback); +} + +static PyObject* +tracemalloc_pyfilter_repr(FilterObject *self) +{ + char lineno[30]; + if (self->filter.lineno > 1) + PyOS_snprintf(lineno, sizeof(lineno), "%i", self->filter.lineno); + else + strcpy(lineno, "None"); + return PyUnicode_FromFormat("", + self->filter.include ? "True" : "False", + self->filter.pattern, + lineno, + self->filter.traceback ? "True" : "False"); +} + +static int +filter_compare(filter_t *f1, filter_t *f2) +{ + if (f1->include != f2->include) + return 0; + if (f1->lineno != f2->lineno) + return 0; + if (f1->traceback != f2->traceback) + return 0; + if (PyUnicode_Compare(f1->pattern, f2->pattern) != 0) + return 0; +#ifndef TRACE_NORMALIZE_FILENAME + assert(f1->use_joker == f2->use_joker); +#endif + return 1; +} + +static Py_hash_t +tracemalloc_pyfilter_hash(FilterObject *self) +{ + Py_hash_t hash; + + hash = PyObject_Hash(self->filter.pattern); + hash ^= self->filter.lineno; + hash ^= ((Py_hash_t)self->filter.include << 20); + hash ^= ((Py_hash_t)self->filter.traceback << 21); + return hash; +} + +static PyObject * +tracemalloc_pyfilter_richcompare(FilterObject *self, FilterObject *other, int op) +{ + if (op == Py_EQ || op == Py_NE) { + int eq; + PyObject *res; + + eq = filter_compare(&self->filter, &other->filter); + if (op == Py_NE) + eq = !eq; + + if (eq) + res = Py_True; + else + res = Py_False; + Py_INCREF(res); + return res; + } + else { + Py_RETURN_NOTIMPLEMENTED; + } +} + +static PyGetSetDef tracemalloc_pyfilter_getset[] = { + {"include", (getter) tracemalloc_pyfilter_get_include, NULL, + "Include or exclude the trace?"}, + {"pattern", (getter) tracemalloc_pyfilter_get_pattern, NULL, + "Pattern matching a filename, can contain one " + "or many '*' joker characters"}, + {"lineno", (getter) tracemalloc_pyfilter_get_lineno, NULL, + "Line number"}, + {"traceback", (getter) tracemalloc_pyfilter_get_traceback, NULL, + "Check the whole traceback, or only the most recent frame?"}, + {NULL} +}; + +static PyMethodDef tracemalloc_pyfilter_methods[] = { + {"match", (PyCFunction)tracemalloc_pyfilter_match, + METH_VARARGS, + PyDoc_STR("match(filename: str, lineno: int) -> bool")}, + {"match_traceback", (PyCFunction)tracemalloc_pyfilter_match_traceback, + METH_VARARGS, + PyDoc_STR("match_traceback(traceback) -> bool")}, + {"match_filename", (PyCFunction)tracemalloc_pyfilter_match_filename, + METH_VARARGS, + PyDoc_STR("match_filename(filename: str) -> bool")}, + {"match_lineno", (PyCFunction)tracemalloc_pyfilter_match_lineno, + METH_VARARGS, + PyDoc_STR("match_lineno(lineno: int) -> bool")}, + {NULL, NULL} /* sentinel */ +}; + +PyDoc_STRVAR(tracemalloc_pyfilter_doc, +"Filter(include: bool, filename: str, lineno: int=None, traceback: bool=False)"); + +static PyTypeObject FilterType = { + /* The ob_type field must be initialized in the module init function + * to be portable to Windows without using C++. */ + PyVarObject_HEAD_INIT(NULL, 0) + "tracemalloc.Filter", /*tp_name*/ + sizeof(FilterObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)tracemalloc_pyfilter_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + (getattrfunc)0, /*tp_getattr*/ + (setattrfunc)0, /*tp_setattr*/ + 0, /*tp_reserved*/ + (reprfunc)tracemalloc_pyfilter_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + (hashfunc)tracemalloc_pyfilter_hash, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + (getattrofunc)0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + tracemalloc_pyfilter_doc, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + (richcmpfunc)tracemalloc_pyfilter_richcompare, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + tracemalloc_pyfilter_methods, /*tp_methods*/ + 0, /*tp_members*/ + tracemalloc_pyfilter_getset, /* tp_getset */ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + (initproc)tracemalloc_pyfilter_init, /* tp_init */ + 0, /*tp_alloc*/ + PyType_GenericNew, /*tp_new*/ + 0, /*tp_free*/ + 0, /*tp_is_gc*/ +}; + +typedef struct { + PyObject_HEAD + task_t task; +} TaskObject; + +/* Forward declaration */ +static PyTypeObject TaskType; + +static PyObject* +pytask_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + TaskObject *op; + + op = (TaskObject *)type->tp_alloc(type, 0); + if (op == NULL) + return NULL; + + task_init(&op->task); + return (PyObject *)op; +} + +static int +pytask_init(TaskObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *func, *func_args; + Py_ssize_t narg; + + narg = PyTuple_GET_SIZE(args); + if (narg == 0) { + PyErr_SetString(PyExc_ValueError, "missing func parameter"); + return -1; + } + + func = PyTuple_GET_ITEM(args, 0); + if (!PyCallable_Check(func)) { + PyErr_Format(PyExc_TypeError, + "func must be a callable object, not %s", + Py_TYPE(func)->tp_name); + return -1; + } + + func_args = PyTuple_GetSlice(args, 1, narg); + if (func_args == NULL) + return -1; + + Py_INCREF(func); + self->task.func = func; + self->task.func_args = func_args; + Py_XINCREF(kwargs); + self->task.func_kwargs = kwargs; + + self->task.ncall = -1; + return 0; +} + +static PyObject * +pytask_is_scheduled(TaskObject *self) +{ + int scheduled = PySequence_Contains(tracemalloc_tasks, (PyObject*)self); + if (scheduled == -1) + return NULL; + return PyBool_FromLong(scheduled); +} + +static PyObject * +pytask_get_func(TaskObject *self, void *closure) +{ + Py_INCREF(self->task.func); + return self->task.func; +} + +static int +pytask_set_func(TaskObject *self, PyObject *func) +{ + if (!PyCallable_Check(func)) { + PyErr_Format(PyExc_TypeError, + "func must be a callable object, not %s", + Py_TYPE(func)->tp_name); + return -1; + } + + Py_DECREF(self->task.func); + Py_INCREF(func); + self->task.func = func; + return 0; +} + +static int +pytask_set_func_args(TaskObject *self, PyObject *func_args) +{ + if (!PyTuple_Check(func_args)) { + PyErr_Format(PyExc_TypeError, + "func_args must be a tuple, not %s", + Py_TYPE(func_args)->tp_name); + return -1; + } + + Py_DECREF(self->task.func_args); + Py_INCREF(func_args); + self->task.func_args = func_args; + return 0; +} + +static PyObject * +pytask_get_func_args(TaskObject *self, void *closure) +{ + Py_INCREF(self->task.func_args); + return self->task.func_args; +} + +static PyObject * +pytask_get_func_kwargs(TaskObject *self, void *closure) +{ + if (self->task.func_kwargs) { + Py_INCREF(self->task.func_kwargs); + return self->task.func_kwargs; + } + else + Py_RETURN_NONE; +} + +static int +pytask_set_func_kwargs(TaskObject *self, PyObject *func_kwargs) +{ + if (!PyDict_Check(func_kwargs) && func_kwargs != Py_None) { + PyErr_Format(PyExc_TypeError, + "func_kwargs must be a dict or None, not %s", + Py_TYPE(func_kwargs)->tp_name); + return -1; + } + + Py_DECREF(self->task.func_kwargs); + Py_INCREF(func_kwargs); + self->task.func_kwargs = func_kwargs; + return 0; +} + +static int +pytask_reschedule(TaskObject *self) +{ + int scheduled; + + if (!tracemalloc_config.enabled) { + /* a task cannot be scheduled if the tracemalloc module is disabled */ + return 0; + } + + /* task_call_later() may have scheduled a call to the task: ensure + that the pending call list is empty */ + if (Py_MakePendingCalls() < 0) + return -1; + + set_reentrant(1); + scheduled = PySequence_Contains(tracemalloc_tasks, (PyObject*)self); + set_reentrant(0); + if (scheduled == -1) + return -1; + + if (scheduled == 1) + task_reschedule(&self->task); + + return 0; +} + +static PyObject * +pytask_schedule(TaskObject *self, PyObject *args) +{ + PyObject *repeat_obj = Py_None; + Py_ssize_t repeat; + int scheduled; + + if (!PyArg_ParseTuple(args, "|O:schedule", + &repeat_obj)) + return NULL; + + if (repeat_obj != Py_None) { + repeat = PyLong_AsSsize_t(repeat_obj); + if (repeat == -1 && PyErr_Occurred()) + return NULL; + if (repeat <= 0) { + PyErr_SetString(PyExc_ValueError, + "repeat must be positive or None"); + return NULL; + } + } + else + repeat = -1; + + if (!tracemalloc_config.enabled) { + PyErr_SetString(PyExc_RuntimeError, + "the tracemalloc module must be enabled " + "to schedule a task"); + return NULL; + } + + if (self->task.delay <= 0 && self->task.memory_threshold <= 0) { + PyErr_SetString(PyExc_ValueError, + "delay and memory_threshold are None"); + return NULL; + } + + scheduled = PySequence_Contains(tracemalloc_tasks, (PyObject*)self); + if (scheduled == -1) + return NULL; + if (!scheduled) { + if (PyList_Append(tracemalloc_tasks, (PyObject *)self) < 0) + return NULL; + + self->task.ncall = repeat; + task_schedule(&self->task); + } + else { + if (pytask_reschedule(self) < 0) + return NULL; + } + + Py_RETURN_NONE; +} + +static PyObject * +pytask_cancel(TaskObject *self) +{ + PyObject *res; + _Py_IDENTIFIER(remove); + + res = _PyObject_CallMethodId(tracemalloc_tasks, &PyId_remove, "O", self); + if (res == NULL) { + if (!PyErr_ExceptionMatches(PyExc_ValueError)) + return NULL; + /* task not scheduled: ignore the error */ + PyErr_Clear(); + } + else + Py_DECREF(res); + + task_cancel(&self->task); + + Py_RETURN_NONE; +} + +static PyObject * +pytask_call(TaskObject *self) +{ + return task_call(&self->task); +} + +static PyObject * +pytask_get_delay(TaskObject *self) +{ + if (self->task.delay > 0) + return PyLong_FromLong(self->task.delay); + else + Py_RETURN_NONE; +} + +static PyObject* +pytask_set_delay(TaskObject *self, PyObject *delay_obj) +{ + int delay; + + if (delay_obj != Py_None) { + delay = _PyLong_AsInt(delay_obj); + if (delay == -1 && PyErr_Occurred()) + return NULL; + if (delay <= 0) { + PyErr_SetString(PyExc_ValueError, + "delay must be positive or None"); + return NULL; + } + } + else + delay = -1; + + self->task.delay = delay; + if (pytask_reschedule(self) < 0) + return NULL; + + Py_RETURN_NONE; +} + +static PyObject * +pytask_get_memory_threshold(TaskObject *self) +{ + if (self->task.memory_threshold > 0) + return PyLong_FromSsize_t(self->task.memory_threshold); + else + Py_RETURN_NONE; +} + +static PyObject* +pytask_set_memory_threshold(TaskObject *self, PyObject *threshold_obj) +{ + Py_ssize_t memory_threshold; + + if (threshold_obj != Py_None) { + memory_threshold = PyLong_AsSsize_t(threshold_obj); + if (memory_threshold == -1 && PyErr_Occurred()) + return NULL; + if (memory_threshold <= 0) { + PyErr_SetString(PyExc_ValueError, + "size must be greater than zero " + "or None"); + return NULL; + } + } + else + memory_threshold = -1; + + self->task.memory_threshold = memory_threshold; + if (pytask_reschedule(self) < 0) + return NULL; + + Py_RETURN_NONE; +} + +static PyObject* +pytask_repr(TaskObject *self) +{ + return PyUnicode_FromFormat("", + self->task.func); +} + +static void +pytask_dealloc(TaskObject *self) +{ + assert(tracemalloc_tasks == NULL + || PySequence_Contains(tracemalloc_tasks, (PyObject*)self) == 0); + + Py_CLEAR(self->task.func); + Py_CLEAR(self->task.func_args); + Py_CLEAR(self->task.func_kwargs); + + Py_TYPE(self)->tp_free((PyObject *)self); +} + +static PyGetSetDef tracemalloc_pytask_getset[] = { + {"func", + (getter)pytask_get_func, (setter)pytask_set_func, + "Function"}, + {"func_args", + (getter)pytask_get_func_args, (setter)pytask_set_func_args, + "Function arguments"}, + {"func_kwargs", + (getter)pytask_get_func_kwargs, (setter)pytask_set_func_kwargs, + "Function keyword arguments"}, + {NULL} +}; + +static PyMethodDef tracemalloc_pytask_methods[] = { + {"call", (PyCFunction)pytask_call, + METH_NOARGS, + PyDoc_STR("call()")}, + {"is_scheduled", (PyCFunction)pytask_is_scheduled, + METH_NOARGS, + PyDoc_STR("is_enabled()")}, + {"schedule", (PyCFunction)pytask_schedule, + METH_VARARGS, + PyDoc_STR("schedule(repeat: int=None)")}, + {"cancel", (PyCFunction)pytask_cancel, + METH_NOARGS, + PyDoc_STR("cancel()")}, + {"get_delay", (PyCFunction)pytask_get_delay, + METH_NOARGS, + PyDoc_STR("get_delay()->int|None")}, + {"set_delay", (PyCFunction)pytask_set_delay, + METH_O, + PyDoc_STR("set_delay(seconds: int)")}, + {"get_memory_threshold", (PyCFunction)pytask_get_memory_threshold, + METH_NOARGS, + PyDoc_STR("get_memory_threshold()->int|None")}, + {"set_memory_threshold", (PyCFunction)pytask_set_memory_threshold, + METH_O, + PyDoc_STR("set_memory_threshold(size: int)")}, + {NULL, NULL} /* sentinel */ +}; + +PyDoc_STRVAR(tracemalloc_pytask_doc, +"Task(func, *args, **kw)"); + +static PyTypeObject TaskType = { + /* The ob_type field must be initialized in the module init function + * to be portable to Windows without using C++. */ + PyVarObject_HEAD_INIT(NULL, 0) + "tracemalloc.Task", /*tp_name*/ + sizeof(TaskObject), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + /* methods */ + (destructor)pytask_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + (reprfunc)pytask_repr, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ + tracemalloc_pytask_doc, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + tracemalloc_pytask_methods, /*tp_methods*/ + 0, /*tp_members*/ + tracemalloc_pytask_getset, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + (initproc)pytask_init, /*tp_init*/ + PyType_GenericAlloc, /*tp_alloc*/ + pytask_new, /*tp_new*/ + PyObject_Del, /*tp_free*/ + 0, /*tp_is_gc*/ +}; + +static void +task_list_check(void) +{ + Py_ssize_t i, len; + TaskObject *obj; + int res, reset_reentrant; + + len = PyList_GET_SIZE(tracemalloc_tasks); + for (i=0; itask.enabled) + task_check(&obj->task); + } + + reset_reentrant = 0; + for (i=len-1; i>=0; i--) { + obj = (TaskObject *)PyList_GET_ITEM(tracemalloc_tasks, i); + if (!obj->task.failed) + continue; + + if (!reset_reentrant) { + /* the list may be reallocated to be shrinked */ + set_reentrant(1); + reset_reentrant = 1; + } + + res = PyList_SetSlice(tracemalloc_tasks, i, i+1, NULL); + if (res < 0) { + PyErr_WriteUnraisable(NULL); + break; + } + } + if (reset_reentrant) + set_reentrant(0); +} + +static int +task_list_clear(void) +{ + Py_ssize_t len; + int res; + + /* task_call_later() may have scheduled calls to a task which will be + destroyed: ensure that the pending call list is empty */ + if (Py_MakePendingCalls() < 0) + return -1; + + len = PyList_GET_SIZE(tracemalloc_tasks); + + set_reentrant(1); + res = PyList_SetSlice(tracemalloc_tasks, 0, len, NULL); + set_reentrant(0); + + return res; +} + +static PyObject* +py_tracemalloc_is_enabled(PyObject *self) +{ + return PyBool_FromLong(tracemalloc_config.enabled); +} + +PyDoc_STRVAR(tracemalloc_clear_traces_doc, + "clear_traces()\n" + "\n" + "Clear all traces and statistics of memory allocations."); + +static PyObject* +py_tracemalloc_clear_traces(PyObject *self) +{ + if (!tracemalloc_config.enabled) + Py_RETURN_NONE; + + set_reentrant(1); + tracemalloc_clear_traces(); + set_reentrant(0); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_stats_doc, +"get_stats() -> dict\n" +"\n" +"Get statistics on Python memory allocations per Python filename and per\n" +"line number.\n" +"\n" +"Return a dictionary {filename: str -> {line_number: int -> stats}}\n" +"where stats in a (size: int, count: int) tuple.\n" +"\n" +"Return an empty dictionary if the module tracemalloc is disabled."); + +static PyObject* +tracemalloc_get_stats(PyObject *self) +{ + get_stats_t get_stats; + int err; + + if (!tracemalloc_config.enabled) + return PyDict_New(); + + pool_init(&get_stats.pool, tracemalloc_file_stats->pool->item_size); + get_stats.file_dict = PyDict_New(); + if (get_stats.file_dict == NULL) + goto error; + + TABLES_LOCK(); + get_stats.file_stats = hash_copy_with_pool(tracemalloc_file_stats, + &get_stats.pool); + TABLES_UNLOCK(); + + if (get_stats.file_stats == NULL) { + PyErr_NoMemory(); + goto error; + } + + set_reentrant(1); + err = hash_foreach(get_stats.file_stats, + tracemalloc_get_stats_fill_file, &get_stats); + set_reentrant(0); + if (err) { + goto error; + } + + goto finally; + +error: + Py_CLEAR(get_stats.file_dict); + +finally: + if (get_stats.file_stats != NULL) + hash_destroy(get_stats.file_stats); +#ifdef USE_MEMORY_POOL + pool_clear(&get_stats.pool); +#endif + return get_stats.file_dict; +} + +static PyObject* +frame_to_pyobject(frame_t *frame) +{ + PyObject *frame_obj, *lineno_obj; + + frame_obj = PyTuple_New(2); + if (frame_obj == NULL) + return NULL; + + if (frame->filename == NULL) + frame->filename = Py_None; + Py_INCREF(frame->filename); + PyTuple_SET_ITEM(frame_obj, 0, frame->filename); + + lineno_obj = tracemalloc_lineno_as_obj(frame->lineno); + if (lineno_obj == NULL) { + Py_DECREF(frame_obj); + return NULL; + } + PyTuple_SET_ITEM(frame_obj, 1, lineno_obj); + + return frame_obj; +} + +static PyObject* +traceback_to_pyobject(traceback_t *traceback, hash_t *intern_table) +{ + int i; + PyObject *frames, *frame; + + if (intern_table != NULL) { + if (HASH_GET_DATA(intern_table, traceback, frames)) { + Py_INCREF(frames); + return frames; + } + } + + frames = PyTuple_New(traceback->nframe); + if (frames == NULL) + return NULL; + + for (i=0; i < traceback->nframe; i++) { + frame = frame_to_pyobject(&traceback->frames[i]); + if (frame == NULL) { + Py_DECREF(frames); + return NULL; + } + PyTuple_SET_ITEM(frames, i, frame); + } + + if (intern_table != NULL) { + if (HASH_PUT_DATA(intern_table, traceback, frames) < 0) { + Py_DECREF(frames); + PyErr_NoMemory(); + return NULL; + } + /* intern_table keeps a new reference to frames */ + Py_INCREF(frames); + } + return frames; +} + +static PyObject* +trace_to_pyobject(trace_t *trace, hash_t *intern_tracebacks) +{ + PyObject *trace_obj = NULL; + PyObject *size, *traceback; + + trace_obj = PyTuple_New(2); + if (trace_obj == NULL) + return NULL; + + size = PyLong_FromSize_t(trace->size); + if (size == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyTuple_SET_ITEM(trace_obj, 0, size); + + traceback = traceback_to_pyobject(trace->traceback, intern_tracebacks); + if (traceback == NULL) { + Py_DECREF(trace_obj); + return NULL; + } + PyTuple_SET_ITEM(trace_obj, 1, traceback); + + return trace_obj; +} + +typedef struct { + pool_t tracebacks_pool; + pool_t traces_pool; + hash_t *traces; + hash_t *tracebacks; + PyObject *dict; +} get_traces_t; + +static int +tracemalloc_get_traces_fill(hash_entry_t *entry, void *user_data) +{ + get_traces_t *get_traces = user_data; + const void *ptr; + trace_t *trace; + PyObject *key_obj, *tracemalloc_obj; + int res; + + ptr = entry->key; + trace = (trace_t *)ENTRY_DATA_PTR(entry); + + key_obj = PyLong_FromVoidPtr((void *)ptr); + if (key_obj == NULL) + return 1; + + tracemalloc_obj = trace_to_pyobject(trace, get_traces->tracebacks); + if (tracemalloc_obj == NULL) { + Py_DECREF(key_obj); + return 1; + } + + res = PyDict_SetItem(get_traces->dict, key_obj, tracemalloc_obj); + Py_DECREF(key_obj); + Py_DECREF(tracemalloc_obj); + if (res < 0) + return 1; + + return 0; +} + +static int +tracemalloc_pyobject_decref_cb(hash_entry_t *entry, void *user_data) +{ + PyObject *obj = (PyObject *)HASH_ENTRY_DATA_AS_VOID_P(entry); + Py_DECREF(obj); + return 0; +} + +PyDoc_STRVAR(tracemalloc_get_traces_doc, +"get_stats() -> dict\n" +"\n" +"Get all traces of allocated Python memory blocks.\n" +"Return a dictionary: {pointer: int -> trace: structseq).\n" +"Return an empty dictionary if the tracemalloc module is disabled."); + +static PyObject* +py_tracemalloc_get_traces(PyObject *self, PyObject *obj) +{ + get_traces_t get_traces; + int err; + + if (!tracemalloc_config.enabled) + return PyDict_New(); + + pool_init(&get_traces.tracebacks_pool, + sizeof(hash_entry_t) + sizeof(PyObject *)); + pool_init(&get_traces.traces_pool, tracemalloc_traces->pool->item_size); + get_traces.traces = NULL; + get_traces.tracebacks = NULL; + get_traces.dict = PyDict_New(); + if (get_traces.dict == NULL) + goto error; + + get_traces.tracebacks = hash_new(&get_traces.tracebacks_pool, + sizeof(PyObject *), + key_hash_int, key_cmp_direct); + if (get_traces.tracebacks == NULL) { + PyErr_NoMemory(); + goto error; + } + + TABLES_LOCK(); + /* allocate the exact number of traces */ + if (pool_alloc(&get_traces.traces_pool, tracemalloc_traces->entries) < 0) + get_traces.traces = NULL; + else + get_traces.traces = hash_copy_with_pool(tracemalloc_traces, + &get_traces.traces_pool); + TABLES_UNLOCK(); + + if (get_traces.traces == NULL) { + PyErr_NoMemory(); + goto error; + } + + set_reentrant(1); + err = hash_foreach(get_traces.traces, + tracemalloc_get_traces_fill, &get_traces); + set_reentrant(0); + if (err) + goto error; + + goto finally; + +error: + Py_CLEAR(get_traces.dict); + +finally: + if (get_traces.tracebacks != NULL) { + hash_foreach(get_traces.tracebacks, + tracemalloc_pyobject_decref_cb, NULL); + hash_destroy(get_traces.tracebacks); + } + if (get_traces.traces != NULL) + hash_destroy(get_traces.traces); +#ifdef USE_MEMORY_POOL + pool_clear(&get_traces.traces_pool); + pool_clear(&get_traces.tracebacks_pool); +#endif + return get_traces.dict; +} + +void* +tracemalloc_get_object_address(PyObject *obj) +{ + PyTypeObject *type = Py_TYPE(obj); + if (PyType_IS_GC(type)) + return (void *)((char *)obj - sizeof(PyGC_Head)); + else + return (void *)obj; +} + +PyDoc_STRVAR(tracemalloc_get_object_address_doc, +"get_object_address(obj) -> int\n" +"\n" +"Return the address of the memory block of the specified\n" +"Python object."); + +static PyObject* +py_tracemalloc_get_object_address(PyObject *self, PyObject *obj) +{ + void *ptr = tracemalloc_get_object_address(obj); + return PyLong_FromVoidPtr(ptr); +} + +static PyObject* +tracemalloc_get_trace(void *ptr) +{ + trace_t trace; + int found; + + if (!tracemalloc_config.enabled) + Py_RETURN_NONE; + + TABLES_LOCK(); + found = HASH_GET_DATA(tracemalloc_traces, ptr, trace); + TABLES_UNLOCK(); + + if (found) + return trace_to_pyobject(&trace, NULL); + else + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_trace_doc, +"get_trace(address) -> trace\n" +"\n" +"Get the trace of the Python memory block allocated at specified address."); + +static PyObject* +py_tracemalloc_get_trace(PyObject *self, PyObject *obj) +{ + void *ptr = PyLong_AsVoidPtr(obj); + if (ptr == NULL && PyErr_Occurred()) + return NULL; + + return tracemalloc_get_trace(ptr); +} + +PyDoc_STRVAR(tracemalloc_get_object_trace_doc, +"get_object_trace(obj) -> trace\n" +"\n" +"Get the trace of the Python object 'obj' as trace structseq.\n" +"Return None if tracemalloc module did not save the location\n" +"when the object was allocated, for example if tracemalloc was disabled."); + +static PyObject* +py_tracemalloc_get_object_trace(PyObject *self, PyObject *obj) +{ + void *ptr; + + ptr = tracemalloc_get_object_address(obj); + + return tracemalloc_get_trace(ptr); +} + +static int +tracemalloc_atexit_register(PyObject *module) +{ + PyObject *method = NULL, *atexit = NULL, *func = NULL; + PyObject *result; + int ret = -1; + + method = PyObject_GetAttrString(module, "_atexit"); + if (method == NULL) + goto done; + + atexit = PyImport_ImportModule("atexit"); + if (atexit == NULL) { + if (!PyErr_Warn(PyExc_ImportWarning, + "atexit module is missing: " + "cannot automatically disable tracemalloc at exit")) + { + PyErr_Clear(); + return 0; + } + goto done; + } + + func = PyObject_GetAttrString(atexit, "register"); + if (func == NULL) + goto done; + + result = PyObject_CallFunction(func, "O", method); + if (result == NULL) + goto done; + Py_DECREF(result); + + ret = 0; + +done: + Py_XDECREF(method); + Py_XDECREF(func); + Py_XDECREF(atexit); + return ret; +} + +PyDoc_STRVAR(tracemalloc_cancel_tasks_doc, + "cancel_tasks()\n" + "\n" + "Stop scheduled tasks."); + +PyObject* +tracemalloc_cancel_tasks(PyObject *module) +{ + int res; + + res = task_list_clear(); + if (res < 0) + return NULL; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_tasks_doc, + "get_tasks()\n" + "\n" + "Get the list of scheduled tasks."); + +PyObject* +tracemalloc_get_tasks(PyObject *module) +{ + PyObject *list; + + list = PyList_New(0); + if (list == NULL) + return NULL; + + if (_PyList_Extend((PyListObject*)list, tracemalloc_tasks) < 0) { + Py_DECREF(list); + return NULL; + } + return list; +} + +PyDoc_STRVAR(tracemalloc_enable_doc, + "enable()\n" + "\n" + "Start tracing Python memory allocations."); + +static PyObject* +py_tracemalloc_enable(PyObject *self) +{ + if (tracemalloc_enable() < 0) + return NULL; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_disable_doc, + "disable()\n" + "\n" + "Stop tracing Python memory allocations."); + +static PyObject* +py_tracemalloc_disable(PyObject *self) +{ + if (tracemalloc_disable() < 0) + return NULL; + + Py_RETURN_NONE; +} + +static PyObject* +tracemalloc_atexit(PyObject *self) +{ + assert(PyGILState_Check()); + tracemalloc_deinit(); + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_traceback_limit_doc, + "get_traceback_limit() -> int\n" + "\n" + "Get the maximum number of frames stored in a trace of a memory\n" + "allocation."); + +static PyObject* +py_tracemalloc_get_traceback_limit(PyObject *self) +{ + return PyLong_FromLong(tracemalloc_config.max_nframe); +} + +PyDoc_STRVAR(tracemalloc_set_traceback_limit_doc, + "set_traceback_limit(nframe: int)\n" + "\n" + "Set the maximum number of frames stored in the traceback attribute\n" + "of a trace of a memory allocation.\n" + "\n" + "If the tracemalloc is enabled, all traces and statistics of memory\n" + "allocations are cleared."); + +static PyObject* +tracemalloc_set_traceback_limit(PyObject *self, PyObject *args) +{ + int nframe; + + if (!PyArg_ParseTuple(args, "i:set_traceback_limit", + &nframe)) + return NULL; + + if (nframe < 0 || nframe > MAX_NFRAME) { + PyErr_Format(PyExc_ValueError, + "the number of frames must be in range [0; %i]", + MAX_NFRAME); + return NULL; + } + tracemalloc_config.max_nframe = nframe; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_get_tracemalloc_memory_doc, + "get_tracemalloc_memory() -> int\n" + "\n" + "Get the memory usage in bytes of the _tracemalloc module."); + +static PyObject* +tracemalloc_get_tracemalloc_memory(PyObject *self) +{ + mem_stats_t stats; + PyObject *size_obj, *free_obj; + + memset(&stats, 0, sizeof(stats)); + +#ifdef PRINT_STATS + hash_print_stats(tracemalloc_filenames); + hash_print_stats(tracemalloc_tracebacks); + hash_print_stats(tracemalloc_arenas); + TABLES_LOCK(); + hash_print_stats(tracemalloc_traces); + hash_print_stats(tracemalloc_file_stats); + TABLES_UNLOCK(); + + pool_print_stats(&tracemalloc_pools.no_data); + TABLES_LOCK(); + pool_print_stats(&tracemalloc_pools.traces); + pool_print_stats(&tracemalloc_pools.hash_tables); + TABLES_UNLOCK(); +#endif + + /* hash tables */ + hash_mem_stats(tracemalloc_tracebacks, &stats); + hash_mem_stats(tracemalloc_filenames, &stats); + hash_mem_stats(tracemalloc_arenas, &stats); + + TABLES_LOCK(); + hash_mem_stats(tracemalloc_traces, &stats); + hash_mem_stats(tracemalloc_file_stats, &stats); + TABLES_UNLOCK(); + + /* memory pools */ + pool_mem_stats(&tracemalloc_pools.no_data, &stats); + pool_mem_stats(&tracemalloc_pools.traces, &stats); + pool_mem_stats(&tracemalloc_pools.hash_tables, &stats); + + size_obj = PyLong_FromSize_t(stats.data + stats.free); + free_obj = PyLong_FromSize_t(stats.free); + return Py_BuildValue("NN", size_obj, free_obj); +} + +PyDoc_STRVAR(tracemalloc_get_traced_memory_doc, + "get_traced_memory() -> int\n" + "\n" + "Get the total size in bytes of all memory blocks allocated\n" + "by Python currently."); + +static PyObject* +tracemalloc_get_traced_memory(PyObject *self) +{ + size_t size, max_size; + PyObject *size_obj, *max_size_obj; + + TABLES_LOCK(); + size = tracemalloc_traced_memory; + max_size = tracemalloc_max_traced_memory; + TABLES_UNLOCK(); + + size_obj = PyLong_FromSize_t(size); + max_size_obj = PyLong_FromSize_t(max_size); + return Py_BuildValue("NN", size_obj, max_size_obj); +} + +PyDoc_STRVAR(tracemalloc_get_arena_size_doc, + "get_arena_size() -> int\n" + "\n" + "Get the total size in bytes of all arenas."); + +static PyObject* +tracemalloc_get_arena_size(PyObject *self) +{ + return PyLong_FromSize_t(tracemalloc_arena_size); +} + +#ifdef WITH_PYMALLOC +PyDoc_STRVAR(tracemalloc_get_pymalloc_stats_doc, + "get_pymalloc_stats() -> dict\n" + "\n" + "Get statistics on pymalloc allocator."); + +static PyObject* +tracemalloc_get_pymalloc_stats(PyObject *self) +{ + pymalloc_stats_t stats; + PyObject *dict, *value; + int err; + + _PyObject_GetMallocStats(&stats); + + dict = PyDict_New(); + if (dict == NULL) + return NULL; + +#define ADD_STAT(NAME) \ + do { \ + value = PyLong_FromSize_t(stats.NAME); \ + if (value == NULL) \ + goto error; \ + err = PyDict_SetItemString(dict, #NAME, value); \ + Py_DECREF(value); \ + if (err < 0) \ + goto error; \ + } while (0) + + ADD_STAT(alignment); + ADD_STAT(threshold); + ADD_STAT(nalloc); + ADD_STAT(arena_size); + ADD_STAT(total_arenas); + ADD_STAT(max_arenas); + ADD_STAT(arenas); + ADD_STAT(allocated_bytes); + ADD_STAT(available_bytes); + ADD_STAT(pool_size); + ADD_STAT(free_pools); + ADD_STAT(pool_headers); + ADD_STAT(quantization); + ADD_STAT(arena_alignment); + +#undef ADD_STAT + + return dict; + +error: + Py_DECREF(dict); + return NULL; +} +#endif + +static int +tracemalloc_add_filter(filter_t *filter) +{ + size_t i, nfilter; + filter_list_t *filters; + filter_t *new_filters; + + if (filter->include) + filters = &tracemalloc_include_filters; + else + filters = &tracemalloc_exclude_filters; + + for(i=0; infilter; i++) { + if (filter_compare(&filters->filters[i], filter) == 1) { + /* filter already present, don't add a duplicate */ + return 0; + } + } + + nfilter = (filters->nfilter + 1); + new_filters = raw_realloc(filters->filters, nfilter * sizeof(filter_t)); + if (new_filters == NULL) { + PyErr_NoMemory(); + return -1; + } + + Py_INCREF(filter->pattern); + new_filters[filters->nfilter] = *filter; + + filters->nfilter = nfilter; + filters->filters = new_filters; + return 0; +} + +PyDoc_STRVAR(tracemalloc_get_unicode_interned_doc, + "get_unicode_interned() -> (int, int)\n" + "\n" + "Get the length and the size in bytes of dictionary of Unicode\n" + "interned strings. Return a (length, size) tuple."); + +static PyObject* +tracemalloc_get_unicode_interned(PyObject *self) +{ + extern PyObject* _PyUnicode_GetInterned(void); + PyObject *interned; + PyObject *size; + Py_ssize_t len; + _Py_IDENTIFIER(__sizeof__); + + interned = _PyUnicode_GetInterned(); + if (interned == NULL || !PyDict_Check(interned)) { + PyErr_SetString(PyExc_ValueError, "unicode interned is not a dictionary"); + return NULL; + } + + len = PyDict_Size(interned); + if (len == -1) + return NULL; + + size = _PyObject_CallMethodId(interned, &PyId___sizeof__, NULL); + if (size == NULL) + return NULL; + + return Py_BuildValue("(Nn)", size, len); +} + +PyDoc_STRVAR(tracemalloc_get_allocated_blocks_doc, + "get_allocated_blocks() -> int\n" + "\n" + "Get the current number of allocated memory blocks."); + +static PyObject* +tracemalloc_get_allocated_blocks(PyObject *self) +{ + return PyLong_FromSsize_t(_Py_GetAllocatedBlocks()); +} + +PyDoc_STRVAR(tracemalloc_add_filter_doc, + "add_filter(include: bool, filename: str, lineno: int=None, traceback: bool=True)\n" + "\n" + "Add a filter. If include is True, only trace memory blocks allocated\n" + "in a file with a name matching filename at line number lineno. If\n" + "include is True, don't trace memory blocks allocated in a file with a\n" + "name matching filename at line number lineno.\n" + "\n" + "The filename can contain one or many '*' joker characters which\n" + "matchs any substring, including an empty string. The '.pyc' and '.pyo'\n" + "suffixes are automatically replaced with '.py'. On Windows, the\n" + "comparison is case insensitive and the alternative separator '/' is\n" + "replaced with the standard separator '\'.\n" + "\n" + "If lineno is None or lesser than 1, it matches any line number."); + +static PyObject* +py_tracemalloc_add_filter(PyObject *self, PyObject *args) +{ + FilterObject *pyfilter; + + if (!PyArg_ParseTuple(args, "O!:add_filter", + &FilterType, (PyObject **)&pyfilter)) + return NULL; + + if (tracemalloc_add_filter(&pyfilter->filter) < 0) + return NULL; + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_add_include_filter_doc, + "add_include_filter(filename: str, lineno: int=None, traceback: bool=False)"); + +static PyObject* +tracemalloc_add_include_filter(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"filename", "lineno", "traceback", 0}; + PyObject *filename; + int lineno = -1; + int traceback = 0; + filter_t filter; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "U|O&i:add_include_filter", kwlist, + &filename, + &tracemalloc_parse_lineno, &lineno, + &traceback)) + return NULL; + + if (filter_init(&filter, 1, filename, lineno, traceback) < 0) + return NULL; + + if (tracemalloc_add_filter(&filter) < 0) { + filter_deinit(&filter); + return NULL; + } + filter_deinit(&filter); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(tracemalloc_add_exclude_filter_doc, + "add_exclude_filter(filename: str, lineno: int=None, traceback: bool=False)"); + +static PyObject* +tracemalloc_add_exclude_filter(PyObject *self, PyObject *args, PyObject *kwds) +{ + static char *kwlist[] = {"filename", "lineno", "traceback", 0}; + PyObject *filename; + int lineno = -1; + int traceback = 0; + filter_t filter; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, "U|O&i:add_exclude_filter", kwlist, + &filename, + &tracemalloc_parse_lineno, &lineno, + &traceback)) + return NULL; + + if (filter_init(&filter, 0, filename, lineno, traceback) < 0) + return NULL; + + if (tracemalloc_add_filter(&filter) < 0) { + filter_deinit(&filter); + return NULL; + } + filter_deinit(&filter); + + Py_RETURN_NONE; +} + +static PyObject* +filter_as_obj(filter_t *filter) +{ + FilterObject *pyfilter; + + pyfilter = PyObject_New(FilterObject, &FilterType); + if (pyfilter == NULL) + return NULL; + + Py_INCREF(filter->pattern); + pyfilter->filter = *filter; + return (PyObject *)pyfilter; +} + +PyDoc_STRVAR(tracemalloc_get_filters_doc, + "get_filters()\n" + "\n" + "Get the filters as list of (include: bool, filename: str, lineno: int)" + "tuples.\n" + "\n" + "If *lineno* is ``None``, a filter matchs any line number."); + +static size_t +tracemalloc_get_filters(PyObject *list, size_t first_index, + filter_list_t *filters) +{ + size_t i; + filter_t *filter; + PyObject *pyfilter; + + for (i=0; infilter; i++) { + filter = &filters->filters[i]; + + pyfilter = filter_as_obj(filter); + if (pyfilter == NULL) + return (size_t)-1; + + PyList_SET_ITEM(list, first_index + i, pyfilter); + } + return filters->nfilter; +} + +static PyObject* +py_tracemalloc_get_filters(PyObject *self) +{ + PyObject *filters = NULL; + size_t number; + + filters = PyList_New(tracemalloc_include_filters.nfilter + + tracemalloc_exclude_filters.nfilter); + if (filters == NULL) + return NULL; + + number = tracemalloc_get_filters(filters, 0, &tracemalloc_include_filters); + if (number == (size_t)-1) { + Py_DECREF(filters); + return NULL; + } + + number = tracemalloc_get_filters(filters, number, &tracemalloc_exclude_filters); + if (number == (size_t)-1) { + Py_DECREF(filters); + return NULL; + } + + return filters; +} + +PyDoc_STRVAR(tracemalloc_clear_filters_doc, + "clear_filters()\n" + "\n" + "Reset the filter list."); + +static PyObject* +py_tracemalloc_clear_filters(PyObject *self) +{ + tracemalloc_clear_filters(); + Py_RETURN_NONE; +} + +#ifdef MS_WINDOWS +PyDoc_STRVAR(tracemalloc_get_process_memory_doc, + "get_process_memory()\n" + "\n" + "Get the memory usage of the current process as a tuple:\n" + "(rss: int, vms: int)."); + +static PyObject* +tracemalloc_get_process_memory(PyObject *self) +{ + typedef BOOL (_stdcall *GPMI_PROC) (HANDLE Process, PPROCESS_MEMORY_COUNTERS ppsmemCounters, DWORD cb); + static HINSTANCE psapi = NULL; + static GPMI_PROC GetProcessMemoryInfo = NULL; + HANDLE hProcess; + PROCESS_MEMORY_COUNTERS counters; + PyObject *rss, *vms; + + if (psapi == NULL) { + psapi = LoadLibraryW(L"psapi.dll"); + if (psapi == NULL) + return PyErr_SetFromWindowsErr(0); + } + + if (GetProcessMemoryInfo == NULL) { + GetProcessMemoryInfo = (GPMI_PROC)GetProcAddress(psapi, "GetProcessMemoryInfo"); + if (GetProcessMemoryInfo == NULL) + return PyErr_SetFromWindowsErr(0); + } + + hProcess = GetCurrentProcess(); + + if (!GetProcessMemoryInfo(hProcess, &counters, sizeof(counters))) + return PyErr_SetFromWindowsErr(0); + + rss = PyLong_FromSize_t(counters.WorkingSetSize); + if (rss == NULL) + return NULL; + vms = PyLong_FromSize_t(counters.PagefileUsage); + if (vms == NULL) { + Py_DECREF(vms); + return NULL; + } + return Py_BuildValue("NN", rss, vms); +} +#endif + +static PyMethodDef module_methods[] = { + {"is_enabled", (PyCFunction)py_tracemalloc_is_enabled, METH_NOARGS, + PyDoc_STR("is_enabled()->bool")}, + {"clear_traces", (PyCFunction)py_tracemalloc_clear_traces, METH_NOARGS, + tracemalloc_clear_traces_doc}, + {"get_stats", (PyCFunction)tracemalloc_get_stats, METH_NOARGS, + tracemalloc_get_stats_doc}, + {"get_traces", (PyCFunction)py_tracemalloc_get_traces, METH_NOARGS, + tracemalloc_get_traces_doc}, + {"get_object_address", (PyCFunction)py_tracemalloc_get_object_address, METH_O, + tracemalloc_get_object_address_doc}, + {"get_object_trace", (PyCFunction)py_tracemalloc_get_object_trace, METH_O, + tracemalloc_get_object_trace_doc}, + {"get_trace", (PyCFunction)py_tracemalloc_get_trace, METH_O, + tracemalloc_get_trace_doc}, + {"cancel_tasks", (PyCFunction)tracemalloc_cancel_tasks, METH_NOARGS, + tracemalloc_cancel_tasks_doc}, + {"get_tasks", (PyCFunction)tracemalloc_get_tasks, METH_NOARGS, + tracemalloc_get_tasks_doc}, + {"enable", (PyCFunction)py_tracemalloc_enable, METH_NOARGS, + tracemalloc_enable_doc}, + {"disable", (PyCFunction)py_tracemalloc_disable, METH_NOARGS, + tracemalloc_disable_doc}, + {"get_traceback_limit", (PyCFunction)py_tracemalloc_get_traceback_limit, + METH_NOARGS, tracemalloc_get_traceback_limit_doc}, + {"set_traceback_limit", (PyCFunction)tracemalloc_set_traceback_limit, + METH_VARARGS, tracemalloc_set_traceback_limit_doc}, + {"get_tracemalloc_memory", (PyCFunction)tracemalloc_get_tracemalloc_memory, + METH_NOARGS, tracemalloc_get_tracemalloc_memory_doc}, + {"get_traced_memory", (PyCFunction)tracemalloc_get_traced_memory, + METH_NOARGS, tracemalloc_get_traced_memory_doc}, + {"get_arena_size", (PyCFunction)tracemalloc_get_arena_size, + METH_NOARGS, tracemalloc_get_arena_size_doc}, + {"get_unicode_interned", (PyCFunction)tracemalloc_get_unicode_interned, + METH_NOARGS, tracemalloc_get_unicode_interned_doc}, + {"get_allocated_blocks", (PyCFunction)tracemalloc_get_allocated_blocks, + METH_NOARGS, tracemalloc_get_allocated_blocks_doc}, + {"add_filter", (PyCFunction)py_tracemalloc_add_filter, + METH_VARARGS, tracemalloc_add_filter_doc}, + {"add_include_filter", (PyCFunction)tracemalloc_add_include_filter, + METH_VARARGS | METH_KEYWORDS, tracemalloc_add_include_filter_doc}, + {"add_exclude_filter", (PyCFunction)tracemalloc_add_exclude_filter, + METH_VARARGS | METH_KEYWORDS, tracemalloc_add_exclude_filter_doc}, + {"get_filters", (PyCFunction)py_tracemalloc_get_filters, METH_NOARGS, + tracemalloc_get_filters_doc}, + {"clear_filters", (PyCFunction)py_tracemalloc_clear_filters, METH_NOARGS, + tracemalloc_clear_filters_doc}, +#ifdef MS_WINDOWS + {"get_process_memory", (PyCFunction)tracemalloc_get_process_memory, METH_NOARGS, + tracemalloc_get_process_memory_doc}, +#endif +#ifdef WITH_PYMALLOC + {"get_pymalloc_stats", (PyCFunction)tracemalloc_get_pymalloc_stats, + METH_NOARGS, tracemalloc_get_pymalloc_stats_doc}, +#endif + + /* private functions */ + {"_atexit", (PyCFunction)tracemalloc_atexit, METH_NOARGS}, + + /* sentinel */ + {NULL, NULL} +}; + +PyDoc_STRVAR(module_doc, +"_tracemalloc module."); + +static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "_tracemalloc", + module_doc, + 0, /* non-negative size to be able to unload the module */ + module_methods, + NULL, +}; + +PyMODINIT_FUNC +PyInit__tracemalloc(void) +{ + PyObject *m; + m = PyModule_Create(&module_def); + if (m == NULL) + return NULL; + + if (tracemalloc_init() < 0) + return NULL; + + if (PyType_Ready(&FilterType) < 0) + return NULL; + if (PyType_Ready(&TaskType) < 0) + return NULL; + + Py_INCREF((PyObject*) &FilterType); + PyModule_AddObject(m, "Filter", (PyObject*)&FilterType); + Py_INCREF((PyObject*) &TaskType); + PyModule_AddObject(m, "Task", (PyObject*)&TaskType); + + if (tracemalloc_atexit_register(m) < 0) + return NULL; + return m; +} + +int +_PyTraceMalloc_Init(void) +{ + char *p; + PyObject *xoptions, *key; + int has_key; + + if ((p = Py_GETENV("PYTHONTRACEMALLOC")) && *p != '\0') { + /* enable */ + } + else { + xoptions = PySys_GetXOptions(); + if (xoptions == NULL) + return -1; + + key = PyUnicode_FromString("tracemalloc"); + if (key == NULL) + return -1; + + has_key = PyDict_Contains(xoptions, key); + Py_DECREF(key); + if (has_key < 0) + return -1; + if (!has_key) + return 0; + } + + assert(PyGILState_Check()); + return tracemalloc_enable(); +} + diff -r 50e4d0c62c2b -r ec121a72e848 Modules/readline.c --- a/Modules/readline.c Sun Oct 06 13:24:52 2013 +0200 +++ b/Modules/readline.c Sun Oct 06 17:53:23 2013 +0200 @@ -1176,7 +1176,7 @@ call_readline(FILE *sys_stdin, FILE *sys /* We got an EOF, return a empty string. */ if (p == NULL) { - p = PyMem_Malloc(1); + p = PyMem_RawMalloc(1); if (p != NULL) *p = '\0'; RESTORE_LOCALE(saved_locale) @@ -1204,7 +1204,7 @@ call_readline(FILE *sys_stdin, FILE *sys /* Copy the malloc'ed buffer into a PyMem_Malloc'ed one and release the original. */ q = p; - p = PyMem_Malloc(n+2); + p = PyMem_RawMalloc(n+2); if (p != NULL) { strncpy(p, q, n); p[n] = '\n'; diff -r 50e4d0c62c2b -r ec121a72e848 Objects/codeobject.c --- a/Objects/codeobject.c Sun Oct 06 13:24:52 2013 +0200 +++ b/Objects/codeobject.c Sun Oct 06 17:53:23 2013 +0200 @@ -74,6 +74,8 @@ PyCode_New(int argcount, int kwonlyargco PyErr_BadInternalCall(); return NULL; } + if (PyUnicode_READY(filename) < 0) + return NULL; n_cellvars = PyTuple_GET_SIZE(cellvars); intern_strings(names); intern_strings(varnames); diff -r 50e4d0c62c2b -r ec121a72e848 Objects/obmalloc.c --- a/Objects/obmalloc.c Sun Oct 06 13:24:52 2013 +0200 +++ b/Objects/obmalloc.c Sun Oct 06 17:53:23 2013 +0200 @@ -125,10 +125,11 @@ static void #define PYRAW_FUNCS _PyMem_RawMalloc, _PyMem_RawRealloc, _PyMem_RawFree #ifdef WITH_PYMALLOC -#define PYOBJECT_FUNCS _PyObject_Malloc, _PyObject_Realloc, _PyObject_Free +# define PYOBJ_FUNCS _PyObject_Malloc, _PyObject_Realloc, _PyObject_Free #else -#define PYOBJECT_FUNCS PYRAW_FUNCS +# define PYOBJ_FUNCS PYRAW_FUNCS #endif +#define PYMEM_FUNCS PYRAW_FUNCS #ifdef PYMALLOC_DEBUG typedef struct { @@ -142,16 +143,16 @@ static struct { debug_alloc_api_t obj; } _PyMem_Debug = { {'r', {NULL, PYRAW_FUNCS}}, - {'m', {NULL, PYRAW_FUNCS}}, - {'o', {NULL, PYOBJECT_FUNCS}} + {'m', {NULL, PYMEM_FUNCS}}, + {'o', {NULL, PYOBJ_FUNCS}} }; -#define PYDEBUG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugRealloc, _PyMem_DebugFree +#define PYDBG_FUNCS _PyMem_DebugMalloc, _PyMem_DebugRealloc, _PyMem_DebugFree #endif static PyMemAllocator _PyMem_Raw = { #ifdef PYMALLOC_DEBUG - &_PyMem_Debug.raw, PYDEBUG_FUNCS + &_PyMem_Debug.raw, PYDBG_FUNCS #else NULL, PYRAW_FUNCS #endif @@ -159,23 +160,24 @@ static PyMemAllocator _PyMem_Raw = { static PyMemAllocator _PyMem = { #ifdef PYMALLOC_DEBUG - &_PyMem_Debug.mem, PYDEBUG_FUNCS + &_PyMem_Debug.mem, PYDBG_FUNCS #else - NULL, PYRAW_FUNCS + NULL, PYMEM_FUNCS #endif }; static PyMemAllocator _PyObject = { #ifdef PYMALLOC_DEBUG - &_PyMem_Debug.obj, PYDEBUG_FUNCS + &_PyMem_Debug.obj, PYDBG_FUNCS #else - NULL, PYOBJECT_FUNCS + NULL, PYOBJ_FUNCS #endif }; #undef PYRAW_FUNCS -#undef PYOBJECT_FUNCS -#undef PYDEBUG_FUNCS +#undef PYMEM_FUNCS +#undef PYOBJ_FUNCS +#undef PYDBG_FUNCS static PyObjectArenaAllocator _PyObject_Arena = {NULL, #ifdef MS_WINDOWS @@ -924,7 +926,7 @@ new_arena(void) return NULL; /* overflow */ #endif nbytes = numarenas * sizeof(*arenas); - arenaobj = (struct arena_object *)PyMem_Realloc(arenas, nbytes); + arenaobj = (struct arena_object *)PyMem_RawRealloc(arenas, nbytes); if (arenaobj == NULL) return NULL; arenas = arenaobj; @@ -1309,7 +1311,7 @@ redirect: * has been reached. */ { - void *result = PyMem_Malloc(nbytes); + void *result = PyMem_RawMalloc(nbytes); if (!result) _Py_AllocatedBlocks--; return result; @@ -1539,7 +1541,7 @@ static void redirect: #endif /* We didn't allocate this address. */ - PyMem_Free(p); + PyMem_RawFree(p); } /* realloc. If p is NULL, this acts like malloc(nbytes). Else if nbytes==0, @@ -1608,14 +1610,14 @@ static void * * at p. Instead we punt: let C continue to manage this block. */ if (nbytes) - return PyMem_Realloc(p, nbytes); + return PyMem_RawRealloc(p, nbytes); /* C doesn't define the result of realloc(p, 0) (it may or may not * return NULL then), but Python's docs promise that nbytes==0 never * returns NULL. We don't pass 0 to realloc(), to avoid that endcase * to begin with. Even then, we can't be sure that realloc() won't * return NULL. */ - bp = PyMem_Realloc(p, 1); + bp = PyMem_RawRealloc(p, 1); return bp ? bp : p; } @@ -2066,50 +2068,41 @@ void #ifdef WITH_PYMALLOC -/* Print summary info to "out" about the state of pymalloc's structures. - * In Py_DEBUG mode, also perform some expensive internal consistency - * checks. - */ +typedef struct { + /* Number of size classes */ + uint numclasses; + + /* Number of pools, allocated blocks, and free blocks per class index */ + size_t numpools[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT]; + size_t numblocks[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT]; + size_t numfreeblocks[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT]; +} pymalloc_classes_stats_t; + void -_PyObject_DebugMallocStats(FILE *out) +_PyObject_GetMallocClassStats(pymalloc_stats_t *stats, + pymalloc_classes_stats_t *class_stats) { uint i; const uint numclasses = SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT; - /* # of pools, allocated blocks, and free blocks per class index */ - size_t numpools[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT]; - size_t numblocks[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT]; - size_t numfreeblocks[SMALL_REQUEST_THRESHOLD >> ALIGNMENT_SHIFT]; - /* total # of allocated bytes in used and full pools */ - size_t allocated_bytes = 0; - /* total # of available bytes in used pools */ - size_t available_bytes = 0; - /* # of free pools + pools not yet carved out of current arena */ - uint numfreepools = 0; - /* # of bytes for arena alignment padding */ - size_t arena_alignment = 0; - /* # of bytes in used and full pools used for pool_headers */ - size_t pool_header_bytes = 0; - /* # of bytes in used and full pools wasted due to quantization, - * i.e. the necessarily leftover space at the ends of used and - * full pools. - */ - size_t quantization = 0; /* # of arenas actually allocated. */ size_t narenas = 0; - /* running total -- should equal narenas * ARENA_SIZE */ - size_t total; - char buf[128]; - fprintf(out, "Small block threshold = %d, in %u size classes.\n", - SMALL_REQUEST_THRESHOLD, numclasses); + stats->alignment = ALIGNMENT; + stats->threshold = SMALL_REQUEST_THRESHOLD; + stats->arena_size = ARENA_SIZE; + stats->pool_size = POOL_SIZE; - for (i = 0; i < numclasses; ++i) - numpools[i] = numblocks[i] = numfreeblocks[i] = 0; + class_stats->numclasses = numclasses; + memset(class_stats->numpools, 0, sizeof(class_stats->numpools)); + memset(class_stats->numblocks, 0, sizeof(class_stats->numblocks)); + memset(class_stats->numfreeblocks, 0, sizeof(class_stats->numfreeblocks)); /* Because full pools aren't linked to from anything, it's easiest * to march over all the arenas. If we're lucky, most of the memory * will be living in full pools -- would be a shame to miss them. */ + stats->arena_alignment = 0; + stats->free_pools = 0; for (i = 0; i < maxarenas; ++i) { uint j; uptr base = arenas[i].address; @@ -2119,11 +2112,11 @@ void continue; narenas += 1; - numfreepools += arenas[i].nfreepools; + stats->free_pools += arenas[i].nfreepools; /* round up to pool alignment */ if (base & (uptr)POOL_SIZE_MASK) { - arena_alignment += POOL_SIZE; + stats->arena_alignment += POOL_SIZE; base &= ~(uptr)POOL_SIZE_MASK; base += POOL_SIZE; } @@ -2142,10 +2135,10 @@ void assert(pool_is_in_list(p, arenas[i].freepools)); continue; } - ++numpools[sz]; - numblocks[sz] += p->ref.count; + ++class_stats->numpools[sz]; + class_stats->numblocks[sz] += p->ref.count; freeblocks = NUMBLOCKS(sz) - p->ref.count; - numfreeblocks[sz] += freeblocks; + class_stats->numfreeblocks[sz] += freeblocks; #ifdef Py_DEBUG if (freeblocks > 0) assert(pool_is_in_list(p, usedpools[sz + sz])); @@ -2154,15 +2147,71 @@ void } assert(narenas == narenas_currently_allocated); + stats->allocated_bytes = 0; + stats->available_bytes = 0; + stats->pool_headers = 0; + stats->quantization = 0; + + for (i = 0; i < numclasses; ++i) { + size_t p = class_stats->numpools[i]; + size_t b = class_stats->numblocks[i]; + size_t f = class_stats->numfreeblocks[i]; + uint size = INDEX2SIZE(i); + if (p == 0) { + assert(b == 0 && f == 0); + continue; + } + stats->allocated_bytes += b * size; + stats->available_bytes += f * size; + stats->pool_headers += p * POOL_OVERHEAD; + stats->quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); + } + +#ifdef PYMALLOC_DEBUG + stats->nalloc = serialno; +#else + stats->nalloc = 0; +#endif + stats->total_arenas = ntimes_arena_allocated; + stats->max_arenas = narenas_highwater; + stats->arenas = narenas; +} + +void +_PyObject_GetMallocStats(pymalloc_stats_t *stats) +{ + pymalloc_classes_stats_t class_stats; + _PyObject_GetMallocClassStats(stats, &class_stats); +} + +/* Print summary info to "out" about the state of pymalloc's structures. + * In Py_DEBUG mode, also perform some expensive internal consistency + * checks. + */ +void +_PyObject_DebugMallocStats(FILE *out) +{ + pymalloc_stats_t stats; + pymalloc_classes_stats_t class_stats; + uint i; + /* running total -- should equal narenas * ARENA_SIZE */ + size_t total; + char buf[128]; + + _PyObject_GetMallocClassStats(&stats, &class_stats); + + fprintf(out, "Small block threshold = %d, in %u size classes.\n", + stats.threshold, class_stats.numclasses); + fputc('\n', out); fputs("class size num pools blocks in use avail blocks\n" "----- ---- --------- ------------- ------------\n", out); - for (i = 0; i < numclasses; ++i) { - size_t p = numpools[i]; - size_t b = numblocks[i]; - size_t f = numfreeblocks[i]; + for (i = 0; i < class_stats.numclasses; ++i) { + size_t p = class_stats.numpools[i]; + size_t b = class_stats.numblocks[i]; + size_t f = class_stats.numfreeblocks[i]; uint size = INDEX2SIZE(i); if (p == 0) { assert(b == 0 && f == 0); @@ -2173,37 +2222,35 @@ void "%15" PY_FORMAT_SIZE_T "u " "%13" PY_FORMAT_SIZE_T "u\n", i, size, p, b, f); - allocated_bytes += b * size; - available_bytes += f * size; - pool_header_bytes += p * POOL_OVERHEAD; - quantization += p * ((POOL_SIZE - POOL_OVERHEAD) % size); } + fputc('\n', out); + #ifdef PYMALLOC_DEBUG - (void)printone(out, "# times object malloc called", serialno); + (void)printone(out, "# times object malloc called", stats.nalloc); #endif - (void)printone(out, "# arenas allocated total", ntimes_arena_allocated); - (void)printone(out, "# arenas reclaimed", ntimes_arena_allocated - narenas); - (void)printone(out, "# arenas highwater mark", narenas_highwater); - (void)printone(out, "# arenas allocated current", narenas); + (void)printone(out, "# arenas allocated total", stats.total_arenas); + (void)printone(out, "# arenas reclaimed", stats.total_arenas - stats.arenas); + (void)printone(out, "# arenas highwater mark", stats.max_arenas); + (void)printone(out, "# arenas allocated current", stats.arenas); PyOS_snprintf(buf, sizeof(buf), "%" PY_FORMAT_SIZE_T "u arenas * %d bytes/arena", - narenas, ARENA_SIZE); - (void)printone(out, buf, narenas * ARENA_SIZE); + stats.arenas, stats.arena_size); + (void)printone(out, buf, stats.arenas * stats.arena_size); fputc('\n', out); - total = printone(out, "# bytes in allocated blocks", allocated_bytes); - total += printone(out, "# bytes in available blocks", available_bytes); + total = printone(out, "# bytes in allocated blocks", stats.allocated_bytes); + total += printone(out, "# bytes in available blocks", stats.available_bytes); PyOS_snprintf(buf, sizeof(buf), - "%u unused pools * %d bytes", numfreepools, POOL_SIZE); - total += printone(out, buf, (size_t)numfreepools * POOL_SIZE); + "%u unused pools * %d bytes", stats.free_pools, stats.pool_size); + total += printone(out, buf, (size_t)stats.free_pools * stats.pool_size); - total += printone(out, "# bytes lost to pool headers", pool_header_bytes); - total += printone(out, "# bytes lost to quantization", quantization); - total += printone(out, "# bytes lost to arena alignment", arena_alignment); + total += printone(out, "# bytes lost to pool headers", stats.pool_headers); + total += printone(out, "# bytes lost to quantization", stats.quantization); + total += printone(out, "# bytes lost to arena alignment", stats.arena_alignment); (void)printone(out, "Total", total); } diff -r 50e4d0c62c2b -r ec121a72e848 Objects/unicodeobject.c --- a/Objects/unicodeobject.c Sun Oct 06 13:24:52 2013 +0200 +++ b/Objects/unicodeobject.c Sun Oct 06 17:53:23 2013 +0200 @@ -299,6 +299,12 @@ PyUnicode_GetMax(void) #endif } +PyObject* +_PyUnicode_GetInterned(void) +{ + return interned; +} + #ifdef Py_DEBUG int _PyUnicode_CheckConsistency(PyObject *op, int check_content) diff -r 50e4d0c62c2b -r ec121a72e848 PC/config.c --- a/PC/config.c Sun Oct 06 13:24:52 2013 +0200 +++ b/PC/config.c Sun Oct 06 17:53:23 2013 +0200 @@ -13,6 +13,7 @@ extern PyObject* PyInit_binascii(void); extern PyObject* PyInit_cmath(void); extern PyObject* PyInit_errno(void); extern PyObject* PyInit_faulthandler(void); +extern PyObject* PyInit__tracemalloc(void); extern PyObject* PyInit_gc(void); extern PyObject* PyInit_math(void); extern PyObject* PyInit__md5(void); @@ -102,6 +103,7 @@ struct _inittab _PyImport_Inittab[] = { {"msvcrt", PyInit_msvcrt}, {"_locale", PyInit__locale}, #endif + {"_tracemalloc", PyInit__tracemalloc}, /* XXX Should _winapi go in a WIN32 block? not WIN64? */ {"_winapi", PyInit__winapi}, diff -r 50e4d0c62c2b -r ec121a72e848 PCbuild/pythoncore.vcxproj --- a/PCbuild/pythoncore.vcxproj Sun Oct 06 13:24:52 2013 +0200 +++ b/PCbuild/pythoncore.vcxproj Sun Oct 06 17:53:23 2013 +0200 @@ -528,6 +528,7 @@ + @@ -679,4 +680,4 @@ - \ No newline at end of file + diff -r 50e4d0c62c2b -r ec121a72e848 Parser/myreadline.c --- a/Parser/myreadline.c Sun Oct 06 13:24:52 2013 +0200 +++ b/Parser/myreadline.c Sun Oct 06 17:53:23 2013 +0200 @@ -114,7 +114,8 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE size_t n; char *p, *pr; n = 100; - if ((p = (char *)PyMem_MALLOC(n)) == NULL) + p = (char *)PyMem_RawMalloc(n); + if (p == NULL) return NULL; fflush(sys_stdout); if (prompt) @@ -124,7 +125,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE case 0: /* Normal case */ break; case 1: /* Interrupt */ - PyMem_FREE(p); + PyMem_RawFree(p); return NULL; case -1: /* EOF */ case -2: /* Error */ @@ -136,13 +137,13 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE while (n > 0 && p[n-1] != '\n') { size_t incr = n+2; if (incr > INT_MAX) { - PyMem_FREE(p); + PyMem_RawFree(p); PyErr_SetString(PyExc_OverflowError, "input line too long"); return NULL; } - pr = (char *)PyMem_REALLOC(p, n + incr); + pr = (char *)PyMem_RawRealloc(p, n + incr); if (pr == NULL) { - PyMem_FREE(p); + PyMem_RawFree(p); PyErr_NoMemory(); return NULL; } @@ -151,9 +152,9 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE break; n += strlen(p+n); } - pr = (char *)PyMem_REALLOC(p, n+1); + pr = (char *)PyMem_RawRealloc(p, n+1); if (pr == NULL) { - PyMem_FREE(p); + PyMem_RawFree(p); PyErr_NoMemory(); return NULL; } @@ -174,7 +175,8 @@ char *(*PyOS_ReadlineFunctionPointer)(FI char * PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt) { - char *rv; + char *rv, *res; + size_t len; if (_PyOS_ReadlineTState == PyThreadState_GET()) { PyErr_SetString(PyExc_RuntimeError, @@ -221,5 +223,14 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys _PyOS_ReadlineTState = NULL; - return rv; + if (rv == NULL) + return NULL; + + len = strlen(rv) + 1; + res = PyMem_Malloc(len); + if (res != NULL) + memcpy(res, rv, len); + PyMem_RawFree(rv); + + return res; } diff -r 50e4d0c62c2b -r ec121a72e848 Python/pythonrun.c --- a/Python/pythonrun.c Sun Oct 06 13:24:52 2013 +0200 +++ b/Python/pythonrun.c Sun Oct 06 17:53:23 2013 +0200 @@ -92,6 +92,7 @@ extern int _PyLong_Init(void); extern void PyLong_Fini(void); extern int _PyFaulthandler_Init(void); extern void _PyFaulthandler_Fini(void); +extern int _PyTraceMalloc_Init(void); #ifdef WITH_THREAD extern void _PyGILState_Init(PyInterpreterState *, PyThreadState *); @@ -342,6 +343,9 @@ void if (_PyStructSequence_Init() < 0) Py_FatalError("Py_Initialize: can't initialize structseq"); + if (_PyTraceMalloc_Init() < 0) + Py_FatalError("Py_Initialize: can't initialize tracemalloc"); + bimod = _PyBuiltin_Init(); if (bimod == NULL) Py_FatalError("Py_Initialize: can't initialize builtins modules");