diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst index dacf8aa..3fb72c7 100644 --- a/Doc/library/linecache.rst +++ b/Doc/library/linecache.rst @@ -43,6 +43,14 @@ The :mod:`linecache` module defines the following functions: changed on disk, and you require the updated version. If *filename* is omitted, it will check all the entries in the cache. +.. function:: deferredcache(filename, module_globals) + + Capture enough detail about a non-file based module to permit getting its + lines later via :func:`getline` even if *module_globals* is None in the later + call. This permits avoiding doing I/O until a line is actually needed, + without having to carry the module globals around indefinitely. + + .. versionadded:: 3.5 Example:: diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst index 15fbedc..f0c590c 100644 --- a/Doc/library/traceback.rst +++ b/Doc/library/traceback.rst @@ -136,6 +136,59 @@ The module defines the following functions: .. versionadded:: 3.4 +.. function:: walk_stack(f) + + Walks a stack yielding the frame and line number for each frame. This + helper is used with *Stack.extract*. + + .. versionadded:: 3.5 + +.. function:: walk_tb(tb) + + Walks a traceback yielding the frame and line number for each frame. This + helper is used with *Stack.extract*. + + .. versionadded:: 3.5 + +The module also defines the following classes: + +:class:`Stack` Objects +---------------------- + +:class:`.Stack` objects represent a call stack ready for formatting. + +.. classmethod:: Stack.extract(frame_gen, limit=None, lookup_lines=True) + + Constructs a Stack object from a frame generator (such as is returned by + `walk_stack` or `walk_tb`. + + If limit is supplied, only this many frames are taken from frame_gen. + If lookup_lines is False, the returned Frame objects will not have read + their lines in yet, making the cost of creating the Stack cheaper (which + may be valuable if it may not actually get formatted). + + .. versionadded:: 3.5 + +.. classmethod:: Stack.from_list(a_list) + + Constructs a Stack object from a supplied old-style list of tuples. + + .. versionadded:: 3.5 + +:class:`Frame` Objects +---------------------- + +Frame objects represent a single frame in a traceback. + +.. class:: Frame(filename, lineno, name, lookup_line=True, locals=None, line=None) + :noindex: + + Represents a single frame in the traceback or stack that is being formatted + or printed. It may optionally have a stringified version of the frames + locals included in it. If *lookup_line* is False, the source code is not + looked up until the Frame has the :attr:`line` attribute accessed (which + also happens when casting it to a tuple). Line may be directly provided, and + will prevent line lookups happening at all. .. _traceback-example: diff --git a/Lib/linecache.py b/Lib/linecache.py index 02a9eb5..421425a 100644 --- a/Lib/linecache.py +++ b/Lib/linecache.py @@ -5,6 +5,7 @@ is not found, it will look down the module search path for a file by that name. """ +import functools import sys import os import tokenize @@ -36,6 +37,9 @@ def getlines(filename, module_globals=None): Update the cache if it doesn't contain an entry for this file already.""" if filename in cache: + entry = cache[filename] + if len(entry) == 1: + return updatecache(filename, module_globals) return cache[filename][2] else: return updatecache(filename, module_globals) @@ -54,7 +58,11 @@ def checkcache(filename=None): return for filename in filenames: - size, mtime, lines, fullname = cache[filename] + entry = cache[filename] + if len(entry) == 1: + # deferred cache entry, leave it deferred. + continue + size, mtime, lines, fullname = entry if mtime is None: continue # no-op for files loaded via a __loader__ try: @@ -72,7 +80,8 @@ def updatecache(filename, module_globals=None): and return an empty list.""" if filename in cache: - del cache[filename] + if len(cache[filename]) != 1: + del cache[filename] if not filename or (filename.startswith('<') and filename.endswith('>')): return [] @@ -82,27 +91,24 @@ def updatecache(filename, module_globals=None): except OSError: basename = filename - # Try for a __loader__, if available - if module_globals and '__loader__' in module_globals: - name = module_globals.get('__name__') - loader = module_globals['__loader__'] - get_source = getattr(loader, 'get_source', None) - - if name and get_source: - try: - data = get_source(name) - except (ImportError, OSError): - pass - else: - if data is None: - # No luck, the PEP302 loader cannot find the source - # for this module. - return [] - cache[filename] = ( - len(data), None, - [line+'\n' for line in data.splitlines()], fullname - ) - return cache[filename][2] + # Realise a deferred loader based lookup if there is one + # otherwise try to lookup right now. + if ((filename in cache and len(cache[filename]) == 1) or + deferredcache(filename, module_globals)): + try: + data = cache[filename][0]() + except (ImportError, OSError): + pass + else: + if data is None: + # No luck, the PEP302 loader cannot find the source + # for this module. + return [] + cache[filename] = ( + len(data), None, + [line+'\n' for line in data.splitlines()], fullname + ) + return cache[filename][2] # Try looking through the module search path, which is only useful # when handling a relative filename. @@ -132,3 +138,32 @@ def updatecache(filename, module_globals=None): size, mtime = stat.st_size, stat.st_mtime cache[filename] = size, mtime, lines, fullname return lines + + +def deferredcache(filename, module_globals): + """Seed the cache for filename with module_globals. + + The module loader will be asked for the source only when getlines is + called, not immediately. + + :return: True if a a deferred load has been registered in the cache, + otherwise False. To register such a load a module loader with a + get_source method must be found, the filename must be a cachable + filename, and the filename must not be already cached. + """ + if not filename or (filename.startswith('<') and filename.endswith('>')): + return False + # If the file is already cached: + if filename in cache: + return False + # Try for a __loader__, if available + if module_globals and '__loader__' in module_globals: + name = module_globals.get('__name__') + loader = module_globals['__loader__'] + get_source = getattr(loader, 'get_source', None) + + if name and get_source: + get_lines = functools.partial(get_source, name) + cache[filename] = (get_lines,) + return True + return False diff --git a/Lib/test/test_linecache.py b/Lib/test/test_linecache.py index 5fe0554..1d0e2cf 100644 --- a/Lib/test/test_linecache.py +++ b/Lib/test/test_linecache.py @@ -7,6 +7,7 @@ from test import support FILENAME = linecache.__file__ +NONEXISTANT_FILENAME = FILENAME + '.missing' INVALID_NAME = '!@$)(!@#_1' EMPTY = '' TESTS = 'inspect_fodder inspect_fodder2 mapping_tests' @@ -126,6 +127,49 @@ class LineCacheTests(unittest.TestCase): self.assertEqual(line, getline(source_name, index + 1)) source_list.append(line) + def test_deferredcache_no_globals(self): + lines = linecache.getlines(FILENAME) + linecache.clearcache() + self.assertEqual(False, linecache.deferredcache(FILENAME, None)) + self.assertEqual(lines, linecache.getlines(FILENAME)) + + def test_deferredcache_smoke(self): + lines = linecache.getlines(NONEXISTANT_FILENAME, globals()) + linecache.clearcache() + self.assertEqual( + True, linecache.deferredcache(NONEXISTANT_FILENAME, globals())) + self.assertEqual(1, len(linecache.cache[NONEXISTANT_FILENAME])) + # Note here that we're looking up a non existant filename with no + # globals: this would error if the deferred value wasn't resolved. + self.assertEqual(lines, linecache.getlines(NONEXISTANT_FILENAME)) + + def test_deferredcache_provide_after_failed_lookup(self): + linecache.clearcache() + lines = linecache.getlines(NONEXISTANT_FILENAME, globals()) + linecache.clearcache() + linecache.getlines(NONEXISTANT_FILENAME) + linecache.deferredcache(NONEXISTANT_FILENAME, globals()) + self.assertEqual(lines, linecache.updatecache(NONEXISTANT_FILENAME)) + + def test_deferredcache_check(self): + linecache.clearcache() + linecache.deferredcache(NONEXISTANT_FILENAME, globals()) + linecache.checkcache() + + def test_deferredcache_bad_filename(self): + linecache.clearcache() + self.assertEqual(False, linecache.deferredcache('', globals())) + self.assertEqual(False, linecache.deferredcache('', globals())) + + def test_deferredcache_already_cached(self): + linecache.clearcache() + lines = linecache.getlines(NONEXISTANT_FILENAME, globals()) + self.assertEqual( + False, + linecache.deferredcache(NONEXISTANT_FILENAME, globals())) + self.assertEqual(4, len(linecache.cache[NONEXISTANT_FILENAME])) + + def test_main(): support.run_unittest(LineCacheTests) diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 6bd6fa6..708358d 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -1,6 +1,8 @@ """Test cases for traceback module""" +from collections import namedtuple from io import StringIO +import linecache import sys import unittest import re @@ -12,6 +14,10 @@ import textwrap import traceback +test_code = namedtuple('code', ['co_filename', 'co_name']) +test_frame = namedtuple('frame', ['f_code', 'f_globals']) + + class SyntaxTracebackCases(unittest.TestCase): # For now, a very minimal set of tests. I want to be sure that # formatting of SyntaxErrors works based on changes for 2.1. @@ -477,6 +483,87 @@ class MiscTracebackCases(unittest.TestCase): self.assertEqual(len(inner_frame.f_locals), 0) +class TestFrame(unittest.TestCase): + + def test_basics(self): + linecache.clearcache() + linecache.deferredcache("f", globals()) + f = traceback.Frame("f", 1, "dummy") + self.assertEqual( + ("f", 1, "dummy", '"""Test cases for traceback module"""'), + tuple(f)) + self.assertEqual(None, f.locals) + + def test_lazy_lines(self): + linecache.clearcache() + f = traceback.Frame("f", 1, "dummy", lookup_line=False) + self.assertEqual(None, f._line) + linecache.deferredcache("f", globals()) + self.assertEqual( + '"""Test cases for traceback module"""', + f.line) + + def test_explicit_line(self): + f = traceback.Frame("f", 1, "dummy", line="line") + self.assertEqual("line", f.line) + + +class TestStack(unittest.TestCase): + + def test_walk_stack(self): + s = list(traceback.walk_stack(None)) + self.assertGreater(len(s), 10) + + def test_walk_tb(self): + try: + 1/0 + except Exception: + _, _, tb = sys.exc_info() + s = list(traceback.walk_tb(tb)) + self.assertEqual(len(s), 1) + + def test_extract_stack(self): + s = traceback.Stack.extract(traceback.walk_stack(None)) + self.assertIsInstance(s, traceback.Stack) + + def test_extract_stack_limit(self): + s = traceback.Stack.extract(traceback.walk_stack(None), limit=5) + self.assertEqual(len(s), 5) + + def test_extract_stack_lookup_lines(self): + linecache.clearcache() + linecache.updatecache('/foo.py', globals()) + c = test_code('/foo.py', 'method') + f = test_frame(c, None) + s = traceback.Stack.extract(iter([(f, 6)]), lookup_lines=True) + linecache.clearcache() + self.assertEqual(s[0].line, "import sys") + + def test_extract_stackup_deferred_lookup_lines(self): + linecache.clearcache() + c = test_code('/foo.py', 'method') + f = test_frame(c, None) + s = traceback.Stack.extract(iter([(f, 6)]), lookup_lines=False) + self.assertEqual({}, linecache.cache) + linecache.updatecache('/foo.py', globals()) + self.assertEqual(s[0].line, "import sys") + + def test_from_list(self): + s = traceback.Stack([('foo.py', 1, 'fred', 'line')]) + self.assertEqual( + [' File "foo.py", line 1, in fred\n line\n'], + s.format()) + + def test_format_smoke(self): + # For detailed tests see the format_list tests, which consume the same + # code. + s = traceback.Stack([('foo.py', 1, 'fred', 'line')]) + self.assertEqual( + [' File "foo.py", line 1, in fred\n line\n'], + s.format()) + + + def test_main(): run_unittest(__name__) diff --git a/Lib/traceback.py b/Lib/traceback.py index c1ab36e..0f8d75d 100644 --- a/Lib/traceback.py +++ b/Lib/traceback.py @@ -14,19 +14,12 @@ __all__ = ['extract_stack', 'extract_tb', 'format_exception', # Formatting and printing lists of traceback lines. # -def _format_list_iter(extracted_list): - for filename, lineno, name, line in extracted_list: - item = ' File "{}", line {}, in {}\n'.format(filename, lineno, name) - if line: - item = item + ' {}\n'.format(line.strip()) - yield item - def print_list(extracted_list, file=None): """Print the list of tuples as returned by extract_tb() or extract_stack() as a formatted stack trace to the given file.""" if file is None: file = sys.stderr - for item in _format_list_iter(extracted_list): + for item in Stack.from_list(extracted_list).format(): print(item, file=file, end="") def format_list(extracted_list): @@ -39,45 +32,12 @@ def format_list(extracted_list): the strings may contain internal newlines as well, for those items whose source text line is not None. """ - return list(_format_list_iter(extracted_list)) + return Stack.from_list(extracted_list).format() # # Printing and Extracting Tracebacks. # -# extractor takes curr and needs to return a tuple of: -# - Frame object -# - Line number -# - Next item (same type as curr) -# In practice, curr is either a traceback or a frame. -def _extract_tb_or_stack_iter(curr, limit, extractor): - if limit is None: - limit = getattr(sys, 'tracebacklimit', None) - - n = 0 - while curr is not None and (limit is None or n < limit): - f, lineno, next_item = extractor(curr) - co = f.f_code - filename = co.co_filename - name = co.co_name - - linecache.checkcache(filename) - line = linecache.getline(filename, lineno, f.f_globals) - - if line: - line = line.strip() - else: - line = None - - yield (filename, lineno, name, line) - curr = next_item - n += 1 - -def _extract_tb_iter(tb, limit): - return _extract_tb_or_stack_iter( - tb, limit, - operator.attrgetter("tb_frame", "tb_lineno", "tb_next")) - def print_tb(tb, limit=None, file=None): """Print up to 'limit' stack trace entries from the traceback 'tb'. @@ -90,7 +50,7 @@ def print_tb(tb, limit=None, file=None): def format_tb(tb, limit=None): """A shorthand for 'format_list(extract_tb(tb, limit))'.""" - return format_list(extract_tb(tb, limit=limit)) + return extract_tb(tb, limit=limit).format() def extract_tb(tb, limit=None): """Return list of up to limit pre-processed entries from traceback. @@ -103,7 +63,7 @@ def extract_tb(tb, limit=None): leading and trailing whitespace stripped; if the source is not available it is None. """ - return list(_extract_tb_iter(tb, limit=limit)) + return Stack.extract(walk_tb(tb), limit=limit) # # Exception formatting and output. @@ -150,7 +110,7 @@ def _format_exception_iter(etype, value, tb, limit, chain): continue if tb: yield 'Traceback (most recent call last):\n' - yield from _format_list_iter(_extract_tb_iter(tb, limit=limit)) + yield from Stack.extract(walk_tb(tb), limit=limit).format() yield from _format_exception_only_iter(type(value), value) def print_exception(etype, value, tb, limit=None, file=None, chain=True): @@ -267,15 +227,6 @@ def print_last(limit=None, file=None, chain=True): # Printing and Extracting Stacks. # -def _extract_stack_iter(f, limit=None): - return _extract_tb_or_stack_iter( - f, limit, lambda f: (f, f.f_lineno, f.f_back)) - -def _get_stack(f): - if f is None: - f = sys._getframe().f_back.f_back - return f - def print_stack(f=None, limit=None, file=None): """Print a stack trace from its invocation point. @@ -283,11 +234,11 @@ def print_stack(f=None, limit=None, file=None): stack frame at which to start. The optional 'limit' and 'file' arguments have the same meaning as for print_exception(). """ - print_list(extract_stack(_get_stack(f), limit=limit), file=file) + print_list(extract_stack(f, limit=limit), file=file) def format_stack(f=None, limit=None): """Shorthand for 'format_list(extract_stack(f, limit))'.""" - return format_list(extract_stack(_get_stack(f), limit=limit)) + return format_list(extract_stack(f, limit=limit)) def extract_stack(f=None, limit=None): """Extract the raw traceback from the current stack frame. @@ -298,7 +249,7 @@ def extract_stack(f=None, limit=None): line number, function name, text), and the entries are in order from oldest to newest stack frame. """ - stack = list(_extract_stack_iter(_get_stack(f), limit=limit)) + stack = Stack.extract(walk_stack(f), limit=limit) stack.reverse() return stack @@ -311,3 +262,136 @@ def clear_frames(tb): # Ignore the exception raised if the frame is still executing. pass tb = tb.tb_next + + +class Frame: + """A single frame from a traceback. + + - :attr:`filename` The filename for the frame. + - :attr:`lineno` The line within filename for the frame that was + active when the frame was captured. + - :attr:`name` The name of the function or method that was executing + when the frame was captured. + - :attr:`line` The text from the linecache module for the + of code that was running when the frame was captured. + - :attr:`locals` Either None if locals were not supplied, or a dict + mapping the name to the str() of the variable. + """ + + __slots__ = ('filename', 'lineno', 'name', '_line', 'locals') + + def __init__(self, filename, lineno, name, lookup_line=True, locals=None, + line=None): + """Construct a Frame. + + :param lookup_line: If True, `linecache` is consulted for the source + code line. Otherwise, the line will be looked up when first needed. + :param locals: If supplied the frame locals, which will be captured as + strings. + :param line: If provided, use this instead of looking up the line in + the linecache. + """ + self.filename = filename + self.lineno = lineno + self.name = name + self._line = line + if lookup_line: + self.line + self.locals = \ + dict((k, str(v)) for k, v in locals.items()) if locals else None + + def __iter__(self): + return iter([self.filename, self.lineno, self.name, self.line]) + + @property + def line(self): + if self._line is None: + self._line = linecache.getline(self.filename, self.lineno).strip() + return self._line + + +def walk_stack(f): + """Walk a stack yielding the frame and line number for each frame. + + Usually used with Stack.extract. + """ + if f is None: + f = sys._getframe().f_back.f_back + while f is not None: + yield f, f.f_lineno + f = f.f_back + + +def walk_tb(tb): + """Walk a traceback yielding the frame and line number for each frame. + + Usually used with Stack.extract. + """ + while tb is not None: + yield tb.tb_frame, tb.tb_lineno + tb = tb.tb_next + + +class Stack(list): + """A stack of frames.""" + + @classmethod + def extract(klass, frame_gen, limit=None, lookup_lines=True): + """Create a Stack from a traceback or stack object. + + :param frame_gen: A generator that yields (frame, lineno) tuples to + include in the stack. + :param limit: None to include all frames or the number of frames to + include. + :param lookup_lines: If True, lookup lines for each frame immediately, + otherwise lookup is deferred until the frame is rendered. + """ + if limit is None: + limit = getattr(sys, 'tracebacklimit', None) + + result = klass() + fnames = set() + for pos, (f, lineno) in enumerate(frame_gen): + if limit is not None and pos >= limit: + break + co = f.f_code + filename = co.co_filename + name = co.co_name + + fnames.add(filename) + linecache.deferredcache(filename, f.f_globals) + # Must defer line lookups until we have called checkcache. + result.append(Frame(filename, lineno, name, lookup_line=False)) + for filename in fnames: + linecache.checkcache(filename) + # If immediate lookup was desired, trigger lookups now. + if lookup_lines: + for f in result: + f.line + return result + + @classmethod + def from_list(klass, a_list): + """Create a Stack from a simple list of Tuples as older Python used.""" + if isinstance(a_list, Stack): + return Stack(a_list) + result = Stack() + for filename, lineno, name, line in a_list: + result.append(Frame(filename, lineno, name, line=line)) + return result + + def format(self): + """Format the stack ready for printing. + + Returns a list of strings ready for printing. Each string in the + resulting list corresponds to a single frame from the stack. + Each string ends in a newline; the strings may contain internal + newlines as well, for those items with source text lines. + """ + result = [] + for filename, lineno, name, line in self: + item = ' File "{}", line {}, in {}\n'.format(filename, lineno, name) + if line: + item = item + ' {}\n'.format(line.strip()) + result.append(item) + return result diff --git a/Misc/NEWS b/Misc/NEWS index 5373f5b..0f84f8a 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -231,6 +231,9 @@ Library now clears its internal reference to the selector mapping to break a reference cycle. Initial patch written by Martin Richard. +- Issue #17911: Provide a way to seed the linecache for a PEP-302 module + without actually loading the code. + - Issue #19777: Provide a home() classmethod on Path objects. Contributed by Victor Salgado and Mayank Tripathi.