# HG changeset patch # Parent b730baee0877bf71ca934bacf157ef69280292ba Issue #25419: Support Readline completion of import statements Supports completing the names in “import . . .”, “from . . .”, and “from import . . .”. Also remove a redundant test, with comment about apparently unrelated Issue #5256 (builtins vs custom namespace). Change the demo in the documentation, because attributes like __doc__ are no longer listed by default (Issue #25011). diff -r b730baee0877 Doc/library/rlcompleter.rst --- a/Doc/library/rlcompleter.rst Tue May 24 16:39:23 2016 -0700 +++ b/Doc/library/rlcompleter.rst Wed May 25 06:43:03 2016 +0000 @@ -13,48 +13,55 @@ :mod:`readline` module by completing valid Python identifiers and keywords. When this module is imported on a Unix platform with the :mod:`readline` module -available, an instance of the :class:`Completer` class is automatically created -and its :meth:`complete` method is set as the :mod:`readline` completer. +available, an instance of the :class:`ReadlineCompleter` class is created +and its :meth:`~ReadlineCompleter.complete` method is set as the +:mod:`readline` completer. Example:: >>> import rlcompleter >>> import readline >>> readline.parse_and_bind("tab: complete") - >>> readline. - readline.__doc__ readline.get_line_buffer( readline.read_init_file( - readline.__file__ readline.insert_text( readline.set_completer( - readline.__name__ readline.parse_and_bind( - >>> readline. + >>> rlcompleter. + rlcompleter.Completer( rlcompleter.get_class_members( + rlcompleter.ReadlineCompleter( rlcompleter.readline + rlcompleter.atexit rlcompleter.sys + rlcompleter.builtins rlcompleter.tokenize + >>> rlcompleter. The :mod:`rlcompleter` module is designed for use with Python's :ref:`interactive mode `. Unless Python is run with the :option:`-S` option, the module is automatically imported and configured (see :ref:`rlcompleter-config`). -On platforms without :mod:`readline`, the :class:`Completer` class defined by -this module can still be used for custom purposes. +.. class:: Completer -.. _completer-objects: + Provides basic completion of words. On platforms without :mod:`readline`, + this class can still be used for custom purposes. -Completer Objects ------------------ -Completer objects have the following method: + .. method:: Completer.complete(text, state) + Return the *state*\ th completion for *text*. -.. method:: Completer.complete(text, state) + If called for *text* that doesn't include a period character (``'.'``), it will + complete from names currently defined in :mod:`__main__`, :mod:`builtins` and + keywords (as defined by the :mod:`keyword` module). - Return the *state*\ th completion for *text*. + If called for a dotted name, it will try to evaluate anything without obvious + side-effects (functions will not be evaluated, but it can generate calls to + :meth:`__getattr__`) up to the last part, and find matches for the rest via the + :func:`dir` function. Any exception raised during the evaluation of the + expression is caught, silenced and :const:`None` is returned. - If called for *text* that doesn't include a period character (``'.'``), it will - complete from names currently defined in :mod:`__main__`, :mod:`builtins` and - keywords (as defined by the :mod:`keyword` module). - If called for a dotted name, it will try to evaluate anything without obvious - side-effects (functions will not be evaluated, but it can generate calls to - :meth:`__getattr__`) up to the last part, and find matches for the rest via the - :func:`dir` function. Any exception raised during the evaluation of the - expression is caught, silenced and :const:`None` is returned. +.. class:: ReadlineCompleter + A more functional :class:`Completer` subclass for use with + :mod:`readline`. It suggests completions based on all of the + user's input up to the cursor, scanning back further than a + single "word". For instance, it can identify and complete + :keyword:`import` statements. It should only be used when + :mod:`readline` is available. + diff -r b730baee0877 Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst Tue May 24 16:39:23 2016 -0700 +++ b/Doc/whatsnew/3.6.rst Wed May 25 06:43:03 2016 +0000 @@ -320,6 +320,11 @@ Previously, names of properties and slots which were not yet created on an instance were excluded. (Contributed by Martin Panter in :issue:`25590`.) +Added the :class:`~rlcompleter.ReadlineCompleter` subclass which is +specifically for use with :mod:`readline`. Module and attribute names are +now completed in :keyword:`import` statements. (Contributed by Martin Panter +in :issue:`25419`.) + site ---- diff -r b730baee0877 Lib/rlcompleter.py --- a/Lib/rlcompleter.py Tue May 24 16:39:23 2016 -0700 +++ b/Lib/rlcompleter.py Wed May 25 06:43:03 2016 +0000 @@ -3,6 +3,7 @@ The completer completes keywords, built-ins and globals in a selectable namespace (which defaults to __main__); when completing NAME.NAME..., it evaluates (!) the expression up to the last dot and completes its attributes. +Module and attribute names in 'import' statements are also completed. It's very cool to do "import sys" type "sys.", hit the completion key (twice), and see the list of names defined by the sys module! @@ -32,8 +33,10 @@ import atexit import builtins import __main__ +import sys +import tokenize -__all__ = ["Completer"] +__all__ = ["Completer", "ReadlineCompleter"] class Completer: def __init__(self, namespace = None): @@ -45,7 +48,7 @@ is __main__ (technically, __main__.__dict__). Namespaces should be given as dictionaries. - Completer instances should be used as the completion mechanism of + Completer instances may be used as the completion mechanism of readline via the set_completer() call: readline.set_completer(Completer(my_namespace).complete) @@ -75,25 +78,22 @@ if not text.strip(): if state == 0: - if _readline_available: - readline.insert_text('\t') - readline.redisplay() - return '' - else: - return '\t' + return '\t' else: return None - if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) + self.matches = self._get_matches(text) try: return self.matches[state] except IndexError: return None + def _get_matches(self, text): + if "." in text: + return self.attr_matches(text) + else: + return self.global_matches(text) + def _callable_postfix(self, val, word): if callable(val): word = word + "(" @@ -135,7 +135,7 @@ (as revealed by dir()) are used as possible completions. (For class instances, class members are also considered.) - WARNING: this can still invoke arbitrary C code, if an object + WARNING: this can still invoke arbitrary code, if an object with a __getattr__ hook is evaluated. """ @@ -157,33 +157,237 @@ words.add('__class__') words.update(get_class_members(thisobject.__class__)) matches = [] - n = len(attr) - if attr == '': - noprefix = '_' - elif attr == '_': - noprefix = '__' + words = self._filter_identifiers(words, attr) + for word in words: + match = "%s.%s" % (expr, word) + try: + val = getattr(thisobject, word) + except Exception: + pass # Include even if attribute not set + else: + match = self._callable_postfix(val, match) + matches.append(match) + matches.sort() + return matches + + def _code_matches(self, code, *, mode='\t'): + """Return a tuple (matches, prefix) for Python source code. + + Each item in 'matches' normally begins with 'prefix', which is the + unfinished identifier at the end of 'code'. + """ + # Find the start of any unfinished identifier at the end + i = len(code) + while i > 0: + c = code[i - 1] + # Assuming Python tokens have to be punctuated by ASCII + if not c.isalnum() and c != '_' and ord(c) < 128: + break + i -= 1 + prefix = code[i:] + code = code[:i] + matches = self._next_identifiers(code, mode=mode) + if matches is not None: + matches = self._filter_identifiers(tuple(matches), prefix) + return (matches, prefix) + + def _filter_identifiers(self, words, prefix): + matched = False + if prefix == '': + exclude = '_' + elif prefix == '_': + exclude = '__' else: - noprefix = None + exclude = () # startswith() matches nothing while True: for word in words: - if (word[:n] == attr and - not (noprefix and word[:n+1] == noprefix)): - match = "%s.%s" % (expr, word) + if (word.startswith(prefix) and + not word.startswith(exclude)): + matched = True + yield word + if matched or not exclude: + break + if exclude == '_': + exclude = '__' + else: + exclude = () + + def _next_identifiers(self, code, *, mode='\t'): + """Return possibilities for the next identifier in 'code'.""" + tokens = self._tokenize(code) + try: # Trap unhandled StopIteration from any parsing level + for token in tokens: + if token.string in {'import', 'from'}: + matches, token = self._parse_import(tokens, mode=mode) + if matches is not None: + return matches + + # Skip until the next statement. This is very dumb, but + # sufficient for the purpose. It would be tricked by + # dictionary displays and lambdas. + while (token.type != tokenize.NEWLINE and + token.string not in {':', ';'}): + token = next(tokens) + except StopIteration: + pass + return None + + def _tokenize(self, code): + from io import StringIO + tokens = tokenize.generate_tokens(StringIO(code).readline) + unimportant = { + tokenize.INDENT, tokenize.DEDENT, tokenize.COMMENT, tokenize.NL} + try: + for token in tokens: + if token.type in unimportant: + continue + if token.type == tokenize.ENDMARKER: + break + yield token + except tokenize.TokenError: + pass + + def _parse_import(self, tokens, *, mode): + while True: # For each module in "import a, b, c" + # Build the "package.subpackage.module" path + path = list() + relative = 0 # Count leading dots (.) + for token in tokens: + if token.string == '.': # Assume at the start of the path + relative += 1 + continue + if token.type != tokenize.NAME or token.string == 'import': + break + path.append(token.string) + token = next(tokens) + if token.string != '.': + break + else: # Module name belongs at cursor position + matches = self._import_matches(relative, path, mode=mode) + return (matches, None) + + if token.string == 'as': + next(tokens) # Consume alias + token = next(tokens) + if token.string != ',': + break + + if token.string == 'import': + # Handle "from module import . . ." + for token in tokens: + if token.type == tokenize.NAME: + token = next(tokens) + if token.string == 'as': + next(tokens) # Consume alias + token = next(tokens) + if token.string != ',': + break + elif token.string != '(': + break + else: # Module attribute name belongs at cursor position + matches = self._from_import_matches(relative, path) + return (matches, None) + + return (None, token) + + def _import_matches(self, relative, package_path, *, mode): + import pkgutil + import importlib + + if relative: + package = self.namespace.get('__package__') + if package is None: + return + # Work backwards to find the parent if relative > 1 + ancestry = package.rsplit('.', relative - 1) + if len(ancestry) < relative: + return # Beyond highest level package + name = ancestry[0] + search_paths = sys.modules[name].__path__ + name += '.' + else: + name = '' + search_paths = None + + # Confirm each "package_path" element is a package before importing + if package_path: + for package in package_path: + name += package + module = sys.modules.get(name) + if module is None: try: - val = getattr(thisobject, word) + loader = pkgutil.find_loader(name) except Exception: - pass # Include even if attribute not set - else: - match = self._callable_postfix(val, match) - matches.append(match) - if matches or not noprefix: - break - if noprefix == '_': - noprefix = '__' + return + is_package = getattr(loader, 'is_package', None) + if not is_package or not is_package(name): + return + elif not hasattr(module, '__path__'): + return # Not a package + name += '.' + if module is None: # Last package on path not loaded + try: + # Cannot use loader.load_module() because it does not set + # up package.submodule attribute + module = importlib.import_module(name.rstrip('.')) + except Exception: + return + search_paths = module.__path__ + + if relative and not package_path: + yield 'import ' # from . [import] + for _, name, ispkg in pkgutil.iter_modules(search_paths): + # Add dot (.) to package names if completion is only for display + if mode == '?' and ispkg: + name += '.' + yield name + + def _from_import_matches(self, relative, path): + import importlib + if relative: + package = self.namespace.get('__package__') + else: + package = None + path = '.' * relative + '.'.join(path) + try: + module = importlib.import_module(path, package) + except Exception: + return + yield from dir(module) + search_path = getattr(module, '__path__', None) + if search_path is not None: + import pkgutil + for _, name, _ in pkgutil.iter_modules(search_path): + yield name + +class ReadlineCompleter(Completer): + """Completer subclass for use with the 'readline' module.""" + def complete(self, text, state): + #~ # Exceptions are silently ignored by the 'readline' module. + #~ # Uncomment this to show exceptions: + #~ try: + if text: + return super().complete(text, state) + if state == 0: + readline.insert_text('\t') + readline.redisplay() + return '' else: - noprefix = None - matches.sort() - return matches + return None + #~ except Exception: + #~ sys.excepthook(*sys.exc_info()) + #~ raise + + def _get_matches(self, text): + code = readline.get_line_buffer()[:readline.get_endidx()] + mode = chr(readline.get_completion_type()) + matches, prefix = self._code_matches(code, mode=mode) + if matches is None: + return super()._get_matches(text) + if mode != '?': + prefix = text[:len(text) - len(prefix)] + matches = (prefix + m for m in matches) + return list(matches) def get_class_members(klass): ret = dir(klass) @@ -195,11 +399,10 @@ try: import readline except ImportError: - _readline_available = False + pass else: - readline.set_completer(Completer().complete) + readline.set_completer(ReadlineCompleter().complete) # Release references early at shutdown (the readline module's # contents are quasi-immortal, and the completer function holds a # reference to globals). atexit.register(lambda: readline.set_completer(None)) - _readline_available = True diff -r b730baee0877 Lib/test/test_rlcompleter.py --- a/Lib/test/test_rlcompleter.py Tue May 24 16:39:23 2016 -0700 +++ b/Lib/test/test_rlcompleter.py Wed May 25 06:43:03 2016 +0000 @@ -1,3 +1,4 @@ +from test import support import unittest from unittest.mock import patch import builtins @@ -43,9 +44,6 @@ ['CompleteMe(']) self.assertEqual(self.completer.global_matches('eg'), ['egg(']) - # XXX: see issue5256 - self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe(']) def test_attr_matches(self): # test with builtins namespace @@ -101,8 +99,9 @@ completer = rlcompleter.Completer(dict(f=Foo())) self.assertEqual(completer.complete('f.', 0), 'f.bar') - @unittest.mock.patch('rlcompleter._readline_available', False) def test_complete(self): + rlcompleter = support.import_fresh_module('rlcompleter', + blocked=('readline',)) completer = rlcompleter.Completer() self.assertEqual(completer.complete('', 0), '\t') self.assertEqual(completer.complete('a', 0), 'and ') @@ -137,5 +136,90 @@ self.assertEqual(completer.complete('Ellipsis', 0), 'Ellipsis(') self.assertIsNone(completer.complete('Ellipsis', 1)) +class TestReadlineCompleter(unittest.TestCase): + @classmethod + def setUpClass(cls): + support.import_module('readline') + + def test_import(self): + cases = ( + 'import ', + 'pass; import ', + 'try:\n' + ' import ', + 'import something.else, ', + 'import something as alias, ', + ) + for line in cases: + with self.subTest(line=line): + completed = self.complete_line(line, 'test.test_rl') + self.assertEqual(completed, 'test.test_rlcompleter') + completed = self.complete_line('import test . ', 'test_rl') + self.assertEqual(completed, 'test_rlcompleter') + + # Invalid imports should not cause exceptions + self.complete_line('import ', 'test.test_rlcompleter.rlcompleter') + self.complete_line( + 'import ', 'test.test_rlcompleter.rlcompleter.x') + self.complete_line('import ', 'test.nonexistant.x') + + def test_from_module(self): + # Test relative to "test.test_rlcompleter" + g = globals() + completed = self.complete_line('from ', 'test.test_rl', g) + self.assertEqual(completed, 'test.test_rlcompleter') + completed = self.complete_line('from ', '.test_rl', g) + self.assertEqual(completed, '.test_rlcompleter') + self.assertEqual(self.complete_line('from . ', 'im', g), 'import ') + # Only interpret "from" at the start of a statement + self.assertEqual(self.complete_line('yield from ', 'Fa', g), 'False') + + # Invalid imports should not cause exceptions + self.complete_line('from ', '..x', g) + self.complete_line('from ', '...x', g) + + def test_from_import(self): + # Test relative to "test.test_rlcompleter" + g = globals() + cases = ( + ('from rlcompleter import ', 'Comp', 'Completer'), + ('from test import ', 'test_rl', 'test_rlcompleter'), + ('from . import ', 'test_rl', 'test_rlcompleter'), + ('from .test_rlcompleter import ', 'TestRead', + 'TestReadlineCompleter'), + ( + 'from rlcompleter import ( # Comment\n' + ' other as alias,\n' + ' ', 'Comp', 'Completer', + ), + ) + for line, word, expected in cases: + with self.subTest(line=line, word=word): + self.assertEqual(self.complete_line(line, word, g), expected) + + # Invalid imports should not cause exceptions + self.complete_line( + 'from test.test_rlcompleter.rlcompleter import ', 'Completer', g) + self.complete_line('from test.nonexistant import ', 'x', g) + self.complete_line('from test.nonexistant.x import ', 'x', g) + self.complete_line('from .. import ', 'rlcompleter', g) + self.complete_line('from ...beyond import ', 'x', g) + + def test_tokenize_errors(self): + # These cause "tokenize" to raise exceptions, rather than yielding + # error tokens + # EOF in multi-line statement: + self.assertEqual(self.complete_line('(', 'Fa'), 'False') + # EOF in multi-line string (use Ctrl+V, Ctrl+J in Gnu Readline): + self.complete_line('"""abc\n', 'blaua') + + def complete_line(self, previous, word, namespace=None): + completer = rlcompleter.ReadlineCompleter(namespace) + line = previous + word + with patch('readline.get_line_buffer', lambda: line), \ + patch('readline.get_begidx', lambda: len(previous)), \ + patch('readline.get_endidx', lambda: len(line)): + return completer.complete(word, 0) + if __name__ == '__main__': unittest.main() diff -r b730baee0877 Misc/NEWS --- a/Misc/NEWS Tue May 24 16:39:23 2016 -0700 +++ b/Misc/NEWS Wed May 25 06:43:03 2016 +0000 @@ -19,6 +19,8 @@ Library ------- +- Issue #25419: Support for Readline completion of "import" statements. + - Issue #23026: winreg.QueryValueEx() now return an integer for REG_QWORD type. - Issue #26741: subprocess.Popen destructor now emits a ResourceWarning warning