# HG changeset patch # Parent 0aaadc1c60fd0d88a860b383ae1bfb836c198802 Support Readline completion of import statements Supports completing the names in “import . . .”, “from . . .”, and “from import . . .”. The completions may not work well for non-ASCII inputs due to Issue #16182, and do not yet work for built-in module names. diff -r 0aaadc1c60fd Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst Fri Oct 16 11:23:31 2015 +0300 +++ b/Doc/whatsnew/3.6.rst Fri Oct 16 11:10:47 2015 +0000 @@ -128,6 +128,8 @@ with underscores. A space or a colon can be added after completed keyword. (Contributed by Serhiy Storchaka in :issue:`25011` and :issue:`25209`.) +Module and attribute names are now completed in :keyword:`import` statements. + urllib.robotparser ------------------ diff -r 0aaadc1c60fd Lib/rlcompleter.py --- a/Lib/rlcompleter.py Fri Oct 16 11:23:31 2015 +0300 +++ b/Lib/rlcompleter.py Fri Oct 16 11:10:47 2015 +0000 @@ -32,6 +32,8 @@ import atexit import builtins import __main__ +import sys +import tokenize __all__ = ["Completer"] @@ -80,15 +82,48 @@ return None if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) + # Exceptions are silently ignored by the "readline" module. + # Uncomment this to show exceptions: + #~ try: + self.matches = self._get_matches(text) + #~ except Exception: + #~ sys.excepthook(*sys.exc_info()) + #~ raise try: return self.matches[state] except IndexError: return None + def _get_matches(self, text): + if readline: + line, prefix = self._get_input() + matches = self._context_matches(line, text, prefix) + if matches is not None: + return matches + if "." in text: + return self.attr_matches(text) + else: + return self.global_matches(text) + + def _get_input(self): + line = readline.get_line_buffer() + + # Work around Issue 16182. Readline indexes are actually byte + # indexes, but this module assumes they correspond to code points. + # This should be fine for ASCII line buffers. For non-ASCII, avoid + # indexing past the end of the string. + i = min(readline.get_endidx(), len(line)) + + # Find the start of any unfinished token at the cursor + while i > 0: + c = line[i - 1] + # Assuming Python tokens have to be punctuated by ASCII + if not c.isalnum() and c != '_' and ord(c) < 128: + break + i -= 1 + line = line[:i] + return (line, line[readline.get_begidx():]) + def _callable_postfix(self, val, word): if callable(val): word = word + "(" @@ -173,6 +208,165 @@ matches.sort() return matches + def _context_matches(self, line, text, prefix): + """Compute matches from the parsing context of the input. + + "Text" ends with the current token to be completed, if any. "Line" + includes all input up to this token. "Text" may also hold extra + tokens preceding the current token (if they are separated by dots, + "."), and "prefix" is a string of these extra tokens. Each match + returned must include this prefix. + + """ + tokens = self._tokenize(line) + try: # Trap unhandled StopIteration from any parsing level + for token in tokens: + if token.string in {'import', 'from'}: + matches, token = self._parse_import(tokens, prefix) + if matches is not None: + filtered = list() + for match in matches: + if match.startswith(text): + filtered.append(match) + return filtered + + # Skip until the next statement. This is very dumb, but + # sufficient for the purpose. It would be tricked by + # dictionary displays and lambdas. + while (token.type != tokenize.NEWLINE and + token.string not in set(':;')): + token = next(tokens) + except StopIteration: + pass + return None + + def _tokenize(self, line): + from io import StringIO + tokens = tokenize.generate_tokens(StringIO(line).readline) + unimportant = { + tokenize.INDENT, tokenize.DEDENT, tokenize.COMMENT, tokenize.NL} + try: + for token in tokens: + if token.type in unimportant: + continue + if token.type == tokenize.ENDMARKER: + break + yield token + except tokenize.TokenError: + pass + + def _parse_import(self, tokens, prefix): + while True: # For each module in "import a, b, c" + # Build the "package.subpackage.module" path + path = list() + relative = 0 # Count leading dots (.) + for token in tokens: + if token.string == '.': # Assume at the start of the path + relative += 1 + continue + if token.type != tokenize.NAME or token.string == 'import': + break + path.append(token.string) + token = next(tokens) + if token.string != '.': + break + else: # Module name belongs at cursor position + return (self._import_matches(relative, path, prefix), None) + + if token.string == 'as': + next(tokens) # Consume alias + token = next(tokens) + if token.string != ',': + break + + if token.string == 'import': + # Handle "from module import . . ." + for token in tokens: + if token.type == tokenize.NAME: + token = next(tokens) + if token.string == 'as': + next(tokens) # Consume alias + token = next(tokens) + if token.string != ',': + break + elif token.string != '(': + break + else: # Module attribute name belongs at cursor position + return (self._from_import_matches(relative, path), None) + + return (None, token) + + def _import_matches(self, relative, package_path, prefix): + import pkgutil + import importlib + + if relative: + package = self.namespace.get('__package__') + if package is None: + return + # Work backwards to find the parent if relative > 1 + ancestry = package.rsplit('.', relative - 1) + if len(ancestry) < relative: + return # Beyond highest level package + name = ancestry[0] + search_paths = sys.modules[name].__path__ + name += '.' + else: + name = '' + search_paths = None + + # Confirm each "package_path" element is a package before importing + if package_path: + for package in package_path: + name += package + module = sys.modules.get(name) + if module is None: + try: + loader = pkgutil.find_loader(name) + except Exception: + return + is_package = getattr(loader, 'is_package', None) + if not is_package or not is_package(name): + return + elif not hasattr(module, '__path__'): + return # Not a package + name += '.' + if module is None: # Last package on path not loaded + try: + # Cannot use loader.load_module() because it does not set + # up package.submodule attribute + module = importlib.import_module(name.rstrip('.')) + except Exception: + return + search_paths = module.__path__ + + if relative and not package_path: + yield 'import ' # from . [import] + # Add dot (.) to package names if completion is only being displayed + indicator = readline.get_completion_type() == ord('?') + for _, name, ispkg in pkgutil.iter_modules(search_paths, prefix): + if indicator and ispkg: + name += '.' + yield name + + def _from_import_matches(self, relative, path): + import importlib + if relative: + package = self.namespace.get('__package__') + else: + package = None + path = '.' * relative + '.'.join(path) + try: + module = importlib.import_module(path, package) + except Exception: + return + yield from dir(module) + search_path = getattr(module, '__path__', None) + if search_path is not None: + import pkgutil + for _, name, _ in pkgutil.iter_modules(search_path): + yield name + def get_class_members(klass): ret = dir(klass) if hasattr(klass,'__bases__'): @@ -183,7 +377,7 @@ try: import readline except ImportError: - pass + readline = None else: readline.set_completer(Completer().complete) # Release references early at shutdown (the readline module's diff -r 0aaadc1c60fd Lib/test/test_rlcompleter.py --- a/Lib/test/test_rlcompleter.py Fri Oct 16 11:23:31 2015 +0300 +++ b/Lib/test/test_rlcompleter.py Fri Oct 16 11:10:47 2015 +0000 @@ -1,6 +1,7 @@ import unittest import builtins import rlcompleter +from unittest.mock import patch class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ @@ -42,9 +43,6 @@ ['CompleteMe(']) self.assertEqual(self.completer.global_matches('eg'), ['egg(']) - # XXX: see issue5256 - self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe(']) def test_attr_matches(self): # test with builtins namespace @@ -92,5 +90,83 @@ self.assertEqual(completer.complete('el', 1), 'else') self.assertEqual(completer.complete('tr', 0), 'try:') + def test_import(self): + cases = ( + 'import ', + 'pass; import ', + 'try:\n' + ' import ', + 'import something.else, ', + 'import something as alias, ', + ) + for line in cases: + with self.subTest(line): + completed = self.complete_line(line, 'test.test_rl') + self.assertEqual(completed, 'test.test_rlcompleter') + completed = self.complete_line('import test . ', 'test_rl') + self.assertEqual(completed, 'test_rlcompleter') + + # Invalid imports should not cause exceptions + self.complete_line('import ', 'test.test_rlcompleter.rlcompleter') + self.complete_line( + 'import ', 'test.test_rlcompleter.rlcompleter.x') + self.complete_line('import ', 'test.nonexistant.x') + + def test_from_module(self): + # Test relative to "test.test_rlcompleter" + self.stdcompleter = rlcompleter.Completer(globals()) + completed = self.complete_line('from ', 'test.test_rl') + self.assertEqual(completed, 'test.test_rlcompleter') + completed = self.complete_line('from ', '.test_rl') + self.assertEqual(completed, '.test_rlcompleter') + self.assertEqual(self.complete_line('from . ', 'im'), 'import ') + # Only interpret "from" at the start of a statement + self.assertEqual(self.complete_line('yield from ', 'Fa'), 'False') + + # Invalid imports should not cause exceptions + self.complete_line('from ', '..x') + self.complete_line('from ', '...x') + + def test_from_import(self): + # Test relative to "test.test_rlcompleter" + self.stdcompleter = rlcompleter.Completer(globals()) + cases = ( + ('from rlcompleter import ', 'Comp', 'Completer'), + ('from test import ', 'test_rl', 'test_rlcompleter'), + ('from . import ', 'test_rl', 'test_rlcompleter'), + ('from .test_rlcompleter import ', 'TestR', 'TestRlcompleter'), + ( + 'from rlcompleter import ( # Comment\n' + ' other as alias,\n' + ' ', 'Comp', 'Completer', + ), + ) + for line, word, expected in cases: + with self.subTest(line=line, word=word): + self.assertEqual(self.complete_line(line, word), expected) + + # Invalid imports should not cause exceptions + self.complete_line( + 'from test.test_rlcompleter.rlcompleter import ', 'Completer') + self.complete_line('from test.nonexistant import ', 'x') + self.complete_line('from test.nonexistant.x import ', 'x') + self.complete_line('from .. import ', 'rlcompleter') + self.complete_line('from ...beyond import ', 'x') + + def test_tokenize_errors(self): + # These cause "tokenize" to raise exceptions, rather than yielding + # error tokens + # EOF in multi-line statement: + self.assertEqual(self.complete_line('(', 'Fa'), 'False') + # EOF in multi-line string (use Ctrl+V, Ctrl+J in Gnu Readline): + self.complete_line('"""abc\n', 'blaua') + + def complete_line(self, previous, word): + line = previous + word + with patch('readline.get_line_buffer', lambda: line), \ + patch('readline.get_begidx', lambda: len(previous)), \ + patch('readline.get_endidx', lambda: len(line)): + return self.stdcompleter.complete(word, 0) + if __name__ == '__main__': unittest.main() diff -r 0aaadc1c60fd Misc/NEWS --- a/Misc/NEWS Fri Oct 16 11:23:31 2015 +0300 +++ b/Misc/NEWS Fri Oct 16 11:10:47 2015 +0000 @@ -63,6 +63,8 @@ Library ------- +- Issue #. . .: Support for Readline completion of "import" statements. + - Issue #25406: Fixed a bug in C implementation of OrderedDict.move_to_end() that caused segmentation fault or hang in iterating after moving several items to the start of ordered dict.