diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst --- a/Doc/whatsnew/3.6.rst +++ b/Doc/whatsnew/3.6.rst @@ -167,6 +167,10 @@ * Optimize :meth:`bytes.fromhex` and :meth:`bytearray.fromhex`: they are now between 2x and 3.5x faster. (Contributed by Victor Stinner in :issue:`25401`). +* Optimized :func:`~glob.glob` and :func:`~glob.iglob` functions in the + :mod:`glob` module; they are now about 2--4 times faster. + (Contributed by Serhiy Storchaka in :issue:`25596`). + Build and C API Changes ======================= diff --git a/Lib/glob.py b/Lib/glob.py --- a/Lib/glob.py +++ b/Lib/glob.py @@ -30,101 +30,128 @@ If recursive is true, the pattern '**' will match any files and zero or more directories and subdirectories. """ - it = _iglob(pathname, recursive) + if not has_magic(pathname): + dirname, basename = os.path.split(pathname) + if basename: + if os.path.lexists(pathname): + return iter([pathname]) + else: + # Patterns ending with a slash should match only directories + if os.path.isdir(dirname): + return iter([pathname]) + return iter([]) + + it = _iglob(pathname, recursive, False) if recursive and _isrecursive(pathname): s = next(it) # skip empty string assert not s return it -def _iglob(pathname, recursive): +def _iglob(pathname, recursive, dironly): dirname, basename = os.path.split(pathname) - if not has_magic(pathname): - if basename: - if os.path.lexists(pathname): - yield pathname - else: - # Patterns ending with a slash should match only directories - if os.path.isdir(dirname): - yield pathname - return if not dirname: if recursive and _isrecursive(basename): - yield from glob2(dirname, basename) + yield from _glob2(dirname, basename, dironly) else: - yield from glob1(dirname, basename) + yield from _glob1(dirname, basename, dironly) return # `os.path.split()` returns the argument itself as a dirname if it is a # drive or UNC path. Prevent an infinite recursion if a drive or UNC path # contains magic characters (i.e. r'\\?\C:'). if dirname != pathname and has_magic(dirname): - dirs = _iglob(dirname, recursive) + dirs = _iglob(dirname, recursive, True) else: dirs = [dirname] - if has_magic(basename): - if recursive and _isrecursive(basename): - glob_in_dir = glob2 - else: - glob_in_dir = glob1 + if not has_magic(basename): + glob_in_dir = _glob0 + elif recursive and _isrecursive(basename): + glob_in_dir = _glob2 else: - glob_in_dir = glob0 + glob_in_dir = _glob1 for dirname in dirs: - for name in glob_in_dir(dirname, basename): + for name in glob_in_dir(dirname, basename, dironly): yield os.path.join(dirname, name) # These 2 helper functions non-recursively glob inside a literal directory. -# They return a list of basenames. `glob1` accepts a pattern while `glob0` +# They return an iterable of basenames. _glob1 accepts a pattern while _glob0 # takes a literal basename (so it only has to check for its existence). -def glob1(dirname, pattern): - if not dirname: - if isinstance(pattern, bytes): - dirname = bytes(os.curdir, 'ASCII') - else: - dirname = os.curdir - try: - names = os.listdir(dirname) - except OSError: - return [] +def _glob1(dirname, pattern, dironly): + names = list(x.name for x in _iterdir(dirname, dironly) + if not dironly or x.is_dir()) if not _ishidden(pattern): - names = [x for x in names if not _ishidden(x)] + names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) -def glob0(dirname, basename): +def _glob0(dirname, basename, dironly): if not basename: # `os.path.split()` returns an empty basename for paths ending with a # directory separator. 'q*x/' should match only directories. if os.path.isdir(dirname): return [basename] + elif dironly: + if os.path.isdir(os.path.join(dirname, basename)): + return [basename] else: if os.path.lexists(os.path.join(dirname, basename)): return [basename] return [] +# Following functions are not public but can be used by third-party code. + +def glob0(dirname, pattern): + import warnings + warnings.warn('glob.glob0() will be removed in 3.7', + DeprecationWarning, stacklevel=2) + return _glob0(dirname, pattern, False) + +def glob1(dirname, pattern): + import warnings + warnings.warn('glob.glob1() will be removed in 3.7', + DeprecationWarning, stacklevel=2) + return _glob1(dirname, pattern, False) + # This helper function recursively yields relative pathnames inside a literal # directory. -def glob2(dirname, pattern): +def _glob2(dirname, pattern, dironly): assert _isrecursive(pattern) yield pattern[:0] - yield from _rlistdir(dirname) + yield from _rlistdir(dirname, dironly) -# Recursively yields relative pathnames inside a literal directory. -def _rlistdir(dirname): +# If dironly is false, yields all file names inside a directory. +# If dironly is true, yields only directory or symlink (that can link to +# a directory) names. +def _iterdir(dirname, dironly): if not dirname: if isinstance(dirname, bytes): dirname = bytes(os.curdir, 'ASCII') else: dirname = os.curdir try: - names = os.listdir(dirname) - except os.error: + for entry in os.scandir(dirname): + try: + if not dironly or entry.is_dir(): + yield entry + except OSError: + pass + except OSError: return - for x in names: - if not _ishidden(x): - yield x - path = os.path.join(dirname, x) if dirname else x - for y in _rlistdir(path): - yield os.path.join(x, y) + +# Recursively yields relative pathnames inside a literal directory. +def _rlistdir(dirname, dironly): + entries = list(_iterdir(dirname, dironly)) + for x in entries: + name = x.name + if not _ishidden(name): + yield name + try: + isdir = x.is_dir() + except OSError: + continue + if isdir: + for y in _rlistdir(x.path, dironly): + yield os.path.join(name, y) magic_check = re.compile('([*?[])')