diff -r 1ca30423b0c9 Doc/library/tempfile.rst --- a/Doc/library/tempfile.rst Fri May 22 00:39:57 2015 -0400 +++ b/Doc/library/tempfile.rst Fri May 22 12:10:21 2015 -0700 @@ -119,7 +119,7 @@ .. versionadded:: 3.2 -.. function:: mkstemp(suffix='', prefix='tmp', dir=None, text=False) +.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) Creates a temporary file in the most secure manner possible. There are no race conditions in the file's creation, assuming that the platform @@ -148,6 +148,16 @@ filename will have any nice properties, such as not requiring quoting when passed to external commands via ``os.popen()``. + *suffix*, *prefix*, and *dir* must all contain the same type, if specified. + If they are bytes, the returned name will be bytes instead of str. + If you want to force a bytes return value with otherwise default behavior, + pass ``suffix=b''``. + + A *prefix* value of ``None`` means use the return value of + :func:`gettempprefix` or :func:`gettempprefixb` as appropraite. + + A *suffix* value of ``None`` means use an appropriate empty value. + If *text* is specified, it indicates whether to open the file in binary mode (the default) or text mode. On some platforms, this makes no difference. @@ -156,8 +166,14 @@ file (as would be returned by :func:`os.open`) and the absolute pathname of that file, in that order. + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + *suffix* and *prefix* now accept and default to ``None`` to cause + an appropriate default value to be used. -.. function:: mkdtemp(suffix='', prefix='tmp', dir=None) + +.. function:: mkdtemp(suffix=None, prefix=None, dir=None) Creates a temporary directory in the most secure manner possible. There are no race conditions in the directory's creation. The directory is @@ -171,6 +187,12 @@ :func:`mkdtemp` returns the absolute pathname of the new directory. + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + *suffix* and *prefix* now accept and default to ``None`` to cause + an appropriate default value to be used. + .. function:: mktemp(suffix='', prefix='tmp', dir=None) @@ -239,12 +261,23 @@ :data:`tempdir` is not ``None``, this simply returns its contents; otherwise, the search described above is performed, and the result returned. +.. function:: gettempdirb() + + Same as :func:`gettempdir` but the return value is in bytes. + + .. versionadded:: 3.5 .. function:: gettempprefix() Return the filename prefix used to create temporary files. This does not contain the directory component. +.. function:: gettempprefixb() + + Same as :func:`gettempprefixb` but the return value is in bytes. + + .. versionadded:: 3.5 + Examples -------- diff -r 1ca30423b0c9 Lib/tempfile.py --- a/Lib/tempfile.py Fri May 22 00:39:57 2015 -0400 +++ b/Lib/tempfile.py Fri May 22 12:10:21 2015 -0700 @@ -6,6 +6,14 @@ except for 'mktemp'. 'mktemp' is subject to race conditions and should not be used; it is provided for backward compatibility only. +The default path names are returned as str. If you supply bytes as +input, all return values will be in bytes. Ex: + + >>> tempfile.mkstemp() + (4, '/tmp/tmptpu9nin8') + >>> tempfile.mkdtemp(suffix=b'') + b'/tmp/tmppbi8f0hy' + This module also provides some data items to the user: TMP_MAX - maximum number of names that will be tried before @@ -21,7 +29,8 @@ "mkstemp", "mkdtemp", # low level safe interfaces "mktemp", # deprecated unsafe interface "TMP_MAX", "gettempprefix", # constants - "tempdir", "gettempdir" + "tempdir", "gettempdir", + "gettempprefixb", "gettempdirb", ] @@ -32,6 +41,7 @@ import io as _io import os as _os import shutil as _shutil +import sys as _sys import errno as _errno from random import Random as _Random import weakref as _weakref @@ -55,8 +65,10 @@ else: TMP_MAX = 10000 -# Although it does not have an underscore for historical reasons, this -# variable is an internal implementation detail (see issue 10354). +# This variable _was_ unused for legacy reasons, see issue 10354. +# But as of 3.5 we actually use it at runtime so changing it would +# have a possibly desirable side effect... But we do not want to support +# that as an API. It is undocumented on purpose. Do not depend on this. template = "tmp" # Internal routines. @@ -82,6 +94,46 @@ else: return True + +def _infer_return_type(*args): + """Look at the type of all args and divine their implied return type.""" + return_type = None + for arg in args: + if arg is None: + continue + if isinstance(arg, bytes): + if return_type and return_type is not bytes: + raise TypeError("Can't mix bytes and non-bytes in " + "path components.") + return_type = bytes + else: + if return_type and return_type is not str: + raise TypeError("Can't mix bytes and non-bytes in " + "path components.") + return_type = str + if return_type is None: + return str # tempfile APIs return a str by default. + return return_type + + +def _sanitize_params(prefix, suffix, dir): + """Common parameter processing for most APIs in this module.""" + output_type = _infer_return_type(prefix, suffix, dir) + if suffix is None: + suffix = output_type() + if prefix is None: + if output_type is str: + prefix = template + else: + prefix = _os.fsencode(template) + if dir is None: + if output_type is str: + dir = gettempdir() + else: + dir = gettempdirb() + return prefix, suffix, dir, output_type + + class _RandomNameSequence: """An instance of _RandomNameSequence generates an endless sequence of unpredictable strings which can safely be incorporated @@ -195,17 +247,18 @@ return _name_sequence -def _mkstemp_inner(dir, pre, suf, flags): +def _mkstemp_inner(dir, pre, suf, flags, output_type): """Code common to mkstemp, TemporaryFile, and NamedTemporaryFile.""" names = _get_candidate_names() + if output_type is bytes: + names = map(_os.fsencode, names) for seq in range(TMP_MAX): name = next(names) file = _os.path.join(dir, pre + name + suf) try: fd = _os.open(file, flags, 0o600) - return (fd, _os.path.abspath(file)) except FileExistsError: continue # try again except PermissionError: @@ -216,6 +269,7 @@ continue else: raise + return (fd, _os.path.abspath(file)) raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") @@ -224,9 +278,13 @@ # User visible interfaces. def gettempprefix(): - """Accessor for tempdir.template.""" + """The default prefix for temporary directories.""" return template +def gettempprefixb(): + """The default prefix for temporary directories as bytes.""" + return _os.fsencode(gettempprefix()) + tempdir = None def gettempdir(): @@ -241,7 +299,11 @@ _once_lock.release() return tempdir -def mkstemp(suffix="", prefix=template, dir=None, text=False): +def gettempdirb(): + """A bytes version of tempfile.gettempdir().""" + return _os.fsencode(gettempdir()) + +def mkstemp(suffix=None, prefix=None, dir=None, text=False): """User-callable function to create and return a unique temporary file. The return value is a pair (fd, name) where fd is the file descriptor returned by os.open, and name is the filename. @@ -259,6 +321,10 @@ mode. Else (the default) the file is opened in binary mode. On some operating systems, this makes no difference. + suffix, prefix and dir must all contain the same type if specified. + If they are bytes, the returned name will be bytes; str otherwise. + A value of None will cause an appropriate default to be used. + The file is readable and writable only by the creating user ID. If the operating system uses permission bits to indicate whether a file is executable, the file is executable by no one. The file @@ -267,18 +333,17 @@ Caller is responsible for deleting the file when done with it. """ - if dir is None: - dir = gettempdir() + prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) if text: flags = _text_openflags else: flags = _bin_openflags - return _mkstemp_inner(dir, prefix, suffix, flags) + return _mkstemp_inner(dir, prefix, suffix, flags, output_type) -def mkdtemp(suffix="", prefix=template, dir=None): +def mkdtemp(suffix=None, prefix=None, dir=None): """User-callable function to create and return a unique temporary directory. The return value is the pathname of the directory. @@ -291,17 +356,17 @@ Caller is responsible for deleting the directory when done with it. """ - if dir is None: - dir = gettempdir() + prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) names = _get_candidate_names() for seq in range(TMP_MAX): name = next(names) + if output_type is bytes: + name = _os.fsencode(name) file = _os.path.join(dir, prefix + name + suffix) try: _os.mkdir(file, 0o700) - return file except FileExistsError: continue # try again except PermissionError: @@ -312,6 +377,7 @@ continue else: raise + return file raise FileExistsError(_errno.EEXIST, "No usable temporary directory name found") @@ -323,8 +389,8 @@ Arguments are as for mkstemp, except that the 'text' argument is not accepted. - This function is unsafe and should not be used. The file name - refers to a file that did not exist at some point, but by the time + THIS FUNCTION IS UNSAFE AND SHOULD NOT BE USED. The file name may + refer to a file that did not exist at some point, but by the time you get around to creating it, someone else may have beaten you to the punch. """ @@ -454,7 +520,7 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, - newline=None, suffix="", prefix=template, + newline=None, suffix=None, prefix=None, dir=None, delete=True): """Create and return a temporary file. Arguments: @@ -471,8 +537,7 @@ when it is closed unless the 'delete' argument is set to False. """ - if dir is None: - dir = gettempdir() + prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) flags = _bin_openflags @@ -481,7 +546,7 @@ if _os.name == 'nt' and delete: flags |= _os.O_TEMPORARY - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags) + (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: file = _io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) @@ -503,7 +568,7 @@ _O_TMPFILE_WORKS = hasattr(_os, 'O_TMPFILE') def TemporaryFile(mode='w+b', buffering=-1, encoding=None, - newline=None, suffix="", prefix=template, + newline=None, suffix=None, prefix=None, dir=None): """Create and return a temporary file. Arguments: @@ -519,8 +584,7 @@ """ global _O_TMPFILE_WORKS - if dir is None: - dir = gettempdir() + prefix, suffix, dir, output_type = _sanitize_params(prefix, suffix, dir) flags = _bin_openflags if _O_TMPFILE_WORKS: @@ -544,7 +608,7 @@ raise # Fallback to _mkstemp_inner(). - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags) + (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: _os.unlink(name) return _io.open(fd, mode, buffering=buffering, @@ -562,7 +626,7 @@ def __init__(self, max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, - suffix="", prefix=template, dir=None): + suffix=None, prefix=None, dir=None): if 'b' in mode: self._file = _io.BytesIO() else: @@ -713,7 +777,7 @@ in it are removed. """ - def __init__(self, suffix="", prefix=template, dir=None): + def __init__(self, suffix=None, prefix=None, dir=None): self.name = mkdtemp(suffix, prefix, dir) self._finalizer = _weakref.finalize( self, self._cleanup, self.name, diff -r 1ca30423b0c9 Lib/test/test_tempfile.py --- a/Lib/test/test_tempfile.py Fri May 22 00:39:57 2015 -0400 +++ b/Lib/test/test_tempfile.py Fri May 22 12:10:21 2015 -0700 @@ -36,10 +36,38 @@ # in order of their appearance in the file. Testing which requires # threads is not done here. +class TestLowLevelInternals(unittest.TestCase): + def test_infer_return_type_singles(self): + self.assertIs(str, tempfile._infer_return_type('')) + self.assertIs(bytes, tempfile._infer_return_type(b'')) + self.assertIs(str, tempfile._infer_return_type(None)) + + def test_infer_return_type_multiples(self): + self.assertIs(str, tempfile._infer_return_type('', '')) + self.assertIs(bytes, tempfile._infer_return_type(b'', b'')) + with self.assertRaises(TypeError): + tempfile._infer_return_type('', b'') + with self.assertRaises(TypeError): + tempfile._infer_return_type(b'', '') + + def test_infer_return_type_multiples_and_none(self): + self.assertIs(str, tempfile._infer_return_type(None, '')) + self.assertIs(str, tempfile._infer_return_type('', None)) + self.assertIs(str, tempfile._infer_return_type(None, None)) + self.assertIs(bytes, tempfile._infer_return_type(b'', None)) + self.assertIs(bytes, tempfile._infer_return_type(None, b'')) + with self.assertRaises(TypeError): + tempfile._infer_return_type('', None, b'') + with self.assertRaises(TypeError): + tempfile._infer_return_type(b'', None, '') + + # Common functionality. + class BaseTestCase(unittest.TestCase): str_check = re.compile(r"^[a-z0-9_-]{8}$") + b_check = re.compile(br"^[a-z0-9_-]{8}$") def setUp(self): self._warnings_manager = support.check_warnings() @@ -56,18 +84,31 @@ npre = nbase[:len(pre)] nsuf = nbase[len(nbase)-len(suf):] + if dir is not None: + self.assertIs(type(name), str if type(dir) is str else bytes, + "unexpected return type") + if pre is not None: + self.assertIs(type(name), str if type(pre) is str else bytes, + "unexpected return type") + if suf is not None: + self.assertIs(type(name), str if type(suf) is str else bytes, + "unexpected return type") + if (dir, pre, suf) == (None, None, None): + self.assertIs(type(name), str, "default return type must be str") + # check for equality of the absolute paths! self.assertEqual(os.path.abspath(ndir), os.path.abspath(dir), - "file '%s' not in directory '%s'" % (name, dir)) + "file %r not in directory %r" % (name, dir)) self.assertEqual(npre, pre, - "file '%s' does not begin with '%s'" % (nbase, pre)) + "file %r does not begin with %r" % (nbase, pre)) self.assertEqual(nsuf, suf, - "file '%s' does not end with '%s'" % (nbase, suf)) + "file %r does not end with %r" % (nbase, suf)) nbase = nbase[len(pre):len(nbase)-len(suf)] - self.assertTrue(self.str_check.match(nbase), - "random string '%s' does not match ^[a-z0-9_-]{8}$" - % nbase) + check = self.str_check if isinstance(nbase, str) else self.b_check + self.assertTrue(check.match(nbase), + "random characters %r do not match %r" + % (nbase, check.pattern)) class TestExports(BaseTestCase): @@ -83,7 +124,9 @@ "mktemp" : 1, "TMP_MAX" : 1, "gettempprefix" : 1, + "gettempprefixb" : 1, "gettempdir" : 1, + "gettempdirb" : 1, "tempdir" : 1, "template" : 1, "SpooledTemporaryFile" : 1, @@ -320,7 +363,8 @@ if bin: flags = self._bflags else: flags = self._tflags - (self.fd, self.name) = tempfile._mkstemp_inner(dir, pre, suf, flags) + output_type = tempfile._infer_return_type(dir, pre, suf) + (self.fd, self.name) = tempfile._mkstemp_inner(dir, pre, suf, flags, output_type) def write(self, str): os.write(self.fd, str) @@ -329,9 +373,17 @@ self._close(self.fd) self._unlink(self.name) - def do_create(self, dir=None, pre="", suf="", bin=1): + def do_create(self, dir=None, pre=None, suf=None, bin=1): + output_type = tempfile._infer_return_type(dir, pre, suf) if dir is None: - dir = tempfile.gettempdir() + if output_type is str: + dir = tempfile.gettempdir() + else: + dir = tempfile.gettempdirb() + if pre is None: + pre = output_type() + if suf is None: + suf = output_type() file = self.mkstemped(dir, pre, suf, bin) self.nameCheck(file.name, dir, pre, suf) @@ -345,6 +397,23 @@ self.do_create(pre="a", suf="b").write(b"blat") self.do_create(pre="aa", suf=".txt").write(b"blat") + def test_basic_with_bytes_names(self): + # _mkstemp_inner can create files when given name parts all + # specified as bytes. + dir_b = tempfile.gettempdirb() + self.do_create(dir=dir_b, suf=b"").write(b"blat") + self.do_create(dir=dir_b, pre=b"a").write(b"blat") + self.do_create(dir=dir_b, suf=b"b").write(b"blat") + self.do_create(dir=dir_b, pre=b"a", suf=b"b").write(b"blat") + self.do_create(dir=dir_b, pre=b"aa", suf=b".txt").write(b"blat") + # Can't mix str & binary types in the args. + with self.assertRaises(TypeError): + self.do_create(dir="", suf=b"").write(b"blat") + with self.assertRaises(TypeError): + self.do_create(dir=dir_b, pre="").write(b"blat") + with self.assertRaises(TypeError): + self.do_create(dir=dir_b, pre=b"", suf="").write(b"blat") + def test_basic_many(self): # _mkstemp_inner can create many files (stochastic) extant = list(range(TEST_FILES)) @@ -424,9 +493,10 @@ def make_temp(self): return tempfile._mkstemp_inner(tempfile.gettempdir(), - tempfile.template, + tempfile.gettempprefix(), '', - tempfile._bin_openflags) + tempfile._bin_openflags, + str) def test_collision_with_existing_file(self): # _mkstemp_inner tries another name when a file with @@ -462,7 +532,12 @@ p = tempfile.gettempprefix() self.assertIsInstance(p, str) - self.assertTrue(len(p) > 0) + self.assertGreater(len(p), 0) + + pb = tempfile.gettempprefixb() + + self.assertIsInstance(pb, bytes) + self.assertGreater(len(pb), 0) def test_usable_template(self): # gettempprefix returns a usable prefix string @@ -487,11 +562,11 @@ def test_directory_exists(self): # gettempdir returns a directory which exists - dir = tempfile.gettempdir() - self.assertTrue(os.path.isabs(dir) or dir == os.curdir, - "%s is not an absolute path" % dir) - self.assertTrue(os.path.isdir(dir), - "%s is not a directory" % dir) + for d in (tempfile.gettempdir(), tempfile.gettempdirb()): + self.assertTrue(os.path.isabs(d) or d == os.curdir, + "%r is not an absolute path" % d) + self.assertTrue(os.path.isdir(d), + "%r is not a directory" % d) def test_directory_writable(self): # gettempdir returns a directory writable by the user @@ -507,8 +582,11 @@ # gettempdir always returns the same object a = tempfile.gettempdir() b = tempfile.gettempdir() + c = tempfile.gettempdirb() self.assertTrue(a is b) + self.assertNotEqual(type(a), type(c)) + self.assertEqual(a, os.fsdecode(c)) def test_case_sensitive(self): # gettempdir should not flatten its case @@ -528,9 +606,17 @@ class TestMkstemp(BaseTestCase): """Test mkstemp().""" - def do_create(self, dir=None, pre="", suf=""): + def do_create(self, dir=None, pre=None, suf=None): + output_type = tempfile._infer_return_type(dir, pre, suf) if dir is None: - dir = tempfile.gettempdir() + if output_type is str: + dir = tempfile.gettempdir() + else: + dir = tempfile.gettempdirb() + if pre is None: + pre = output_type() + if suf is None: + suf = output_type() (fd, name) = tempfile.mkstemp(dir=dir, prefix=pre, suffix=suf) (ndir, nbase) = os.path.split(name) adir = os.path.abspath(dir) @@ -552,6 +638,24 @@ self.do_create(pre="aa", suf=".txt") self.do_create(dir=".") + def test_basic_with_bytes_names(self): + # mkstemp can create files when given name parts all + # specified as bytes. + d = tempfile.gettempdirb() + self.do_create(dir=d, suf=b"") + self.do_create(dir=d, pre=b"a") + self.do_create(dir=d, suf=b"b") + self.do_create(dir=d, pre=b"a", suf=b"b") + self.do_create(dir=d, pre=b"aa", suf=b".txt") + self.do_create(dir=b".") + with self.assertRaises(TypeError): + self.do_create(dir=".", pre=b"aa", suf=b".txt") + with self.assertRaises(TypeError): + self.do_create(dir=b".", pre="aa", suf=b".txt") + with self.assertRaises(TypeError): + self.do_create(dir=b".", pre=b"aa", suf=".txt") + + def test_choose_directory(self): # mkstemp can create directories in a user-selected directory dir = tempfile.mkdtemp() @@ -567,9 +671,17 @@ def make_temp(self): return tempfile.mkdtemp() - def do_create(self, dir=None, pre="", suf=""): + def do_create(self, dir=None, pre=None, suf=None): + output_type = tempfile._infer_return_type(dir, pre, suf) if dir is None: - dir = tempfile.gettempdir() + if output_type is str: + dir = tempfile.gettempdir() + else: + dir = tempfile.gettempdirb() + if pre is None: + pre = output_type() + if suf is None: + suf = output_type() name = tempfile.mkdtemp(dir=dir, prefix=pre, suffix=suf) try: @@ -587,6 +699,21 @@ os.rmdir(self.do_create(pre="a", suf="b")) os.rmdir(self.do_create(pre="aa", suf=".txt")) + def test_basic_with_bytes_names(self): + # mkdtemp can create directories when given all binary parts + d = tempfile.gettempdirb() + os.rmdir(self.do_create(dir=d)) + os.rmdir(self.do_create(dir=d, pre=b"a")) + os.rmdir(self.do_create(dir=d, suf=b"b")) + os.rmdir(self.do_create(dir=d, pre=b"a", suf=b"b")) + os.rmdir(self.do_create(dir=d, pre=b"aa", suf=b".txt")) + with self.assertRaises(TypeError): + os.rmdir(self.do_create(dir=d, pre="aa", suf=b".txt")) + with self.assertRaises(TypeError): + os.rmdir(self.do_create(dir=d, pre=b"aa", suf=".txt")) + with self.assertRaises(TypeError): + os.rmdir(self.do_create(dir="", pre=b"aa", suf=b".txt")) + def test_basic_many(self): # mkdtemp can create many directories (stochastic) extant = list(range(TEST_FILES))