diff -r 1134198e23bd Doc/library/tempfile.rst --- a/Doc/library/tempfile.rst Wed May 20 00:14:00 2015 +0300 +++ b/Doc/library/tempfile.rst Tue May 19 22:06:48 2015 -0700 @@ -148,6 +148,10 @@ filename will have any nice properties, such as not requiring quoting when passed to external commands via ``os.popen()``. + *suffix*, *prefix*, and *dir* must all contain the same type, if specified. + If they are bytes or bytearray, the returned name will be bytes + instead of str. + If *text* is specified, it indicates whether to open the file in binary mode (the default) or text mode. On some platforms, this makes no difference. @@ -156,6 +160,10 @@ file (as would be returned by :func:`os.open`) and the absolute pathname of that file, in that order. + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + .. function:: mkdtemp(suffix='', prefix='tmp', dir=None) @@ -171,6 +179,10 @@ :func:`mkdtemp` returns the absolute pathname of the new directory. + .. versionchanged:: 3.5 + *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to + obtain a bytes return value. Prior to this, only str was allowed. + .. function:: mktemp(suffix='', prefix='tmp', dir=None) @@ -239,12 +251,23 @@ :data:`tempdir` is not ``None``, this simply returns its contents; otherwise, the search described above is performed, and the result returned. +.. function:: gettempdirb() + + Same as :func:`gettempdir` but the return value is in bytes. + + .. versionadded:: 3.5 .. function:: gettempprefix() Return the filename prefix used to create temporary files. This does not contain the directory component. +.. function:: gettempprefixb() + + Same as :func:`gettempprefixb` but the return value is in bytes. + + .. versionadded:: 3.5 + Examples -------- diff -r 1134198e23bd Lib/tempfile.py --- a/Lib/tempfile.py Wed May 20 00:14:00 2015 +0300 +++ b/Lib/tempfile.py Tue May 19 22:06:48 2015 -0700 @@ -6,6 +6,14 @@ except for 'mktemp'. 'mktemp' is subject to race conditions and should not be used; it is provided for backward compatibility only. +The default path names are returned as str. If you supply bytes as +input, all return values will be in bytes. Ex: + + >>> tempfile.mkstemp() + (4, '/tmp/tmptpu9nin8') + >>> tempfile.mkdtemp(suffix=b'') + b'/tmp/tmppbi8f0hy' + This module also provides some data items to the user: TMP_MAX - maximum number of names that will be tried before @@ -21,7 +29,8 @@ "mkstemp", "mkdtemp", # low level safe interfaces "mktemp", # deprecated unsafe interface "TMP_MAX", "gettempprefix", # constants - "tempdir", "gettempdir" + "tempdir", "gettempdir", + "gettempprefixb", "gettempdirb", ] @@ -32,6 +41,7 @@ import io as _io import os as _os import shutil as _shutil +import sys as _sys import errno as _errno from random import Random as _Random import weakref as _weakref @@ -55,8 +65,10 @@ else: TMP_MAX = 10000 -# Although it does not have an underscore for historical reasons, this -# variable is an internal implementation detail (see issue 10354). +# This variable _was_ unused for legacy reasons, see issue 10354. +# But as of 3.5 we actually use it at runtime so changing it would +# have a possibly desirable side effect... But we do not want to support +# that as an API. It is undocumented on purpose. Do not depend on this. template = "tmp" # Internal routines. @@ -82,6 +94,41 @@ else: return True + +# TODO(gps): Does this function exist somewhere else in the stdlib? +def _convert_path_to_bytes(path_str): + return path_str.encode(_sys.getfilesystemencoding(), + 'surrogateescape') + + +def _default_prefix(desired_type): + if desired_type is str: + return template + return _convert_path_to_bytes(template) + + +def _infer_return_type(*args): + """Look at the type of all args and divine their implied return type.""" + return_type = None + for arg in args: + if arg is None: + continue + if isinstance(arg, str): + if return_type and return_type is not str: + raise TypeError("Can't mix strings and bytes in path components.") + return_type = str + elif isinstance(arg, (bytes, bytearray)): + if return_type and return_type is not bytes: + raise TypeError("Can't mix strings and bytes in path components.") + return_type = bytes + else: + raise TypeError('Unsupported input type {!r}'.format(type(arg))) + if return_type is None: + return str # Return a str when no indication is given at all. + return return_type + + + class _RandomNameSequence: """An instance of _RandomNameSequence generates an endless sequence of unpredictable strings which can safely be incorporated @@ -195,17 +242,18 @@ return _name_sequence -def _mkstemp_inner(dir, pre, suf, flags): +def _mkstemp_inner(dir, pre, suf, flags, output_type): """Code common to mkstemp, TemporaryFile, and NamedTemporaryFile.""" names = _get_candidate_names() for seq in range(TMP_MAX): name = next(names) + if output_type is bytes: + name = _convert_path_to_bytes(name) file = _os.path.join(dir, pre + name + suf) try: fd = _os.open(file, flags, 0o600) - return (fd, _os.path.abspath(file)) except FileExistsError: continue # try again except PermissionError: @@ -216,6 +264,7 @@ continue else: raise + return (fd, _os.path.abspath(file)) raise FileExistsError(_errno.EEXIST, "No usable temporary file name found") @@ -224,9 +273,13 @@ # User visible interfaces. def gettempprefix(): - """Accessor for tempdir.template.""" + """The default prefix for temporary directories.""" return template +def gettempprefixb(): + """The default prefix for temporary directories as bytes.""" + return _convert_path_to_bytes(gettempprefix()) + tempdir = None def gettempdir(): @@ -241,7 +294,11 @@ _once_lock.release() return tempdir -def mkstemp(suffix="", prefix=template, dir=None, text=False): +def gettempdirb(): + """A bytes version of tempfile.gettempdir().""" + return _convert_path_to_bytes(gettempdir()) + +def mkstemp(suffix=None, prefix=None, dir=None, text=False): """User-callable function to create and return a unique temporary file. The return value is a pair (fd, name) where fd is the file descriptor returned by os.open, and name is the filename. @@ -259,6 +316,10 @@ mode. Else (the default) the file is opened in binary mode. On some operating systems, this makes no difference. + suffix, prefix and dir must all contain the same type if specified. + If they are bytes or bytearray, the returned name will be bytes + instead of str. + The file is readable and writable only by the creating user ID. If the operating system uses permission bits to indicate whether a file is executable, the file is executable by no one. The file @@ -267,18 +328,25 @@ Caller is responsible for deleting the file when done with it. """ + output_type = _infer_return_type(prefix, suffix, dir) + if suffix is None: + suffix = output_type() + if prefix is None: + prefix = _default_prefix(output_type) if dir is None: dir = gettempdir() + if output_type is bytes: + dir = _convert_path_to_bytes(dir) if text: flags = _text_openflags else: flags = _bin_openflags - return _mkstemp_inner(dir, prefix, suffix, flags) + return _mkstemp_inner(dir, prefix, suffix, flags, output_type) -def mkdtemp(suffix="", prefix=template, dir=None): +def mkdtemp(suffix=None, prefix=None, dir=None): """User-callable function to create and return a unique temporary directory. The return value is the pathname of the directory. @@ -291,17 +359,26 @@ Caller is responsible for deleting the directory when done with it. """ + output_type = _infer_return_type(prefix, suffix, dir) + if suffix is None: + suffix = output_type() + if prefix is None: + prefix = _default_prefix(output_type) if dir is None: - dir = gettempdir() + if output_type is str: + dir = gettempdir() + else: + dir = gettempdirb() names = _get_candidate_names() for seq in range(TMP_MAX): name = next(names) + if output_type is bytes: + name = _convert_path_to_bytes(name) file = _os.path.join(dir, prefix + name + suffix) try: _os.mkdir(file, 0o700) - return file except FileExistsError: continue # try again except PermissionError: @@ -312,6 +389,7 @@ continue else: raise + return file raise FileExistsError(_errno.EEXIST, "No usable temporary directory name found") @@ -323,8 +401,8 @@ Arguments are as for mkstemp, except that the 'text' argument is not accepted. - This function is unsafe and should not be used. The file name - refers to a file that did not exist at some point, but by the time + THIS FUNCTION IS UNSAFE AND SHOULD NOT BE USED. The file name may + refer to a file that did not exist at some point, but by the time you get around to creating it, someone else may have beaten you to the punch. """ @@ -454,7 +532,7 @@ def NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, - newline=None, suffix="", prefix=template, + newline=None, suffix=None, prefix=None, dir=None, delete=True): """Create and return a temporary file. Arguments: @@ -471,8 +549,15 @@ when it is closed unless the 'delete' argument is set to False. """ + output_type = _infer_return_type(prefix, suffix, dir) + if suffix is None: + suffix = output_type() + if prefix is None: + prefix = _default_prefix(output_type) if dir is None: dir = gettempdir() + if output_type is bytes: + dir = _convert_path_to_bytes(dir) flags = _bin_openflags @@ -481,7 +566,7 @@ if _os.name == 'nt' and delete: flags |= _os.O_TEMPORARY - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags) + (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: file = _io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) @@ -503,7 +588,7 @@ _O_TMPFILE_WORKS = hasattr(_os, 'O_TMPFILE') def TemporaryFile(mode='w+b', buffering=-1, encoding=None, - newline=None, suffix="", prefix=template, + newline=None, suffix=None, prefix=None, dir=None): """Create and return a temporary file. Arguments: @@ -519,8 +604,15 @@ """ global _O_TMPFILE_WORKS + output_type = _infer_return_type(prefix, suffix, dir) + if suffix is None: + suffix = output_type() + if prefix is None: + prefix = _default_prefix(output_type) if dir is None: dir = gettempdir() + if output_type is bytes: + dir = _convert_path_to_bytes(dir) flags = _bin_openflags if _O_TMPFILE_WORKS: @@ -544,7 +636,7 @@ raise # Fallback to _mkstemp_inner(). - (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags) + (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: _os.unlink(name) return _io.open(fd, mode, buffering=buffering, @@ -562,7 +654,7 @@ def __init__(self, max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, - suffix="", prefix=template, dir=None): + suffix=None, prefix=None, dir=None): if 'b' in mode: self._file = _io.BytesIO() else: @@ -713,7 +805,7 @@ in it are removed. """ - def __init__(self, suffix="", prefix=template, dir=None): + def __init__(self, suffix=None, prefix=None, dir=None): self.name = mkdtemp(suffix, prefix, dir) self._finalizer = _weakref.finalize( self, self._cleanup, self.name, diff -r 1134198e23bd Lib/test/test_tempfile.py --- a/Lib/test/test_tempfile.py Wed May 20 00:14:00 2015 +0300 +++ b/Lib/test/test_tempfile.py Tue May 19 22:06:48 2015 -0700 @@ -36,10 +36,50 @@ # in order of their appearance in the file. Testing which requires # threads is not done here. +class TestLowLevelInternals(unittest.TestCase): + def test_convert_path_to_bytes(self): + self.assertEqual(b'g', tempfile._convert_path_to_bytes('g')) + + def test_default_prefix(self): + self.assertEqual('tmp', tempfile._default_prefix(str)) + self.assertEqual(b'tmp', tempfile._default_prefix(bytes)) + self.assertEqual(b'tmp', tempfile._default_prefix(bytearray)) + + def test_infer_return_type_singles(self): + self.assertIs(str, tempfile._infer_return_type('')) + self.assertIs(bytes, tempfile._infer_return_type(b'')) + self.assertIs(bytes, tempfile._infer_return_type(bytearray(b''))) + self.assertIs(str, tempfile._infer_return_type(None)) + with self.assertRaises(TypeError): + tempfile._infer_return_type(5) + + def test_infer_return_type_multiples_and_none(self): + self.assertIs(str, tempfile._infer_return_type('', '')) + self.assertIs(bytes, tempfile._infer_return_type(b'', b'')) + self.assertIs(bytes, tempfile._infer_return_type(b'', bytearray(b''))) + with self.assertRaises(TypeError): + tempfile._infer_return_type('', b'') + with self.assertRaises(TypeError): + tempfile._infer_return_type(b'', '') + + def test_infer_return_type_multiples_and_none(self): + self.assertIs(str, tempfile._infer_return_type(None, '')) + self.assertIs(str, tempfile._infer_return_type('', None)) + self.assertIs(str, tempfile._infer_return_type(None, None)) + self.assertIs(bytes, tempfile._infer_return_type(b'', None)) + self.assertIs(bytes, tempfile._infer_return_type(None, b'')) + with self.assertRaises(TypeError): + tempfile._infer_return_type('', None, b'') + with self.assertRaises(TypeError): + tempfile._infer_return_type(b'', None, '') + + # Common functionality. + class BaseTestCase(unittest.TestCase): str_check = re.compile(r"^[a-z0-9_-]{8}$") + b_check = re.compile(br"^[a-z0-9_-]{8}$") def setUp(self): self._warnings_manager = support.check_warnings() @@ -56,18 +96,31 @@ npre = nbase[:len(pre)] nsuf = nbase[len(nbase)-len(suf):] + if dir is not None: + self.assertIs(type(name), str if type(dir) is str else bytes, + "unexpected return type") + if pre is not None: + self.assertIs(type(name), str if type(pre) is str else bytes, + "unexpected return type") + if suf is not None: + self.assertIs(type(name), str if type(suf) is str else bytes, + "unexpected return type") + if (dir, pre, suf) == (None, None, None): + self.assertIs(type(name), str, "default return type must be str") + # check for equality of the absolute paths! self.assertEqual(os.path.abspath(ndir), os.path.abspath(dir), - "file '%s' not in directory '%s'" % (name, dir)) + "file %r not in directory %r" % (name, dir)) self.assertEqual(npre, pre, - "file '%s' does not begin with '%s'" % (nbase, pre)) + "file %r does not begin with %r" % (nbase, pre)) self.assertEqual(nsuf, suf, - "file '%s' does not end with '%s'" % (nbase, suf)) + "file %r does not end with %r" % (nbase, suf)) nbase = nbase[len(pre):len(nbase)-len(suf)] - self.assertTrue(self.str_check.match(nbase), - "random string '%s' does not match ^[a-z0-9_-]{8}$" - % nbase) + check = self.str_check if isinstance(nbase, str) else self.b_check + self.assertTrue(check.match(nbase), + "random characters %r do not match r%r" + % (nbase, check.pattern)) class TestExports(BaseTestCase): @@ -83,7 +136,9 @@ "mktemp" : 1, "TMP_MAX" : 1, "gettempprefix" : 1, + "gettempprefixb" : 1, "gettempdir" : 1, + "gettempdirb" : 1, "tempdir" : 1, "template" : 1, "SpooledTemporaryFile" : 1, @@ -320,7 +375,8 @@ if bin: flags = self._bflags else: flags = self._tflags - (self.fd, self.name) = tempfile._mkstemp_inner(dir, pre, suf, flags) + output_type = tempfile._infer_return_type(dir, pre, suf) + (self.fd, self.name) = tempfile._mkstemp_inner(dir, pre, suf, flags, output_type) def write(self, str): os.write(self.fd, str) @@ -329,9 +385,17 @@ self._close(self.fd) self._unlink(self.name) - def do_create(self, dir=None, pre="", suf="", bin=1): + def do_create(self, dir=None, pre=None, suf=None, bin=1): + output_type = tempfile._infer_return_type(dir, pre, suf) if dir is None: - dir = tempfile.gettempdir() + if output_type is str: + dir = tempfile.gettempdir() + else: + dir = tempfile.gettempdirb() + if pre is None: + pre = output_type() + if suf is None: + suf = output_type() file = self.mkstemped(dir, pre, suf, bin) self.nameCheck(file.name, dir, pre, suf) @@ -345,6 +409,23 @@ self.do_create(pre="a", suf="b").write(b"blat") self.do_create(pre="aa", suf=".txt").write(b"blat") + def test_basic_with_bytes_names(self): + # _mkstemp_inner can create files when given name parts all + # specified as bytes or bytearrays. + dir_b = tempfile.gettempdirb() + self.do_create(dir=dir_b, suf=b"").write(b"blat") + self.do_create(dir=dir_b, pre=b"a").write(b"blat") + self.do_create(dir=dir_b, suf=b"b").write(b"blat") + self.do_create(dir=dir_b, pre=b"a", suf=bytearray(b"b")).write(b"blat") + self.do_create(dir=dir_b, pre=b"aa", suf=b".txt").write(b"blat") + # Can't mix str & binary types in the args. + with self.assertRaises(TypeError): + self.do_create(dir="", suf=b"").write(b"blat") + with self.assertRaises(TypeError): + self.do_create(dir=dir_b, pre="").write(b"blat") + with self.assertRaises(TypeError): + self.do_create(dir=dir_b, pre=b"", suf="").write(b"blat") + def test_basic_many(self): # _mkstemp_inner can create many files (stochastic) extant = list(range(TEST_FILES)) @@ -424,9 +505,10 @@ def make_temp(self): return tempfile._mkstemp_inner(tempfile.gettempdir(), - tempfile.template, + tempfile.gettempprefix(), '', - tempfile._bin_openflags) + tempfile._bin_openflags, + str) def test_collision_with_existing_file(self): # _mkstemp_inner tries another name when a file with @@ -462,7 +544,12 @@ p = tempfile.gettempprefix() self.assertIsInstance(p, str) - self.assertTrue(len(p) > 0) + self.assertGreater(len(p), 0) + + pb = tempfile.gettempprefixb() + + self.assertIsInstance(pb, bytes) + self.assertGreater(len(pb), 0) def test_usable_template(self): # gettempprefix returns a usable prefix string @@ -487,11 +574,11 @@ def test_directory_exists(self): # gettempdir returns a directory which exists - dir = tempfile.gettempdir() - self.assertTrue(os.path.isabs(dir) or dir == os.curdir, - "%s is not an absolute path" % dir) - self.assertTrue(os.path.isdir(dir), - "%s is not a directory" % dir) + for d in (tempfile.gettempdir(), tempfile.gettempdirb()): + self.assertTrue(os.path.isabs(d) or d == os.curdir, + "%r is not an absolute path" % d) + self.assertTrue(os.path.isdir(d), + "%r is not a directory" % d) def test_directory_writable(self): # gettempdir returns a directory writable by the user @@ -507,8 +594,10 @@ # gettempdir always returns the same object a = tempfile.gettempdir() b = tempfile.gettempdir() + c = tempfile.gettempdirb() self.assertTrue(a is b) + self.assertEqual(a, c.decode(sys.getfilesystemencoding())) def test_case_sensitive(self): # gettempdir should not flatten its case @@ -528,9 +617,17 @@ class TestMkstemp(BaseTestCase): """Test mkstemp().""" - def do_create(self, dir=None, pre="", suf=""): + def do_create(self, dir=None, pre=None, suf=None): + output_type = tempfile._infer_return_type(dir, pre, suf) if dir is None: - dir = tempfile.gettempdir() + if output_type is str: + dir = tempfile.gettempdir() + else: + dir = tempfile.gettempdirb() + if pre is None: + pre = output_type() + if suf is None: + suf = output_type() (fd, name) = tempfile.mkstemp(dir=dir, prefix=pre, suffix=suf) (ndir, nbase) = os.path.split(name) adir = os.path.abspath(dir) @@ -552,6 +649,24 @@ self.do_create(pre="aa", suf=".txt") self.do_create(dir=".") + def test_basic_with_bytes_names(self): + # mkstemp can create files when given name parts all + # specified as bytes or bytearrays. + d = tempfile.gettempdirb() + self.do_create(dir=d, suf=b"") + self.do_create(dir=d, pre=b"a") + self.do_create(dir=d, suf=b"b") + self.do_create(dir=d, pre=b"a", suf=b"b") + self.do_create(dir=d, pre=b"aa", suf=b".txt") + self.do_create(dir=b".") + with self.assertRaises(TypeError): + self.do_create(dir=".", pre=b"aa", suf=b".txt") + with self.assertRaises(TypeError): + self.do_create(dir=b".", pre="aa", suf=b".txt") + with self.assertRaises(TypeError): + self.do_create(dir=b".", pre=b"aa", suf=".txt") + + def test_choose_directory(self): # mkstemp can create directories in a user-selected directory dir = tempfile.mkdtemp() @@ -567,9 +682,17 @@ def make_temp(self): return tempfile.mkdtemp() - def do_create(self, dir=None, pre="", suf=""): + def do_create(self, dir=None, pre=None, suf=None): + output_type = tempfile._infer_return_type(dir, pre, suf) if dir is None: - dir = tempfile.gettempdir() + if output_type is str: + dir = tempfile.gettempdir() + else: + dir = tempfile.gettempdirb() + if pre is None: + pre = output_type() + if suf is None: + suf = output_type() name = tempfile.mkdtemp(dir=dir, prefix=pre, suffix=suf) try: @@ -587,6 +710,21 @@ os.rmdir(self.do_create(pre="a", suf="b")) os.rmdir(self.do_create(pre="aa", suf=".txt")) + def test_basic_with_bytes_names(self): + # mkdtemp can create directories when given all binary parts + d = tempfile.gettempdirb() + os.rmdir(self.do_create(dir=d)) + os.rmdir(self.do_create(dir=d, pre=b"a")) + os.rmdir(self.do_create(dir=d, suf=b"b")) + os.rmdir(self.do_create(dir=d, pre=b"a", suf=b"b")) + os.rmdir(self.do_create(dir=d, pre=b"aa", suf=b".txt")) + with self.assertRaises(TypeError): + os.rmdir(self.do_create(dir=d, pre="aa", suf=b".txt")) + with self.assertRaises(TypeError): + os.rmdir(self.do_create(dir=d, pre=b"aa", suf=".txt")) + with self.assertRaises(TypeError): + os.rmdir(self.do_create(dir="", pre=b"aa", suf=b".txt")) + def test_basic_many(self): # mkdtemp can create many directories (stochastic) extant = list(range(TEST_FILES))