diff --git a/Lib/functools.py b/Lib/functools.py --- a/Lib/functools.py +++ b/Lib/functools.py @@ -21,6 +21,7 @@ from abc import get_cache_token from collections import namedtuple from types import MappingProxyType from weakref import WeakKeyDictionary +from reprlib import recursive_repr try: from _thread import RLock except ImportError: @@ -237,26 +238,91 @@ except ImportError: ################################################################################ # Purely functional, no descriptor behaviour -def partial(func, *args, **keywords): +class partial: """New function with partial application of the given arguments and keywords. """ - if hasattr(func, 'func'): - args = func.args + args - tmpkw = func.keywords.copy() - tmpkw.update(keywords) - keywords = tmpkw - del tmpkw - func = func.func - - def newfunc(*fargs, **fkeywords): - newkeywords = keywords.copy() - newkeywords.update(fkeywords) - return func(*(args + fargs), **newkeywords) - newfunc.func = func - newfunc.args = args - newfunc.keywords = keywords - return newfunc + + def __new__(*args, **keywords): + if not args: + raise TypeError("descriptor '__new__' of partial needs an argument") + if len(args) < 2: + raise TypeError("type 'partial' takes at least one argument") + cls, func, *args = args + if not callable(func): + raise TypeError("the first argument must be callable") + args = tuple(args) + + if hasattr(func, "func"): + args = func.args + args + tmpkw = func.keywords.copy() + tmpkw.update(keywords) + keywords = tmpkw + del tmpkw + func = func.func + + self = super(partial, cls).__new__(cls) + + self.func = func + self.args = args + self.keywords = keywords + return self + + def __call__(*args, **keywords): + if not args: + raise TypeError("descriptor '__call__' of partial needs an argument") + self, *args = args + newkeywords = self.keywords.copy() + newkeywords.update(keywords) + return self.func(*self.args, *args, **newkeywords) + + @recursive_repr() + def __repr__(self): + qualname = type(self).__qualname__ + args = [str(self.func)] + args.extend(repr(x) for x in self.args) + args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) + if type(self).__module__ == "functools": + return f"functools.{qualname}({', '.join(args)})" + return f"{qualname}({', '.join(args)})" + + def __reduce__(self): + return type(self), (self.func,), (self.func, self.args, + self.keywords or None, self.__dict__ or None) + + def __setstate__(self, state): + if not isinstance(state, tuple): + raise TypeError("argument to __setstate__ must be a tuple") + if len(state) != 4: + raise TypeError(f"expected 4 items in state, got {len(state)}") + func, args, kwds, namespace = state + if (not callable(func) or not isinstance(args, tuple) or + (kwds is not None and not isinstance(kwds, dict)) or + (namespace is not None and not isinstance(namespace, dict))): + raise TypeError("invalid partial state") + + if type(args) is not tuple: + args = tuple(args) # coerce subclasses + if kwds is None: + kwds = {} + elif type(kwds) is not dict: + kwds = dict(kwds) + if namespace is None: + namespace = {} + + err_msg = "{0} and __dict__[{0!r}] are not the same" + for name, attr in namespace.items(): + if name == "func" and attr != func: + raise TypeError(err_msg.format("func")) + elif name == "args" and attr != args: + raise TypeError(err_msg.format("args")) + elif name == "keywords" and attr != kwds: + raise TypeError(err_msg.format("keywords")) + + self.__dict__ = namespace + self.func = func + self.args = args + self.keywords = kwds try: from _functools import partial diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -8,6 +8,7 @@ import sys from test import support import unittest from weakref import proxy +import contextlib try: import threading except ImportError: @@ -20,6 +21,14 @@ c_functools = support.import_fresh_module('functools', fresh=['_functools']) decimal = support.import_fresh_module('decimal', fresh=['_decimal']) +@contextlib.contextmanager +def replaced_module(name, replacement): + original_module = sys.modules[name] + sys.modules[name] = replacement + try: + yield + finally: + sys.modules[name] = original_module def capture(*args, **kw): """capture all positional and keyword arguments""" @@ -167,58 +176,35 @@ class TestPartial: p2.new_attr = 'spam' self.assertEqual(p2.new_attr, 'spam') - -@unittest.skipUnless(c_functools, 'requires the C _functools module') -class TestPartialC(TestPartial, unittest.TestCase): - if c_functools: - partial = c_functools.partial - - def test_attributes_unwritable(self): - # attributes should not be writable - p = self.partial(capture, 1, 2, a=10, b=20) - self.assertRaises(AttributeError, setattr, p, 'func', map) - self.assertRaises(AttributeError, setattr, p, 'args', (1, 2)) - self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2)) - - p = self.partial(hex) - try: - del p.__dict__ - except TypeError: - pass - else: - self.fail('partial object allowed __dict__ to be deleted') - def test_repr(self): args = (object(), object()) args_repr = ', '.join(repr(a) for a in args) kwargs = {'a': object(), 'b': object()} kwargs_reprs = ['a={a!r}, b={b!r}'.format_map(kwargs), 'b={b!r}, a={a!r}'.format_map(kwargs)] - if self.partial is c_functools.partial: + if self.partial in (c_functools.partial, py_functools.partial): name = 'functools.partial' else: name = self.partial.__name__ f = self.partial(capture) - self.assertEqual('{}({!r})'.format(name, capture), - repr(f)) + self.assertEqual(f'{name}({capture!r})', repr(f)) f = self.partial(capture, *args) - self.assertEqual('{}({!r}, {})'.format(name, capture, args_repr), - repr(f)) + self.assertEqual(f'{name}({capture!r}, {args_repr})', repr(f)) f = self.partial(capture, **kwargs) self.assertIn(repr(f), - ['{}({!r}, {})'.format(name, capture, kwargs_repr) + [f'{name}({capture!r}, {kwargs_repr})' for kwargs_repr in kwargs_reprs]) f = self.partial(capture, *args, **kwargs) self.assertIn(repr(f), - ['{}({!r}, {}, {})'.format(name, capture, args_repr, kwargs_repr) + [f'{name}({capture!r}, {args_repr}, {kwargs_repr})' for kwargs_repr in kwargs_reprs]) def test_recursive_repr(self): - if self.partial is c_functools.partial: + if self.partial in (c_functools.partial, py_functools.partial): name = 'functools.partial' else: name = self.partial.__name__ @@ -245,11 +231,12 @@ class TestPartialC(TestPartial, unittest.TestCase): f.__setstate__((capture, (), {}, {})) def test_pickle(self): - f = self.partial(signature, ['asdf'], bar=[True]) - f.attr = [] - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - f_copy = pickle.loads(pickle.dumps(f, proto)) - self.assertEqual(signature(f_copy), signature(f)) + with self.AllowPickle(): + f = self.partial(signature, ['asdf'], bar=[True]) + f.attr = [] + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f_copy = pickle.loads(pickle.dumps(f, proto)) + self.assertEqual(signature(f_copy), signature(f)) def test_copy(self): f = self.partial(signature, ['asdf'], bar=[True]) @@ -272,14 +259,24 @@ class TestPartialC(TestPartial, unittest.TestCase): self.assertIsNot(f_copy.keywords['bar'], f.keywords['bar']) def test_setstate(self): + namespace_to_update = dict(func=capture, args=(1,), keywords=dict(a=10)) f = self.partial(signature) f.__setstate__((capture, (1,), dict(a=10), dict(attr=[]))) + namespace = dict(attr=[]) + if self.partial in (py_functools.partial, PyPartialSubclass): + # the Python version(s) hold those in the __dict__ too + namespace.update(namespace_to_update) + self.assertEqual(signature(f), - (capture, (1,), dict(a=10), dict(attr=[]))) + (capture, (1,), dict(a=10), namespace)) self.assertEqual(f(2, b=20), ((1, 2), {'a': 10, 'b': 20})) f.__setstate__((capture, (1,), dict(a=10), None)) - self.assertEqual(signature(f), (capture, (1,), dict(a=10), {})) + namespace = {} + if self.partial in (py_functools.partial, PyPartialSubclass): + namespace.update(namespace_to_update) + + self.assertEqual(signature(f), (capture, (1,), dict(a=10), namespace)) self.assertEqual(f(2, b=20), ((1, 2), {'a': 10, 'b': 20})) f.__setstate__((capture, (1,), None, None)) @@ -289,7 +286,10 @@ class TestPartialC(TestPartial, unittest.TestCase): self.assertEqual(f(), ((1,), {})) f.__setstate__((capture, (), {}, None)) - self.assertEqual(signature(f), (capture, (), {}, {})) + namespace = {} + if self.partial in (py_functools.partial, PyPartialSubclass): + namespace.update(dict(func=capture, args=(), keywords={})) + self.assertEqual(signature(f), (capture, (), {}, namespace)) self.assertEqual(f(2, b=20), ((2,), {'b': 20})) self.assertEqual(f(2), ((2,), {})) self.assertEqual(f(), ((), {})) @@ -308,7 +308,10 @@ class TestPartialC(TestPartial, unittest.TestCase): f = self.partial(signature) f.__setstate__((capture, MyTuple((1,)), MyDict(a=10), None)) s = signature(f) - self.assertEqual(s, (capture, (1,), dict(a=10), {})) + namespace = {} + if self.partial in (py_functools.partial, PyPartialSubclass): + namespace.update(dict(func=capture, args=(1,), keywords=dict(a=10))) + self.assertEqual(s, (capture, (1,), dict(a=10), namespace)) self.assertIs(type(s[1]), tuple) self.assertIs(type(s[2]), dict) r = f() @@ -318,45 +321,49 @@ class TestPartialC(TestPartial, unittest.TestCase): f.__setstate__((capture, BadTuple((1,)), {}, None)) s = signature(f) - self.assertEqual(s, (capture, (1,), {}, {})) + namespace = {} + if self.partial in (py_functools.partial, PyPartialSubclass): + namespace.update(dict(func=capture, args=(1,), keywords={})) + self.assertEqual(s, (capture, (1,), {}, namespace)) self.assertIs(type(s[1]), tuple) r = f(2) self.assertEqual(r, ((1, 2), {})) self.assertIs(type(r[0]), tuple) def test_recursive_pickle(self): - f = self.partial(capture) - f.__setstate__((f, (), {}, {})) - try: - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises(RecursionError): - pickle.dumps(f, proto) - finally: - f.__setstate__((capture, (), {}, {})) - - f = self.partial(capture) - f.__setstate__((capture, (f,), {}, {})) - try: - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - f_copy = pickle.loads(pickle.dumps(f, proto)) - try: - self.assertIs(f_copy.args[0], f_copy) - finally: - f_copy.__setstate__((capture, (), {}, {})) - finally: - f.__setstate__((capture, (), {}, {})) - - f = self.partial(capture) - f.__setstate__((capture, (), {'a': f}, {})) - try: - for proto in range(pickle.HIGHEST_PROTOCOL + 1): - f_copy = pickle.loads(pickle.dumps(f, proto)) - try: - self.assertIs(f_copy.keywords['a'], f_copy) - finally: - f_copy.__setstate__((capture, (), {}, {})) - finally: - f.__setstate__((capture, (), {}, {})) + with self.AllowPickle(): + f = self.partial(capture) + f.__setstate__((f, (), {}, {})) + try: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.assertRaises(RecursionError): + pickle.dumps(f, proto) + finally: + f.__setstate__((capture, (), {}, {})) + + f = self.partial(capture) + f.__setstate__((capture, (f,), {}, {})) + try: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f_copy = pickle.loads(pickle.dumps(f, proto)) + try: + self.assertIs(f_copy.args[0], f_copy) + finally: + f_copy.__setstate__((capture, (), {}, {})) + finally: + f.__setstate__((capture, (), {}, {})) + + f = self.partial(capture) + f.__setstate__((capture, (), {'a': f}, {})) + try: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f_copy = pickle.loads(pickle.dumps(f, proto)) + try: + self.assertIs(f_copy.keywords['a'], f_copy) + finally: + f_copy.__setstate__((capture, (), {}, {})) + finally: + f.__setstate__((capture, (), {}, {})) # Issue 6083: Reference counting bug def test_setstate_refcount(self): @@ -375,24 +382,60 @@ class TestPartialC(TestPartial, unittest.TestCase): f = self.partial(object) self.assertRaises(TypeError, f.__setstate__, BadSequence()) +@unittest.skipUnless(c_functools, 'requires the C _functools module') +class TestPartialC(TestPartial, unittest.TestCase): + if c_functools: + partial = c_functools.partial + + class AllowPickle: + def __enter__(self): + return self + def __exit__(self, type, value, tb): + return False + + def test_attributes_unwritable(self): + # attributes should not be writable + p = self.partial(capture, 1, 2, a=10, b=20) + self.assertRaises(AttributeError, setattr, p, 'func', map) + self.assertRaises(AttributeError, setattr, p, 'args', (1, 2)) + self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2)) + + p = self.partial(hex) + try: + del p.__dict__ + except TypeError: + pass + else: + self.fail('partial object allowed __dict__ to be deleted') class TestPartialPy(TestPartial, unittest.TestCase): - partial = staticmethod(py_functools.partial) + partial = py_functools.partial + class AllowPickle: + def __init__(self): + self._cm = replaced_module("functools", py_functools) + def __enter__(self): + return self._cm.__enter__() + def __exit__(self, type, value, tb): + return self._cm.__exit__(type, value, tb) if c_functools: - class PartialSubclass(c_functools.partial): + class CPartialSubclass(c_functools.partial): pass +class PyPartialSubclass(py_functools.partial): + pass @unittest.skipUnless(c_functools, 'requires the C _functools module') class TestPartialCSubclass(TestPartialC): if c_functools: - partial = PartialSubclass + partial = CPartialSubclass # partial subclasses are not optimized for nested calls test_nested_optimization = None +class TestPartialPySubclass(TestPartialPy): + partial = PyPartialSubclass class TestPartialMethod(unittest.TestCase): @@ -683,9 +726,10 @@ class TestWraps(TestUpdateWrapper): self.assertEqual(wrapper.attr, 'This is a different test') self.assertEqual(wrapper.dict_attr, f.dict_attr) - +@unittest.skipUnless(c_functools, 'requires the C _functools module') class TestReduce(unittest.TestCase): - func = functools.reduce + if c_functools: + func = c_functools.reduce def test_reduce(self): class Squares: