diff -r 2598575a2edc Include/abstract.h --- a/Include/abstract.h Tue Jan 17 16:57:43 2017 +0100 +++ b/Include/abstract.h Wed Jan 18 02:17:07 2017 +0100 @@ -177,25 +177,6 @@ PyAPI_FUNC(PyObject *) _PyStack_AsDict( PyObject **values, PyObject *kwnames); -/* Convert (args, nargs, kwargs: dict) into a (stack, nargs, kwnames: tuple). - - Return 0 on success, raise an exception and return -1 on error. - - Write the new stack into *p_stack. If *p_stack is differen than args, it - must be released by PyMem_Free(). - - The stack uses borrowed references. - - The type of keyword keys is not checked, these checks should be done - later (ex: _PyArg_ParseStackAndKeywords). */ -PyAPI_FUNC(int) _PyStack_UnpackDict( - PyObject **args, - Py_ssize_t nargs, - PyObject *kwargs, - PyObject ***p_stack, - PyObject **p_kwnames, - PyObject *func); - /* Suggested size (number of positional arguments) for arrays of PyObject* allocated on a C stack to avoid allocating memory on the heap memory. Such array is used to pass positional arguments to call functions of the @@ -255,12 +236,28 @@ PyAPI_FUNC(PyObject *) _PyObject_Call_Pr PyObject *args, PyObject *kwargs); +PyAPI_FUNC(PyObject *) _PyObject_FastCall_Prepend(PyObject *func, + PyObject *obj, + PyObject **args, + Py_ssize_t nargs, + PyObject *kwnames); + PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *callable, PyObject *result, const char *where); #endif /* Py_LIMITED_API */ +#ifdef Py_BUILD_CORE +PyAPI_FUNC(PyObject *) _Py_RawFastCallDict( + PyObject *self, + fastternaryfunc fastcall, + PyObject **args, + Py_ssize_t nargs, + PyObject *kwargs); +#endif + + /* Call a callable Python object 'callable', with arguments given by the tuple 'args'. If no arguments are needed, then 'args' can be *NULL*. diff -r 2598575a2edc Include/methodobject.h --- a/Include/methodobject.h Tue Jan 17 16:57:43 2017 +0100 +++ b/Include/methodobject.h Wed Jan 18 02:17:07 2017 +0100 @@ -16,8 +16,6 @@ PyAPI_DATA(PyTypeObject) PyCFunction_Typ #define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*PyNoArgsFunction)(PyObject *); diff -r 2598575a2edc Include/object.h --- a/Include/object.h Tue Jan 17 16:57:43 2017 +0100 +++ b/Include/object.h Wed Jan 18 02:17:07 2017 +0100 @@ -339,6 +339,8 @@ typedef int (*descrsetfunc) (PyObject *, typedef int (*initproc)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *); typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t); +typedef PyObject* (*fastternaryfunc) (PyObject *self, PyObject **stack, + Py_ssize_t nargs, PyObject *kwnames); #ifdef Py_LIMITED_API typedef struct _typeobject PyTypeObject; /* opaque */ @@ -424,6 +426,8 @@ typedef struct _typeobject { destructor tp_finalize; + fastternaryfunc tp_fastcall; + #ifdef COUNT_ALLOCS /* these must be last and never explicitly initialized */ Py_ssize_t tp_allocs; @@ -631,12 +635,12 @@ given type object has a specified featur /* Objects support garbage collection (see objimp.h) */ #define Py_TPFLAGS_HAVE_GC (1UL << 14) -/* These two bits are preserved for Stackless Python, next after this is 17 */ +/* Bits 15 and 16 were used by Stackless Python, but are now used for tp_fastcall */ #ifdef STACKLESS -#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3UL << 15) -#else +# error "Stackless is no more supported" +#endif +/* Constant kept for backward compatibility */ #define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION 0 -#endif /* Objects support type attribute cache */ #define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) @@ -655,17 +659,21 @@ given type object has a specified featur #define Py_TPFLAGS_BASE_EXC_SUBCLASS (1UL << 30) #define Py_TPFLAGS_TYPE_SUBCLASS (1UL << 31) -#define Py_TPFLAGS_DEFAULT ( \ - Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \ - Py_TPFLAGS_HAVE_VERSION_TAG | \ - 0) - /* NOTE: The following flags reuse lower bits (removed as part of the * Python 3.0 transition). */ -/* Type structure has tp_finalize member (3.4) */ +/* Type structure has tp_finalize member (3.4), it can be NULL */ #define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0) +/* Type structure has tp_fastcall member (3.7), it can be NULL */ +#define Py_TPFLAGS_HAVE_FASTCALL (1UL << 15) + +#define Py_TPFLAGS_DEFAULT ( \ + Py_TPFLAGS_HAVE_VERSION_TAG | \ + Py_TPFLAGS_HAVE_FINALIZE | \ + Py_TPFLAGS_HAVE_FASTCALL | \ + 0) + #ifdef Py_LIMITED_API #define PyType_HasFeature(t,f) ((PyType_GetFlags(t) & (f)) != 0) #else diff -r 2598575a2edc Lib/test/test_sys.py --- a/Lib/test/test_sys.py Tue Jan 17 16:57:43 2017 +0100 +++ b/Lib/test/test_sys.py Wed Jan 18 02:17:07 2017 +0100 @@ -1092,7 +1092,7 @@ class SizeofTest(unittest.TestCase): check((1,2,3), vsize('') + 3*self.P) # type # static type: PyTypeObject - fmt = 'P2n15Pl4Pn9Pn11PIP' + fmt = 'P2n15Pl4Pn9Pn11PIPP' if hasattr(sys, 'getcounts'): fmt += '3n2P' s = vsize(fmt) diff -r 2598575a2edc Modules/_operator.c --- a/Modules/_operator.c Tue Jan 17 16:57:43 2017 +0100 +++ b/Modules/_operator.c Wed Jan 18 02:17:07 2017 +0100 @@ -455,24 +455,29 @@ itemgetter_traverse(itemgetterobject *ig } static PyObject * -itemgetter_call(itemgetterobject *ig, PyObject *args, PyObject *kw) +itemgetter_call(itemgetterobject *ig, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { PyObject *obj, *result; Py_ssize_t i, nitems=ig->nitems; - if (kw != NULL && !_PyArg_NoKeywords("itemgetter", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("itemgetter", kwnames)) { return NULL; - if (!PyArg_UnpackTuple(args, "itemgetter", 1, 1, &obj)) + } + if (!_PyArg_UnpackStack(args, nargs, "itemgetter", 1, 1, &obj)) { return NULL; - if (nitems == 1) + } + if (nitems == 1) { return PyObject_GetItem(obj, ig->item); + } assert(PyTuple_Check(ig->item)); assert(PyTuple_GET_SIZE(ig->item) == nitems); result = PyTuple_New(nitems); - if (result == NULL) + if (result == NULL) { return NULL; + } for (i=0 ; i < nitems ; i++) { PyObject *item, *val; @@ -545,7 +550,7 @@ static PyTypeObject itemgetter_type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)itemgetter_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -569,7 +574,8 @@ static PyTypeObject itemgetter_type = { 0, /* tp_init */ 0, /* tp_alloc */ itemgetter_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)itemgetter_call, }; @@ -744,24 +750,30 @@ dotted_getattr(PyObject *obj, PyObject * } static PyObject * -attrgetter_call(attrgetterobject *ag, PyObject *args, PyObject *kw) +attrgetter_call(attrgetterobject *ag, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { PyObject *obj, *result; Py_ssize_t i, nattrs=ag->nattrs; - if (kw != NULL && !_PyArg_NoKeywords("attrgetter", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("attrgetter", kwnames)) { return NULL; - if (!PyArg_UnpackTuple(args, "attrgetter", 1, 1, &obj)) + } + if (!_PyArg_UnpackStack(args, nargs, "attrgetter", 1, 1, &obj)) { return NULL; - if (ag->nattrs == 1) /* ag->attr is always a tuple */ + } + /* ag->attr is always a tuple */ + if (ag->nattrs == 1) { return dotted_getattr(obj, PyTuple_GET_ITEM(ag->attr, 0)); + } assert(PyTuple_Check(ag->attr)); assert(PyTuple_GET_SIZE(ag->attr) == nattrs); result = PyTuple_New(nattrs); - if (result == NULL) + if (result == NULL) { return NULL; + } for (i=0 ; i < nattrs ; i++) { PyObject *attr, *val; @@ -776,6 +788,7 @@ attrgetter_call(attrgetterobject *ag, Py return result; } + static PyObject * dotjoinattr(PyObject *attr, PyObject **attrsep) { @@ -888,7 +901,7 @@ static PyTypeObject attrgetter_type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)attrgetter_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -912,7 +925,8 @@ static PyTypeObject attrgetter_type = { 0, /* tp_init */ 0, /* tp_alloc */ attrgetter_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)attrgetter_call, }; @@ -988,13 +1002,14 @@ methodcaller_traverse(methodcallerobject } static PyObject * -methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) +methodcaller_call(methodcallerobject *mc, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *method, *obj, *result; - if (kw != NULL && !_PyArg_NoKeywords("methodcaller", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("methodcaller", kwnames)) return NULL; - if (!PyArg_UnpackTuple(args, "methodcaller", 1, 1, &obj)) + if (!_PyArg_UnpackStack(args, nargs, "methodcaller", 1, 1, &obj)) return NULL; method = PyObject_GetAttr(obj, mc->name); if (method == NULL) @@ -1151,7 +1166,7 @@ static PyTypeObject methodcaller_type = 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)methodcaller_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1175,7 +1190,8 @@ static PyTypeObject methodcaller_type = 0, /* tp_init */ 0, /* tp_alloc */ methodcaller_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)methodcaller_call, }; diff -r 2598575a2edc Objects/abstract.c --- a/Objects/abstract.c Tue Jan 17 16:57:43 2017 +0100 +++ b/Objects/abstract.c Wed Jan 18 02:17:07 2017 +0100 @@ -2249,7 +2249,17 @@ PyObject_Call(PyObject *callable, PyObje if (Py_EnterRecursiveCall(" while calling a Python object")) return NULL; - result = (*call)(callable, args, kwargs); + if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + result = _Py_RawFastCallDict(callable, + Py_TYPE(callable)->tp_fastcall, + &PyTuple_GET_ITEM(args, 0), + PyTuple_GET_SIZE(args), + kwargs); + } + else { + result = (*call)(callable, args, kwargs); + } Py_LeaveRecursiveCall(); @@ -2328,11 +2338,18 @@ PyObject * else if (PyCFunction_Check(callable)) { result = _PyCFunction_FastCallDict(callable, args, nargs, kwargs); } + else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall; + + result = _Py_RawFastCallDict(callable, fastcall, args, nargs, kwargs); + result = _Py_CheckFunctionResult(callable, result, NULL); + } else { PyObject *tuple; /* Slow-path: build a temporary tuple */ - call = callable->ob_type->tp_call; + call = Py_TYPE(callable)->tp_call; if (call == NULL) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", callable->ob_type->tp_name); @@ -2364,17 +2381,17 @@ PyObject * { PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; PyObject **stack; - Py_ssize_t argcount; + Py_ssize_t nargs; PyObject *result; assert(PyTuple_Check(args)); - argcount = PyTuple_GET_SIZE(args); - if (argcount + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + nargs = PyTuple_GET_SIZE(args); + if (nargs + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { stack = small_stack; } else { - stack = PyMem_Malloc((argcount + 1) * sizeof(PyObject *)); + stack = PyMem_Malloc((nargs + 1) * sizeof(PyObject *)); if (stack == NULL) { PyErr_NoMemory(); return NULL; @@ -2384,11 +2401,11 @@ PyObject * /* use borrowed references */ stack[0] = obj; memcpy(&stack[1], - &PyTuple_GET_ITEM(args, 0), - argcount * sizeof(PyObject *)); + &PyTuple_GET_ITEM(args, 0), + nargs * sizeof(PyObject *)); result = _PyObject_FastCallDict(callable, - stack, argcount + 1, + stack, nargs + 1, kwargs); if (stack != small_stack) { PyMem_Free(stack); @@ -2397,6 +2414,46 @@ PyObject * } PyObject * +_PyObject_FastCall_Prepend(PyObject *func, PyObject *obj, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; + PyObject **stack; + PyObject *result; + Py_ssize_t alloc; + + alloc = nargs + 1; + if (kwnames != NULL) { + alloc += PyTuple_GET_SIZE(kwnames); + } + + if (alloc <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + stack = small_stack; + } + else { + stack = PyMem_Malloc(alloc * sizeof(PyObject *)); + if (stack == NULL) { + PyErr_NoMemory(); + return NULL; + } + } + + /* use borrowed references */ + stack[0] = obj; + memcpy(&stack[1], args, (alloc - 1) * sizeof(PyObject *)); + + result = _PyObject_FastCallKeywords(func, + stack, nargs + 1, + kwnames); + if (stack != small_stack) { + PyMem_Free(stack); + } + + result = _Py_CheckFunctionResult(func, result, NULL); + return result; +} + +PyObject * _PyStack_AsDict(PyObject **values, PyObject *kwnames) { Py_ssize_t nkwargs = PyTuple_GET_SIZE(kwnames); @@ -2421,9 +2478,9 @@ PyObject * return kwdict; } -int +static int _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, - PyObject ***p_stack, PyObject **p_kwnames, PyObject *func) + PyObject ***p_stack, PyObject **p_kwnames) { PyObject **stack, **kwstack; Py_ssize_t nkwargs; @@ -2435,6 +2492,7 @@ int assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (kwargs == NULL || (nkwargs = PyDict_GET_SIZE(kwargs)) == 0) { + /* Note: args can be NULL */ *p_stack = args; *p_kwnames = NULL; return 0; @@ -2457,8 +2515,11 @@ int return -1; } - /* Copy position arguments (borrowed references) */ - memcpy(stack, args, nargs * sizeof(stack[0])); + /* args is NULL if nargs==0: don't call memcpy(stack, NULL, 0) */ + if (nargs) { + /* Copy position arguments (borrowed references) */ + memcpy(stack, args, nargs * sizeof(stack[0])); + } kwstack = stack + nargs; pos = i = 0; @@ -2479,6 +2540,29 @@ int } PyObject * +_Py_RawFastCallDict(PyObject *self, fastternaryfunc fastcall, + PyObject **args, Py_ssize_t nargs, PyObject *kwargs) +{ + PyObject **stack; + PyObject *kwnames; + PyObject *result; + + assert(!PyErr_Occurred()); + + if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) { + return NULL; + } + + result = fastcall(self, stack, nargs, kwnames); + if (stack != args) { + PyMem_Free(stack); + } + Py_XDECREF(kwnames); + + return result; +} + +PyObject * _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames) { @@ -2495,6 +2579,22 @@ PyObject * if (PyCFunction_Check(callable)) { return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames); } + else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall; + PyObject *result; + + if (Py_EnterRecursiveCall(" while calling a Python object")) { + return NULL; + } + + result = fastcall(callable, stack, nargs, kwnames); + result = _Py_CheckFunctionResult(callable, result, NULL); + + Py_LeaveRecursiveCall(); + + return result; + } else { /* Slow-path: build a temporary tuple for positional arguments and a temporary dictionary for keyword arguments (if any) */ @@ -2512,7 +2612,7 @@ PyObject * return NULL; } - call = callable->ob_type->tp_call; + call = Py_TYPE(callable)->tp_call; if (call == NULL) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", callable->ob_type->tp_name); @@ -2539,6 +2639,8 @@ PyObject * Py_DECREF(argtuple); Py_XDECREF(kwdict); + result = _Py_CheckFunctionResult(callable, result, NULL); + exit: Py_LeaveRecursiveCall(); return result; diff -r 2598575a2edc Objects/classobject.c --- a/Objects/classobject.c Tue Jan 17 16:57:43 2017 +0100 +++ b/Objects/classobject.c Wed Jan 18 02:17:07 2017 +0100 @@ -302,7 +302,8 @@ method_traverse(PyMethodObject *im, visi } static PyObject * -method_call(PyObject *method, PyObject *args, PyObject *kwargs) +method_call(PyObject *method, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { PyObject *self, *func; @@ -313,8 +314,7 @@ method_call(PyObject *method, PyObject * } func = PyMethod_GET_FUNCTION(method); - - return _PyObject_Call_Prepend(func, self, args, kwargs); + return _PyObject_FastCall_Prepend(func, self, args, nargs, kwnames); } static PyObject * @@ -346,7 +346,7 @@ PyTypeObject PyMethod_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ (hashfunc)method_hash, /* tp_hash */ - method_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ method_getattro, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ @@ -370,6 +370,8 @@ PyTypeObject PyMethod_Type = { 0, /* tp_init */ 0, /* tp_alloc */ method_new, /* tp_new */ + + .tp_fastcall = (fastternaryfunc)method_call, }; /* Clear out the free list */ diff -r 2598575a2edc Objects/descrobject.c --- a/Objects/descrobject.c Tue Jan 17 16:57:43 2017 +0100 +++ b/Objects/descrobject.c Wed Jan 18 02:17:07 2017 +0100 @@ -210,15 +210,13 @@ getset_set(PyGetSetDescrObject *descr, P } static PyObject * -methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject *kwds) +methoddescr_call(PyMethodDescrObject *descr, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { - Py_ssize_t argc; - PyObject *self, *func, *result, **stack; + PyObject *self, *func, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - argc = PyTuple_GET_SIZE(args); - if (argc < 1) { + if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " "object needs an argument", @@ -226,7 +224,7 @@ methoddescr_call(PyMethodDescrObject *de PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self), (PyObject *)PyDescr_TYPE(descr))) { PyErr_Format(PyExc_TypeError, @@ -242,23 +240,19 @@ methoddescr_call(PyMethodDescrObject *de func = PyCFunction_NewEx(descr->d_method, self, NULL); if (func == NULL) return NULL; - stack = &PyTuple_GET_ITEM(args, 1); - result = _PyObject_FastCallDict(func, stack, argc - 1, kwds); + result = _PyCFunction_FastCallKeywords(func, args + 1, nargs - 1, kwnames); Py_DECREF(func); return result; } static PyObject * -classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, - PyObject *kwds) +classmethoddescr_call(PyMethodDescrObject *descr, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) { - Py_ssize_t argc; - PyObject *self, *func, *result, **stack; + PyObject *self, *func, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - argc = PyTuple_GET_SIZE(args); - if (argc < 1) { + if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " "object needs an argument", @@ -266,7 +260,7 @@ classmethoddescr_call(PyMethodDescrObjec PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!PyType_Check(self)) { PyErr_Format(PyExc_TypeError, "descriptor '%V' requires a type " @@ -290,22 +284,19 @@ classmethoddescr_call(PyMethodDescrObjec func = PyCFunction_NewEx(descr->d_method, self, NULL); if (func == NULL) return NULL; - stack = &PyTuple_GET_ITEM(args, 1); - result = _PyObject_FastCallDict(func, stack, argc - 1, kwds); + result = _PyObject_FastCallKeywords(func, args + 1, nargs - 1, kwnames); Py_DECREF(func); return result; } static PyObject * -wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) +wrapperdescr_call(PyMethodDescrObject *descr, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { - Py_ssize_t argc; - PyObject *self, *func, *result, **stack; + PyObject *self, *func, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - argc = PyTuple_GET_SIZE(args); - if (argc < 1) { + if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " "object needs an argument", @@ -313,7 +304,7 @@ wrapperdescr_call(PyWrapperDescrObject * PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self), (PyObject *)PyDescr_TYPE(descr))) { PyErr_Format(PyExc_TypeError, @@ -330,8 +321,7 @@ wrapperdescr_call(PyWrapperDescrObject * if (func == NULL) return NULL; - stack = &PyTuple_GET_ITEM(args, 1); - result = _PyObject_FastCallDict(func, stack, argc - 1, kwds); + result = _PyObject_FastCallKeywords(func, args + 1, nargs - 1, kwnames); Py_DECREF(func); return result; } @@ -491,7 +481,7 @@ PyTypeObject PyMethodDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)methoddescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -510,7 +500,8 @@ PyTypeObject PyMethodDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)method_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)methoddescr_call, }; /* This is for METH_CLASS in C, not for "f = classmethod(f)" in Python! */ @@ -529,12 +520,12 @@ PyTypeObject PyClassMethodDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)classmethoddescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ 0, /* tp_doc */ descr_traverse, /* tp_traverse */ 0, /* tp_clear */ @@ -548,7 +539,8 @@ PyTypeObject PyClassMethodDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)classmethod_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)classmethoddescr_call, }; PyTypeObject PyMemberDescr_Type = { @@ -640,7 +632,7 @@ PyTypeObject PyWrapperDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)wrapperdescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -659,7 +651,8 @@ PyTypeObject PyWrapperDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)wrapperdescr_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)wrapperdescr_call, }; static PyDescrObject * @@ -1359,10 +1352,8 @@ property_dealloc(PyObject *self) static PyObject * property_descr_get(PyObject *self, PyObject *obj, PyObject *type) { - static PyObject * volatile cached_args = NULL; - PyObject *args; - PyObject *ret; propertyobject *gs = (propertyobject *)self; + PyObject *args[1]; if (obj == NULL || obj == Py_None) { Py_INCREF(self); @@ -1372,29 +1363,9 @@ property_descr_get(PyObject *self, PyObj PyErr_SetString(PyExc_AttributeError, "unreadable attribute"); return NULL; } - args = cached_args; - cached_args = NULL; - if (!args) { - args = PyTuple_New(1); - if (!args) - return NULL; - _PyObject_GC_UNTRACK(args); - } - Py_INCREF(obj); - PyTuple_SET_ITEM(args, 0, obj); - ret = PyObject_Call(gs->prop_get, args, NULL); - if (cached_args == NULL && Py_REFCNT(args) == 1) { - assert(Py_SIZE(args) == 1); - assert(PyTuple_GET_ITEM(args, 0) == obj); - cached_args = args; - Py_DECREF(obj); - } - else { - assert(Py_REFCNT(args) >= 1); - _PyObject_GC_TRACK(args); - Py_DECREF(args); - } - return ret; + + args[0] = obj; + return _PyObject_FastCall(gs->prop_get, args, 1); } static int diff -r 2598575a2edc Objects/funcobject.c --- a/Objects/funcobject.c Tue Jan 17 16:57:43 2017 +0100 +++ b/Objects/funcobject.c Wed Jan 18 02:17:07 2017 +0100 @@ -623,6 +623,8 @@ PyTypeObject PyFunction_Type = { 0, /* tp_init */ 0, /* tp_alloc */ func_new, /* tp_new */ + + .tp_fastcall = _PyFunction_FastCallKeywords, }; diff -r 2598575a2edc Objects/methodobject.c --- a/Objects/methodobject.c Tue Jan 17 16:57:43 2017 +0100 +++ b/Objects/methodobject.c Wed Jan 18 02:17:07 2017 +0100 @@ -78,33 +78,41 @@ PyCFunction_GetFlags(PyObject *op) } PyObject * -PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwds) +PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwargs) { - PyCFunctionObject* f = (PyCFunctionObject*)func; - PyCFunction meth = PyCFunction_GET_FUNCTION(func); - PyObject *self = PyCFunction_GET_SELF(func); + PyCFunctionObject* f; + PyCFunction meth; + PyObject *self; PyObject *arg, *res; Py_ssize_t size; int flags; - assert(kwds == NULL || PyDict_Check(kwds)); + assert(func != NULL); + assert(PyCFunction_Check(func)); + assert(PyTuple_Check(args)); + assert(kwargs == NULL || PyDict_Check(kwargs)); /* PyCFunction_Call() must not be called with an exception set, because it can clear it (directly or indirectly) and so the caller loses its exception */ assert(!PyErr_Occurred()); + f = (PyCFunctionObject*)func; + meth = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST); if (flags == (METH_VARARGS | METH_KEYWORDS)) { - res = (*(PyCFunctionWithKeywords)meth)(self, args, kwds); + res = (*(PyCFunctionWithKeywords)meth)(self, args, kwargs); } else if (flags == METH_FASTCALL) { - PyObject **stack = &PyTuple_GET_ITEM(args, 0); - Py_ssize_t nargs = PyTuple_GET_SIZE(args); - res = _PyCFunction_FastCallDict(func, stack, nargs, kwds); + res = _Py_RawFastCallDict(self, + (fastternaryfunc)meth, + &PyTuple_GET_ITEM(args, 0), + PyTuple_GET_SIZE(args), + kwargs); } else { - if (kwds != NULL && PyDict_GET_SIZE(kwds) != 0) { + if (kwargs != NULL && PyDict_GET_SIZE(kwargs) != 0) { PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", f->m_ml->ml_name); return NULL; @@ -236,20 +244,8 @@ PyObject * case METH_FASTCALL: { - PyObject **stack; - PyObject *kwnames; - _PyCFunctionFast fastmeth = (_PyCFunctionFast)meth; - - if (_PyStack_UnpackDict(args, nargs, kwargs, - &stack, &kwnames, func_obj) < 0) { - return NULL; - } - - result = (*fastmeth) (self, stack, nargs, kwnames); - if (stack != args) { - PyMem_Free(stack); - } - Py_XDECREF(kwnames); + result = _Py_RawFastCallDict(self, (fastternaryfunc)meth, + args, nargs, kwargs); break; } @@ -333,7 +329,7 @@ PyObject * case METH_FASTCALL: /* Fast-path: avoid temporary dict to pass keyword arguments */ - result = ((_PyCFunctionFast)meth) (self, args, nargs, kwnames); + result = ((fastternaryfunc)meth) (self, args, nargs, kwnames); break; case METH_VARARGS: @@ -613,8 +609,8 @@ PyTypeObject PyCFunction_Type = { meth_methods, /* tp_methods */ meth_members, /* tp_members */ meth_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ + + .tp_fastcall = _PyCFunction_FastCallKeywords, }; /* Clear out the free list */ diff -r 2598575a2edc Objects/typeobject.c --- a/Objects/typeobject.c Tue Jan 17 16:57:43 2017 +0100 +++ b/Objects/typeobject.c Wed Jan 18 02:17:07 2017 +0100 @@ -4672,6 +4672,36 @@ overrides_hash(PyTypeObject *type) return 0; } +/* tp_call slot which calls tp_fastcall */ +static PyObject * +fastcall_wrapper(PyObject *callable, PyObject *args_tuple, PyObject *kwargs) +{ + fastternaryfunc fastcall; + PyTypeObject *type; + PyObject *result; + + type = Py_TYPE(callable); + while (!PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL)) { + PyTypeObject *base = type->tp_base; + if (base == type || base == NULL) { + PyErr_SetString(PyExc_SystemError, + "unable to find a fastcall in a base class"); + return NULL; + } + type = base; + } + fastcall = type->tp_fastcall; + + result = _Py_RawFastCallDict(callable, + fastcall, + &PyTuple_GET_ITEM(args_tuple, 0), + PyTuple_GET_SIZE(args_tuple), + kwargs); + + result = _Py_CheckFunctionResult(callable, result, NULL); + return result; +} + static void inherit_slots(PyTypeObject *type, PyTypeObject *base) { @@ -4795,7 +4825,16 @@ inherit_slots(PyTypeObject *type, PyType /* tp_reserved is ignored */ COPYSLOT(tp_repr); /* tp_hash see tp_richcompare */ - COPYSLOT(tp_call); + + if ((type->tp_flags & Py_TPFLAGS_HAVE_FASTCALL) + && (base->tp_flags & Py_TPFLAGS_HAVE_FASTCALL)) { + COPYSLOT(tp_fastcall); + } + + if (type->tp_call != fastcall_wrapper) { + COPYSLOT(tp_call); + } + COPYSLOT(tp_str); { /* Copy comparison-related slots only when @@ -4924,6 +4963,15 @@ PyType_Ready(PyTypeObject *type) type->tp_dict = dict; } + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL) + && type->tp_fastcall && !type->tp_call) { + /* tp_fastcall defined, but not tp_call: tp_call will use a wrapper + calling tp_fastcall. We need to define tp_call before calling + add_operators(), otherwise the __call__ descriptor is not + defined. */ + type->tp_call = fastcall_wrapper; + } + /* Add type-specific descriptors to tp_dict */ if (add_operators(type) < 0) goto error; @@ -6151,6 +6199,22 @@ slot_tp_call(PyObject *self, PyObject *a return res; } +static PyObject * +slot_tp_fastcall(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + _Py_IDENTIFIER(__call__); + PyObject *meth = lookup_method(self, &PyId___call__); + PyObject *res; + + if (meth == NULL) + return NULL; + + res = _PyObject_FastCallKeywords(meth, args, nargs, kwnames); + + Py_DECREF(meth); + return res; +} + /* There are two slot dispatch functions for tp_getattro. - slot_tp_getattro() is used when __getattribute__ is overridden @@ -6985,6 +7049,11 @@ fixup_slot_dispatchers(PyTypeObject *typ init_slotdefs(); for (p = slotdefs; p->name; ) p = update_one_slot(type, p); + + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL) + && type->tp_call == slot_tp_call && type->tp_fastcall == NULL) { + type->tp_fastcall = slot_tp_fastcall; + } } static void diff -r 2598575a2edc Objects/weakrefobject.c --- a/Objects/weakrefobject.c Tue Jan 17 16:57:43 2017 +0100 +++ b/Objects/weakrefobject.c Wed Jan 18 02:17:07 2017 +0100 @@ -461,7 +461,15 @@ proxy_checkref(PyWeakReference *proxy) WRAP_BINARY(proxy_getattr, PyObject_GetAttr) WRAP_UNARY(proxy_str, PyObject_Str) -WRAP_TERNARY(proxy_call, PyEval_CallObjectWithKeywords) + +static PyObject * +proxy_call(PyObject *proxy, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) +{ + UNWRAP(proxy); + return _PyObject_FastCallKeywords(proxy, args, nargs, kwnames); +} + static PyObject * proxy_repr(PyWeakReference *proxy) @@ -711,7 +719,7 @@ PyTypeObject &proxy_as_sequence, /* tp_as_sequence */ &proxy_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ - proxy_call, /* tp_call */ + 0, /* tp_call */ proxy_str, /* tp_str */ proxy_getattr, /* tp_getattro */ (setattrofunc)proxy_setattr, /* tp_setattro */ @@ -724,6 +732,8 @@ PyTypeObject 0, /* tp_weaklistoffset */ (getiterfunc)proxy_iter, /* tp_iter */ (iternextfunc)proxy_iternext, /* tp_iternext */ + + .tp_fastcall = (fastternaryfunc)proxy_call, };