diff -r 8feb6a5ce4c6 Include/abstract.h --- a/Include/abstract.h Wed Jan 25 23:33:27 2017 +0100 +++ b/Include/abstract.h Thu Jan 26 02:43:18 2017 +0100 @@ -180,23 +180,18 @@ PyAPI_FUNC(PyObject *) _PyStack_AsDict( PyObject **values, PyObject *kwnames); -/* Convert (args, nargs, kwargs: dict) into a (stack, nargs, kwnames: tuple). - - Return 0 on success, raise an exception and return -1 on error. - - Write the new stack into *p_stack. If *p_stack is differen than args, it - must be released by PyMem_Free(). +/* Convert FASTCALL (stack, nargs, kwnames) arguments + to (args: tuple, kwargs: dict) format. - The stack uses borrowed references. + Return 0 on success. Raise an exception and return -1 on error. - The type of keyword keys is not checked, these checks should be done - later (ex: _PyArg_ParseStackAndKeywords). */ -PyAPI_FUNC(int) _PyStack_UnpackDict( - PyObject **args, + The function combines _PyStack_AsTuple() and _PyStack_AsDict(). */ +PyAPI_FUNC(int) _PyStack_AsTupleAndDict( + PyObject **stack, Py_ssize_t nargs, - PyObject *kwargs, - PyObject ***p_stack, - PyObject **p_kwnames); + PyObject *kwnames, + PyObject **p_args, + PyObject **p_kwargs); /* Suggested size (number of positional arguments) for arrays of PyObject* allocated on a C stack to avoid allocating memory on the heap memory. Such @@ -257,12 +252,34 @@ PyAPI_FUNC(PyObject *) _PyObject_Call_Pr PyObject *args, PyObject *kwargs); +PyAPI_FUNC(PyObject *) _PyObject_FastCall_Prepend(PyObject *func, + PyObject *obj, + PyObject **args, + Py_ssize_t nargs, + PyObject *kwnames); + +PyAPI_FUNC(int) _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, + PyObject *kwargs, + PyObject ***p_stack, PyObject **p_kwnames); + PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *callable, PyObject *result, const char *where); #endif /* Py_LIMITED_API */ +#ifdef Py_BUILD_CORE +/* Low-level function to call a C function using FASTCALL calling convention. + Don't check the result: the caller must check _Py_CheckFunctionResult(). */ +PyAPI_FUNC(PyObject *) _Py_RawFastCallDict( + PyObject *self, + fastternaryfunc fastcall, + PyObject **args, + Py_ssize_t nargs, + PyObject *kwargs); +#endif + + /* Call a callable Python object 'callable', with arguments given by the tuple 'args'. If no arguments are needed, then 'args' can be *NULL*. diff -r 8feb6a5ce4c6 Include/descrobject.h --- a/Include/descrobject.h Wed Jan 25 23:33:27 2017 +0100 +++ b/Include/descrobject.h Thu Jan 26 02:43:18 2017 +0100 @@ -23,11 +23,16 @@ typedef PyObject *(*wrapperfunc)(PyObjec typedef PyObject *(*wrapperfunc_kwds)(PyObject *self, PyObject *args, void *wrapped, PyObject *kwds); +typedef PyObject *(*wrapperfunc_fast)(PyObject *self, void *wrapped, + PyObject **args, Py_ssize_t nargs, + PyObject *kwnames); + struct wrapperbase { const char *name; int offset; void *function; wrapperfunc wrapper; + wrapperfunc_fast fastwrapper; const char *doc; int flags; PyObject *name_strobj; @@ -69,6 +74,7 @@ typedef struct { PyDescr_COMMON; struct wrapperbase *d_base; void *d_wrapped; /* This can be any function pointer */ + int d_use_fastwrapper; } PyWrapperDescrObject; #endif /* Py_LIMITED_API */ @@ -93,6 +99,11 @@ PyAPI_FUNC(PyObject *) PyDescr_NewGetSet PyAPI_FUNC(PyObject *) PyDescr_NewWrapper(PyTypeObject *, struct wrapperbase *, void *); #define PyDescr_IsData(d) (Py_TYPE(d)->tp_descr_set != NULL) + +PyAPI_FUNC(PyObject *) _PyDescr_NewWrapperEx(PyTypeObject *type, + struct wrapperbase *base, + void *wrapped, + int use_fastwrapper); #endif PyAPI_FUNC(PyObject *) PyDictProxy_New(PyObject *); diff -r 8feb6a5ce4c6 Include/methodobject.h --- a/Include/methodobject.h Wed Jan 25 23:33:27 2017 +0100 +++ b/Include/methodobject.h Thu Jan 26 02:43:18 2017 +0100 @@ -16,8 +16,6 @@ PyAPI_DATA(PyTypeObject) PyCFunction_Typ #define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*PyNoArgsFunction)(PyObject *); @@ -96,12 +94,12 @@ typedef struct { PyObject *m_weakreflist; /* List of weak references */ } PyCFunctionObject; -PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallDict( +PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallKeywords( PyMethodDef *method, PyObject *self, PyObject **args, Py_ssize_t nargs, - PyObject *kwargs); + PyObject *kwnames); #endif PyAPI_FUNC(int) PyCFunction_ClearFreeList(void); diff -r 8feb6a5ce4c6 Include/object.h --- a/Include/object.h Wed Jan 25 23:33:27 2017 +0100 +++ b/Include/object.h Thu Jan 26 02:43:18 2017 +0100 @@ -339,6 +339,8 @@ typedef int (*descrsetfunc) (PyObject *, typedef int (*initproc)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *); typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t); +typedef PyObject* (*fastternaryfunc) (PyObject *self, PyObject **stack, + Py_ssize_t nargs, PyObject *kwnames); #ifdef Py_LIMITED_API typedef struct _typeobject PyTypeObject; /* opaque */ @@ -424,6 +426,8 @@ typedef struct _typeobject { destructor tp_finalize; + fastternaryfunc tp_fastcall; + #ifdef COUNT_ALLOCS /* these must be last and never explicitly initialized */ Py_ssize_t tp_allocs; @@ -631,12 +635,12 @@ given type object has a specified featur /* Objects support garbage collection (see objimp.h) */ #define Py_TPFLAGS_HAVE_GC (1UL << 14) -/* These two bits are preserved for Stackless Python, next after this is 17 */ +/* Bits 15 and 16 were used by Stackless Python, but are now used for tp_fastcall */ #ifdef STACKLESS -#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3UL << 15) -#else +# error "Stackless is no more supported" +#endif +/* Constant kept for backward compatibility */ #define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION 0 -#endif /* Objects support type attribute cache */ #define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) @@ -655,17 +659,21 @@ given type object has a specified featur #define Py_TPFLAGS_BASE_EXC_SUBCLASS (1UL << 30) #define Py_TPFLAGS_TYPE_SUBCLASS (1UL << 31) -#define Py_TPFLAGS_DEFAULT ( \ - Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \ - Py_TPFLAGS_HAVE_VERSION_TAG | \ - 0) - /* NOTE: The following flags reuse lower bits (removed as part of the * Python 3.0 transition). */ -/* Type structure has tp_finalize member (3.4) */ +/* Type structure has tp_finalize member (3.4), it can be NULL */ #define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0) +/* Type structure has tp_fastcall member (3.7), it can be NULL */ +#define Py_TPFLAGS_HAVE_FASTCALL (1UL << 15) + +#define Py_TPFLAGS_DEFAULT ( \ + Py_TPFLAGS_HAVE_VERSION_TAG | \ + Py_TPFLAGS_HAVE_FINALIZE | \ + Py_TPFLAGS_HAVE_FASTCALL | \ + 0) + #ifdef Py_LIMITED_API #define PyType_HasFeature(t,f) ((PyType_GetFlags(t) & (f)) != 0) #else diff -r 8feb6a5ce4c6 Lib/test/test_gdb.py --- a/Lib/test/test_gdb.py Wed Jan 25 23:33:27 2017 +0100 +++ b/Lib/test/test_gdb.py Thu Jan 26 02:43:18 2017 +0100 @@ -845,7 +845,7 @@ id(42) breakpoint='time_gmtime', cmds_after_breakpoint=['py-bt-full'], ) - self.assertIn('#1 nitems; - if (kw != NULL && !_PyArg_NoKeywords("itemgetter", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("itemgetter", kwnames)) { return NULL; - if (!PyArg_UnpackTuple(args, "itemgetter", 1, 1, &obj)) + } + if (!_PyArg_UnpackStack(args, nargs, "itemgetter", 1, 1, &obj)) { return NULL; - if (nitems == 1) + } + if (nitems == 1) { return PyObject_GetItem(obj, ig->item); + } assert(PyTuple_Check(ig->item)); assert(PyTuple_GET_SIZE(ig->item) == nitems); result = PyTuple_New(nitems); - if (result == NULL) + if (result == NULL) { return NULL; + } for (i=0 ; i < nitems ; i++) { PyObject *item, *val; @@ -1078,7 +1083,7 @@ static PyTypeObject itemgetter_type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)itemgetter_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1102,7 +1107,8 @@ static PyTypeObject itemgetter_type = { 0, /* tp_init */ 0, /* tp_alloc */ itemgetter_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)itemgetter_call, }; @@ -1278,24 +1284,30 @@ dotted_getattr(PyObject *obj, PyObject * } static PyObject * -attrgetter_call(attrgetterobject *ag, PyObject *args, PyObject *kw) +attrgetter_call(attrgetterobject *ag, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { PyObject *obj, *result; Py_ssize_t i, nattrs=ag->nattrs; - if (kw != NULL && !_PyArg_NoKeywords("attrgetter", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("attrgetter", kwnames)) { return NULL; - if (!PyArg_UnpackTuple(args, "attrgetter", 1, 1, &obj)) + } + if (!_PyArg_UnpackStack(args, nargs, "attrgetter", 1, 1, &obj)) { return NULL; - if (ag->nattrs == 1) /* ag->attr is always a tuple */ + } + /* ag->attr is always a tuple */ + if (ag->nattrs == 1) { return dotted_getattr(obj, PyTuple_GET_ITEM(ag->attr, 0)); + } assert(PyTuple_Check(ag->attr)); assert(PyTuple_GET_SIZE(ag->attr) == nattrs); result = PyTuple_New(nattrs); - if (result == NULL) + if (result == NULL) { return NULL; + } for (i=0 ; i < nattrs ; i++) { PyObject *attr, *val; @@ -1310,6 +1322,7 @@ attrgetter_call(attrgetterobject *ag, Py return result; } + static PyObject * dotjoinattr(PyObject *attr, PyObject **attrsep) { @@ -1422,7 +1435,7 @@ static PyTypeObject attrgetter_type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)attrgetter_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1446,7 +1459,8 @@ static PyTypeObject attrgetter_type = { 0, /* tp_init */ 0, /* tp_alloc */ attrgetter_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)attrgetter_call, }; @@ -1523,13 +1537,14 @@ methodcaller_traverse(methodcallerobject } static PyObject * -methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) +methodcaller_call(methodcallerobject *mc, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *method, *obj, *result; - if (kw != NULL && !_PyArg_NoKeywords("methodcaller", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("methodcaller", kwnames)) return NULL; - if (!PyArg_UnpackTuple(args, "methodcaller", 1, 1, &obj)) + if (!_PyArg_UnpackStack(args, nargs, "methodcaller", 1, 1, &obj)) return NULL; method = PyObject_GetAttr(obj, mc->name); if (method == NULL) @@ -1686,7 +1701,7 @@ static PyTypeObject methodcaller_type = 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)methodcaller_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1710,7 +1725,8 @@ static PyTypeObject methodcaller_type = 0, /* tp_init */ 0, /* tp_alloc */ methodcaller_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)methodcaller_call, }; diff -r 8feb6a5ce4c6 Objects/abstract.c --- a/Objects/abstract.c Wed Jan 25 23:33:27 2017 +0100 +++ b/Objects/abstract.c Thu Jan 26 02:43:18 2017 +0100 @@ -2249,7 +2249,17 @@ PyObject_Call(PyObject *callable, PyObje if (Py_EnterRecursiveCall(" while calling a Python object")) return NULL; - result = (*call)(callable, args, kwargs); + if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + result = _Py_RawFastCallDict(callable, + Py_TYPE(callable)->tp_fastcall, + &PyTuple_GET_ITEM(args, 0), + PyTuple_GET_SIZE(args), + kwargs); + } + else { + result = (*call)(callable, args, kwargs); + } Py_LeaveRecursiveCall(); @@ -2277,6 +2287,33 @@ PyObject* _Py_NO_INLINE return args; } +int _Py_NO_INLINE +_PyStack_AsTupleAndDict(PyObject **stack, Py_ssize_t nargs, PyObject *kwnames, + PyObject **p_args, PyObject **p_kwargs) +{ + PyObject *args, *kwargs; + + args = _PyStack_AsTuple(stack, nargs); + if (args == NULL) { + return -1; + } + + if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) { + kwargs = _PyStack_AsDict(stack + nargs, kwnames); + if (kwargs == NULL) { + Py_DECREF(args); + return -1; + } + } + else { + kwargs = NULL; + } + + *p_args = args; + *p_kwargs = kwargs; + return 0; +} + PyObject* _PyStack_AsTupleSlice(PyObject **stack, Py_ssize_t nargs, Py_ssize_t start, Py_ssize_t end) @@ -2328,11 +2365,18 @@ PyObject * else if (PyCFunction_Check(callable)) { result = _PyCFunction_FastCallDict(callable, args, nargs, kwargs); } + else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall; + + result = _Py_RawFastCallDict(callable, fastcall, args, nargs, kwargs); + result = _Py_CheckFunctionResult(callable, result, NULL); + } else { PyObject *tuple; /* Slow-path: build a temporary tuple */ - call = callable->ob_type->tp_call; + call = Py_TYPE(callable)->tp_call; if (call == NULL) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", callable->ob_type->tp_name); @@ -2364,17 +2408,17 @@ PyObject * { PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; PyObject **stack; - Py_ssize_t argcount; + Py_ssize_t nargs; PyObject *result; assert(PyTuple_Check(args)); - argcount = PyTuple_GET_SIZE(args); - if (argcount + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + nargs = PyTuple_GET_SIZE(args); + if (nargs + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { stack = small_stack; } else { - stack = PyMem_Malloc((argcount + 1) * sizeof(PyObject *)); + stack = PyMem_Malloc((nargs + 1) * sizeof(PyObject *)); if (stack == NULL) { PyErr_NoMemory(); return NULL; @@ -2384,11 +2428,11 @@ PyObject * /* use borrowed references */ stack[0] = obj; memcpy(&stack[1], - &PyTuple_GET_ITEM(args, 0), - argcount * sizeof(PyObject *)); + &PyTuple_GET_ITEM(args, 0), + nargs * sizeof(PyObject *)); result = _PyObject_FastCallDict(callable, - stack, argcount + 1, + stack, nargs + 1, kwargs); if (stack != small_stack) { PyMem_Free(stack); @@ -2397,6 +2441,45 @@ PyObject * } PyObject * +_PyObject_FastCall_Prepend(PyObject *func, PyObject *obj, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; + PyObject **stack; + PyObject *result; + Py_ssize_t alloc; + + alloc = nargs + 1; + if (kwnames != NULL) { + alloc += PyTuple_GET_SIZE(kwnames); + } + + if (alloc <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + stack = small_stack; + } + else { + stack = PyMem_Malloc(alloc * sizeof(PyObject *)); + if (stack == NULL) { + PyErr_NoMemory(); + return NULL; + } + } + + /* use borrowed references */ + stack[0] = obj; + memcpy(&stack[1], args, (alloc - 1) * sizeof(PyObject *)); + + result = _PyObject_FastCallKeywords(func, + stack, nargs + 1, + kwnames); + if (stack != small_stack) { + PyMem_Free(stack); + } + + return result; +} + +PyObject * _PyStack_AsDict(PyObject **values, PyObject *kwnames) { Py_ssize_t nkwargs; @@ -2436,6 +2519,7 @@ int assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (kwargs == NULL || (nkwargs = PyDict_GET_SIZE(kwargs)) == 0) { + /* Note: args can be NULL */ *p_stack = args; *p_kwnames = NULL; return 0; @@ -2458,8 +2542,11 @@ int return -1; } - /* Copy position arguments (borrowed references) */ - memcpy(stack, args, nargs * sizeof(stack[0])); + /* args is NULL if nargs==0: don't call memcpy(stack, NULL, 0) */ + if (nargs) { + /* Copy position arguments (borrowed references) */ + memcpy(stack, args, nargs * sizeof(stack[0])); + } kwstack = stack + nargs; pos = i = 0; @@ -2480,6 +2567,30 @@ int } PyObject * +_Py_RawFastCallDict(PyObject *self, fastternaryfunc fastcall, + PyObject **args, Py_ssize_t nargs, PyObject *kwargs) +{ + PyObject **stack; + PyObject *kwnames; + PyObject *result; + + assert(!PyErr_Occurred()); + assert(fastcall != NULL); + assert(kwargs == NULL || PyDict_Check(kwargs)); + + if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) { + return NULL; + } + + result = fastcall(self, stack, nargs, kwnames); + if (stack != args) { + PyMem_Free(stack); + } + Py_XDECREF(kwnames); + return result; +} + +PyObject * _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames) { @@ -2496,11 +2607,28 @@ PyObject * _PyArg_ParseStackAndKeywords(). */ if (PyFunction_Check(callable)) { + /* FIXME: should we use Py_EnterRecursiveCall() here? */ return _PyFunction_FastCallKeywords(callable, stack, nargs, kwnames); } if (PyCFunction_Check(callable)) { return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames); } + else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall; + PyObject *result; + + if (Py_EnterRecursiveCall(" while calling a Python object")) { + return NULL; + } + + result = fastcall(callable, stack, nargs, kwnames); + result = _Py_CheckFunctionResult(callable, result, NULL); + + Py_LeaveRecursiveCall(); + + return result; + } else { /* Slow-path: build a temporary tuple for positional arguments and a temporary dictionary for keyword arguments (if any) */ @@ -2518,7 +2646,7 @@ PyObject * return NULL; } - call = callable->ob_type->tp_call; + call = Py_TYPE(callable)->tp_call; if (call == NULL) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", callable->ob_type->tp_name); diff -r 8feb6a5ce4c6 Objects/classobject.c --- a/Objects/classobject.c Wed Jan 25 23:33:27 2017 +0100 +++ b/Objects/classobject.c Thu Jan 26 02:43:18 2017 +0100 @@ -302,7 +302,8 @@ method_traverse(PyMethodObject *im, visi } static PyObject * -method_call(PyObject *method, PyObject *args, PyObject *kwargs) +method_call(PyObject *method, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { PyObject *self, *func; @@ -313,8 +314,7 @@ method_call(PyObject *method, PyObject * } func = PyMethod_GET_FUNCTION(method); - - return _PyObject_Call_Prepend(func, self, args, kwargs); + return _PyObject_FastCall_Prepend(func, self, args, nargs, kwnames); } static PyObject * @@ -346,7 +346,7 @@ PyTypeObject PyMethod_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ (hashfunc)method_hash, /* tp_hash */ - method_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ method_getattro, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ @@ -370,6 +370,8 @@ PyTypeObject PyMethod_Type = { 0, /* tp_init */ 0, /* tp_alloc */ method_new, /* tp_new */ + + .tp_fastcall = (fastternaryfunc)method_call, }; /* Clear out the free list */ diff -r 8feb6a5ce4c6 Objects/descrobject.c --- a/Objects/descrobject.c Wed Jan 25 23:33:27 2017 +0100 +++ b/Objects/descrobject.c Thu Jan 26 02:43:18 2017 +0100 @@ -210,14 +210,12 @@ getset_set(PyGetSetDescrObject *descr, P } static PyObject * -methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject *kwargs) +methoddescr_call(PyMethodDescrObject *descr, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { - Py_ssize_t nargs; PyObject *self, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - nargs = PyTuple_GET_SIZE(args); if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " @@ -226,7 +224,7 @@ methoddescr_call(PyMethodDescrObject *de PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self), (PyObject *)PyDescr_TYPE(descr))) { PyErr_Format(PyExc_TypeError, @@ -239,24 +237,21 @@ methoddescr_call(PyMethodDescrObject *de return NULL; } - result = _PyMethodDef_RawFastCallDict(descr->d_method, self, - &PyTuple_GET_ITEM(args, 1), nargs - 1, - kwargs); + result = _PyMethodDef_RawFastCallKeywords(descr->d_method, self, + args + 1, nargs - 1, + kwnames); result = _Py_CheckFunctionResult((PyObject *)descr, result, NULL); return result; } static PyObject * -classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, - PyObject *kwds) +classmethoddescr_call(PyMethodDescrObject *descr, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) { - Py_ssize_t argc; - PyObject *self, *func, *result, **stack; + PyObject *self, *func, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - argc = PyTuple_GET_SIZE(args); - if (argc < 1) { + if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " "object needs an argument", @@ -264,7 +259,7 @@ classmethoddescr_call(PyMethodDescrObjec PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!PyType_Check(self)) { PyErr_Format(PyExc_TypeError, "descriptor '%V' requires a type " @@ -288,22 +283,19 @@ classmethoddescr_call(PyMethodDescrObjec func = PyCFunction_NewEx(descr->d_method, self, NULL); if (func == NULL) return NULL; - stack = &PyTuple_GET_ITEM(args, 1); - result = _PyObject_FastCallDict(func, stack, argc - 1, kwds); + result = _PyObject_FastCallKeywords(func, args + 1, nargs - 1, kwnames); Py_DECREF(func); return result; } static PyObject * -wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) +wrapperdescr_call(PyMethodDescrObject *descr, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { - Py_ssize_t argc; - PyObject *self, *func, *result, **stack; + PyObject *self, *func, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - argc = PyTuple_GET_SIZE(args); - if (argc < 1) { + if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " "object needs an argument", @@ -311,7 +303,7 @@ wrapperdescr_call(PyWrapperDescrObject * PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self), (PyObject *)PyDescr_TYPE(descr))) { PyErr_Format(PyExc_TypeError, @@ -328,8 +320,7 @@ wrapperdescr_call(PyWrapperDescrObject * if (func == NULL) return NULL; - stack = &PyTuple_GET_ITEM(args, 1); - result = _PyObject_FastCallDict(func, stack, argc - 1, kwds); + result = _PyObject_FastCallKeywords(func, args + 1, nargs - 1, kwnames); Py_DECREF(func); return result; } @@ -487,7 +478,7 @@ PyTypeObject PyMethodDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)methoddescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -506,7 +497,8 @@ PyTypeObject PyMethodDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)method_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)methoddescr_call, }; /* This is for METH_CLASS in C, not for "f = classmethod(f)" in Python! */ @@ -525,7 +517,7 @@ PyTypeObject PyClassMethodDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)classmethoddescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -544,7 +536,8 @@ PyTypeObject PyClassMethodDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)classmethod_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)classmethoddescr_call, }; PyTypeObject PyMemberDescr_Type = { @@ -636,7 +629,7 @@ PyTypeObject PyWrapperDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)wrapperdescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -655,7 +648,8 @@ PyTypeObject PyWrapperDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)wrapperdescr_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)wrapperdescr_call, }; static PyDescrObject * @@ -664,17 +658,19 @@ descr_new(PyTypeObject *descrtype, PyTyp PyDescrObject *descr; descr = (PyDescrObject *)PyType_GenericAlloc(descrtype, 0); - if (descr != NULL) { - Py_XINCREF(type); - descr->d_type = type; - descr->d_name = PyUnicode_InternFromString(name); - if (descr->d_name == NULL) { - Py_DECREF(descr); - descr = NULL; - } - else { - descr->d_qualname = NULL; - } + if (descr == NULL) { + return NULL; + } + + Py_XINCREF(type); + descr->d_type = type; + descr->d_name = PyUnicode_InternFromString(name); + if (descr->d_name == NULL) { + Py_DECREF(descr); + descr = NULL; + } + else { + descr->d_qualname = NULL; } return descr; } @@ -728,19 +724,28 @@ PyDescr_NewGetSet(PyTypeObject *type, Py } PyObject * -PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *base, void *wrapped) +_PyDescr_NewWrapperEx(PyTypeObject *type, struct wrapperbase *base, void *wrapped, int use_fastwrapper) { PyWrapperDescrObject *descr; descr = (PyWrapperDescrObject *)descr_new(&PyWrapperDescr_Type, type, base->name); - if (descr != NULL) { - descr->d_base = base; - descr->d_wrapped = wrapped; + if (descr == NULL) { + return NULL; } + + descr->d_base = base; + descr->d_wrapped = wrapped; + descr->d_use_fastwrapper = use_fastwrapper; return (PyObject *)descr; } +PyObject * +PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *base, void *wrapped) +{ + return _PyDescr_NewWrapperEx(type, base, wrapped, 0); +} + /* --- mappingproxy: read-only proxy for mappings --- */ @@ -1159,23 +1164,48 @@ static PyGetSetDef wrapper_getsets[] = { }; static PyObject * -wrapper_call(wrapperobject *wp, PyObject *args, PyObject *kwds) +wrapper_call(wrapperobject *wp, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames) { - wrapperfunc wrapper = wp->descr->d_base->wrapper; + wrapperfunc wrapper; PyObject *self = wp->self; + PyObject *args, *kwargs, *result; + if (wp->descr->d_use_fastwrapper) { + wrapperfunc_fast wf = wp->descr->d_base->fastwrapper; + + return (*wf)(self, wp->descr->d_wrapped, stack, nargs, kwnames); + } + + wrapper = wp->descr->d_base->wrapper; if (wp->descr->d_base->flags & PyWrapperFlag_KEYWORDS) { wrapperfunc_kwds wk = (wrapperfunc_kwds)wrapper; - return (*wk)(self, args, wp->descr->d_wrapped, kwds); + + if (_PyStack_AsTupleAndDict(stack, nargs, kwnames, &args, &kwargs) < 0) { + return NULL; + } + + result = (*wk)(self, args, wp->descr->d_wrapped, kwargs); + Py_DECREF(args); + Py_XDECREF(kwargs); + + return result; } - if (kwds != NULL && (!PyDict_Check(kwds) || PyDict_GET_SIZE(kwds) != 0)) { + if (kwnames != NULL && PyTuple_GET_SIZE(kwnames) != 0) { PyErr_Format(PyExc_TypeError, "wrapper %s doesn't take keyword arguments", wp->descr->d_base->name); return NULL; } - return (*wrapper)(self, args, wp->descr->d_wrapped); + + args = _PyStack_AsTuple(stack, nargs); + if (args == NULL) { + return NULL; + } + + result = (*wrapper)(self, args, wp->descr->d_wrapped); + Py_DECREF(args); + return result; } static int @@ -1203,7 +1233,7 @@ PyTypeObject _PyMethodWrapper_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ (hashfunc)wrapper_hash, /* tp_hash */ - (ternaryfunc)wrapper_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1219,10 +1249,8 @@ PyTypeObject _PyMethodWrapper_Type = { wrapper_methods, /* tp_methods */ wrapper_members, /* tp_members */ wrapper_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)wrapper_call, }; PyObject * @@ -1355,10 +1383,8 @@ property_dealloc(PyObject *self) static PyObject * property_descr_get(PyObject *self, PyObject *obj, PyObject *type) { - static PyObject * volatile cached_args = NULL; - PyObject *args; - PyObject *ret; propertyobject *gs = (propertyobject *)self; + PyObject *args[1]; if (obj == NULL || obj == Py_None) { Py_INCREF(self); @@ -1368,29 +1394,9 @@ property_descr_get(PyObject *self, PyObj PyErr_SetString(PyExc_AttributeError, "unreadable attribute"); return NULL; } - args = cached_args; - cached_args = NULL; - if (!args) { - args = PyTuple_New(1); - if (!args) - return NULL; - _PyObject_GC_UNTRACK(args); - } - Py_INCREF(obj); - PyTuple_SET_ITEM(args, 0, obj); - ret = PyObject_Call(gs->prop_get, args, NULL); - if (cached_args == NULL && Py_REFCNT(args) == 1) { - assert(Py_SIZE(args) == 1); - assert(PyTuple_GET_ITEM(args, 0) == obj); - cached_args = args; - Py_DECREF(obj); - } - else { - assert(Py_REFCNT(args) >= 1); - _PyObject_GC_TRACK(args); - Py_DECREF(args); - } - return ret; + + args[0] = obj; + return _PyObject_FastCall(gs->prop_get, args, 1); } static int diff -r 8feb6a5ce4c6 Objects/funcobject.c --- a/Objects/funcobject.c Wed Jan 25 23:33:27 2017 +0100 +++ b/Objects/funcobject.c Thu Jan 26 02:43:18 2017 +0100 @@ -621,6 +621,8 @@ PyTypeObject PyFunction_Type = { 0, /* tp_init */ 0, /* tp_alloc */ func_new, /* tp_new */ + + .tp_fastcall = _PyFunction_FastCallKeywords, }; diff -r 8feb6a5ce4c6 Objects/methodobject.c --- a/Objects/methodobject.c Wed Jan 25 23:33:27 2017 +0100 +++ b/Objects/methodobject.c Thu Jan 26 02:43:18 2017 +0100 @@ -87,26 +87,30 @@ PyCFunction_Call(PyObject *func, PyObjec } PyObject * -_PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwargs) +_PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs, + PyObject *kwargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; PyCFunction meth; - PyObject *result; + PyObject *self, *result; int flags; PyObject *argstuple; - /* _PyMethodDef_RawFastCallDict() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the caller loses its exception */ assert(!PyErr_Occurred()); - assert(method != NULL); + assert(PyCFunction_Check(func_obj)); + assert(func_obj != NULL); assert(nargs >= 0); assert(nargs == 0 || args != NULL); assert(kwargs == NULL || PyDict_Check(kwargs)); - meth = method->ml_meth; - flags = method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + func = (PyCFunctionObject*)func_obj; + meth = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST); switch (flags) { @@ -114,7 +118,7 @@ PyObject * if (nargs != 0) { PyErr_Format(PyExc_TypeError, "%.200s() takes no arguments (%zd given)", - method->ml_name, nargs); + func->m_ml->ml_name, nargs); return NULL; } @@ -129,7 +133,7 @@ PyObject * if (nargs != 1) { PyErr_Format(PyExc_TypeError, "%.200s() takes exactly one argument (%zd given)", - method->ml_name, nargs); + func->m_ml->ml_name, nargs); return NULL; } @@ -165,19 +169,8 @@ PyObject * case METH_FASTCALL: { - PyObject **stack; - PyObject *kwnames; - _PyCFunctionFast fastmeth = (_PyCFunctionFast)meth; - - if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) { - return NULL; - } - - result = (*fastmeth) (self, stack, nargs, kwnames); - if (stack != args) { - PyMem_Free(stack); - } - Py_XDECREF(kwnames); + result = _Py_RawFastCallDict(self, (fastternaryfunc)meth, + args, nargs, kwargs); break; } @@ -188,44 +181,27 @@ PyObject * return NULL; } + result = _Py_CheckFunctionResult(func_obj, result, NULL); + return result; no_keyword_error: PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - method->ml_name, nargs); - + func->m_ml->ml_name, nargs); return NULL; } PyObject * -_PyCFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, - PyObject *kwargs) +_PyMethodDef_RawFastCallKeywords(PyMethodDef *method, PyObject *self, + PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { + PyCFunction meth; PyObject *result; - - assert(func != NULL); - assert(PyCFunction_Check(func)); - - result = _PyMethodDef_RawFastCallDict(((PyCFunctionObject*)func)->m_ml, - PyCFunction_GET_SELF(func), - args, nargs, kwargs); - result = _Py_CheckFunctionResult(func, result, NULL); - return result; -} - -PyObject * -_PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames) -{ - PyCFunctionObject *func; - PyCFunction meth; - PyObject *self, *result; Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); int flags; - assert(func_obj != NULL); - assert(PyCFunction_Check(func_obj)); assert(nargs >= 0); assert(kwnames == NULL || PyTuple_CheckExact(kwnames)); assert((nargs == 0 && nkwargs == 0) || args != NULL); @@ -237,10 +213,8 @@ PyObject * loses its exception */ assert(!PyErr_Occurred()); - func = (PyCFunctionObject*)func_obj; - meth = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + meth = method->ml_meth; + flags = method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); switch (flags) { @@ -248,7 +222,7 @@ PyObject * if (nargs != 0) { PyErr_Format(PyExc_TypeError, "%.200s() takes no arguments (%zd given)", - func->m_ml->ml_name, nargs); + method->ml_name, nargs); return NULL; } @@ -263,7 +237,7 @@ PyObject * if (nargs != 1) { PyErr_Format(PyExc_TypeError, "%.200s() takes exactly one argument (%zd given)", - func->m_ml->ml_name, nargs); + method->ml_name, nargs); return NULL; } @@ -276,7 +250,7 @@ PyObject * case METH_FASTCALL: /* Fast-path: avoid temporary dict to pass keyword arguments */ - result = ((_PyCFunctionFast)meth) (self, args, nargs, kwnames); + result = ((fastternaryfunc)meth) (self, args, nargs, kwnames); break; case METH_VARARGS: @@ -326,16 +300,31 @@ PyObject * return NULL; } - result = _Py_CheckFunctionResult(func_obj, result, NULL); return result; no_keyword_error: PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - func->m_ml->ml_name); + method->ml_name); return NULL; } +PyObject * +_PyCFunction_FastCallKeywords(PyObject *func, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *result; + + assert(func != NULL); + assert(PyCFunction_Check(func)); + + result = _PyMethodDef_RawFastCallKeywords(((PyCFunctionObject*)func)->m_ml, + PyCFunction_GET_SELF(func), + args, nargs, kwnames); + result = _Py_CheckFunctionResult(func, result, NULL); + return result; +} + /* Methods (the standard built-in methods, that is) */ static void @@ -556,8 +545,8 @@ PyTypeObject PyCFunction_Type = { meth_methods, /* tp_methods */ meth_members, /* tp_members */ meth_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ + + .tp_fastcall = _PyCFunction_FastCallKeywords, }; /* Clear out the free list */ diff -r 8feb6a5ce4c6 Objects/typeobject.c --- a/Objects/typeobject.c Wed Jan 25 23:33:27 2017 +0100 +++ b/Objects/typeobject.c Thu Jan 26 02:43:18 2017 +0100 @@ -4690,6 +4690,35 @@ overrides_hash(PyTypeObject *type) return 0; } +/* tp_call slot calling tp_fastcall */ +static PyObject * +fastcall_wrapper(PyObject *callable, PyObject *args_tuple, PyObject *kwargs) +{ + PyTypeObject *type; + PyObject *result; + + type = Py_TYPE(callable); + while (!PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL) + || type->tp_fastcall == NULL) { + PyTypeObject *base = type->tp_base; + if (base == type || base == NULL) { + PyErr_SetString(PyExc_SystemError, + "unable to find tp_fastcall in a base classes"); + return NULL; + } + type = base; + } + + result = _Py_RawFastCallDict(callable, + type->tp_fastcall, + &PyTuple_GET_ITEM(args_tuple, 0), + PyTuple_GET_SIZE(args_tuple), + kwargs); + + result = _Py_CheckFunctionResult(callable, result, NULL); + return result; +} + static void inherit_slots(PyTypeObject *type, PyTypeObject *base) { @@ -4813,7 +4842,19 @@ inherit_slots(PyTypeObject *type, PyType /* tp_reserved is ignored */ COPYSLOT(tp_repr); /* tp_hash see tp_richcompare */ - COPYSLOT(tp_call); + + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL) + && PyType_HasFeature(base, Py_TPFLAGS_HAVE_FASTCALL) + /* don't inherit tp_fastcall if tp_call is defined */ + && (type->tp_call == NULL || type->tp_call != fastcall_wrapper)) { + COPYSLOT(tp_fastcall); + } + + if (!PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL) + || type->tp_call != fastcall_wrapper) { + COPYSLOT(tp_call); + } + COPYSLOT(tp_str); { /* Copy comparison-related slots only when @@ -4942,6 +4983,15 @@ PyType_Ready(PyTypeObject *type) type->tp_dict = dict; } + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL) + && type->tp_fastcall && !type->tp_call) { + /* tp_fastcall defined, but not tp_call: tp_call will use a wrapper + calling tp_fastcall. We need to define tp_call before calling + add_operators(), otherwise the __call__ descriptor is not + defined. */ + type->tp_call = fastcall_wrapper; + } + /* Add type-specific descriptors to tp_dict */ if (add_operators(type) < 0) goto error; @@ -5498,6 +5548,15 @@ wrap_call(PyObject *self, PyObject *args } static PyObject * +wrap_fastcall(PyObject *self, void *wrapped, + PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + fastternaryfunc func = (fastternaryfunc)wrapped; + + return (*func)(self, args, nargs, kwnames); +} + +static PyObject * wrap_del(PyObject *self, PyObject *args, void *wrapped) { destructor func = (destructor)wrapped; @@ -6160,6 +6219,22 @@ slot_tp_call(PyObject *self, PyObject *a return res; } +static PyObject * +slot_tp_fastcall(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + _Py_IDENTIFIER(__call__); + PyObject *meth = lookup_method(self, &PyId___call__); + PyObject *res; + + if (meth == NULL) + return NULL; + + res = _PyObject_FastCallKeywords(meth, args, nargs, kwnames); + + Py_DECREF(meth); + return res; +} + /* There are two slot dispatch functions for tp_getattro. - slot_tp_getattro() is used when __getattribute__ is overridden @@ -6513,13 +6588,16 @@ typedef struct wrapperbase slotdef; #undef RBINSLOT #define TPSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ - {NAME, offsetof(PyTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \ + {NAME, offsetof(PyTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, NULL, \ PyDoc_STR(DOC)} #define FLSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC, FLAGS) \ - {NAME, offsetof(PyTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \ + {NAME, offsetof(PyTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, NULL, \ + PyDoc_STR(DOC), FLAGS} +#define FLSLOT_FAST(NAME, SLOT, FUNCTION, WRAPPER, FAST_WRAPPER, DOC, FLAGS) \ + {NAME, offsetof(PyTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, FAST_WRAPPER, \ PyDoc_STR(DOC), FLAGS} #define ETSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ - {NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, \ + {NAME, offsetof(PyHeapTypeObject, SLOT), (void *)(FUNCTION), WRAPPER, NULL, \ PyDoc_STR(DOC)} #define AMSLOT(NAME, SLOT, FUNCTION, WRAPPER, DOC) \ ETSLOT(NAME, as_async.SLOT, FUNCTION, WRAPPER, DOC) @@ -6557,7 +6635,7 @@ static slotdef slotdefs[] = { "__repr__($self, /)\n--\n\nReturn repr(self)."), TPSLOT("__hash__", tp_hash, slot_tp_hash, wrap_hashfunc, "__hash__($self, /)\n--\n\nReturn hash(self)."), - FLSLOT("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, + FLSLOT_FAST("__call__", tp_call, slot_tp_call, (wrapperfunc)wrap_call, wrap_fastcall, "__call__($self, /, *args, **kwargs)\n--\n\nCall self as a function.", PyWrapperFlag_KEYWORDS), TPSLOT("__str__", tp_str, slot_tp_str, wrap_unaryfunc, @@ -6831,7 +6909,7 @@ update_one_slot(PyTypeObject *type, slot void *generic = NULL, *specific = NULL; int use_generic = 0; int offset = p->offset; - void **ptr = slotptr(type, offset); + void **ptr = slotptr(type, offset), **ptr2 = NULL; if (ptr == NULL) { do { @@ -6839,10 +6917,12 @@ update_one_slot(PyTypeObject *type, slot } while (p->offset == offset); return p; } + do { descr = _PyType_Lookup(type, p->name_strobj); if (descr == NULL) { if (ptr == (void**)&type->tp_iternext) { + ptr2 = NULL; specific = (void *)_PyObject_NextNotImplemented; } continue; @@ -6850,10 +6930,23 @@ update_one_slot(PyTypeObject *type, slot if (Py_TYPE(descr) == &PyWrapperDescr_Type && ((PyWrapperDescrObject *)descr)->d_base->name_strobj == p->name_strobj) { void **tptr = resolve_slotdups(type, p->name_strobj); + int same_wrapper; + if (tptr == NULL || tptr == ptr) generic = p->function; d = (PyWrapperDescrObject *)descr; - if (d->d_base->wrapper == p->wrapper && + + if (d->d_use_fastwrapper) { + same_wrapper = (d->d_base->fastwrapper == p->fastwrapper); + + assert(p->wrapper == (void *)wrap_call); + ptr2 = (void **)&type->tp_fastcall; + } + else { + same_wrapper = (d->d_base->wrapper == p->wrapper); + } + + if (same_wrapper && PyType_IsSubtype(type, PyDescr_TYPE(d))) { if (specific == NULL || @@ -6879,7 +6972,10 @@ update_one_slot(PyTypeObject *type, slot sanity checks and constructing a new argument list. Cut all that nonsense short -- this speeds up instance creation tremendously. */ + specific = (void *)type->tp_new; + ptr2 = NULL; + /* XXX I'm not 100% sure that there isn't a hole in this reasoning that requires additional sanity checks. I'll buy the first person to @@ -6891,16 +6987,36 @@ update_one_slot(PyTypeObject *type, slot to prevent inheritance of the default implementation from object.__hash__ */ specific = (void *)PyObject_HashNotImplemented; + ptr2 = NULL; } else { use_generic = 1; generic = p->function; } } while ((++p)->offset == offset); - if (specific && !use_generic) + + if (specific && !use_generic) { + if (ptr2 != NULL) { + ptr = ptr2; + } *ptr = specific; - else + } + else { *ptr = generic; + } + + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL)) { + /* synchronize tp_call and tp_fastcall slots */ + if (ptr == (void **)&type->tp_call) { + if (type->tp_call == slot_tp_call) { + type->tp_fastcall = slot_tp_fastcall; + } + } + else if (ptr == (void **)&type->tp_fastcall) { + type->tp_call = fastcall_wrapper; + } + } + return p; } @@ -6992,8 +7108,9 @@ fixup_slot_dispatchers(PyTypeObject *typ slotdef *p; init_slotdefs(); - for (p = slotdefs; p->name; ) + for (p = slotdefs; p->name; ) { p = update_one_slot(type, p); + } } static void @@ -7173,7 +7290,24 @@ add_operators(PyTypeObject *type) return -1; } else { - descr = PyDescr_NewWrapper(type, p, *ptr); + int use_fastwrapper; + void *wrapped; + + if (*ptr == fastcall_wrapper) { + /* A wrapper must not call fastcall_wrapper() but the specific + tp_fastcall. fastcall_wrapper() is called with an instance. + From an instance, it's not possible to find the specific + function. */ + use_fastwrapper = 1; + wrapped = type->tp_fastcall; + } + else { + use_fastwrapper = 0; + wrapped = *ptr; + } + assert(wrapped != NULL); + + descr = _PyDescr_NewWrapperEx(type, p, wrapped, use_fastwrapper); if (descr == NULL) return -1; if (PyDict_SetItem(dict, p->name_strobj, descr) < 0) { diff -r 8feb6a5ce4c6 Objects/weakrefobject.c --- a/Objects/weakrefobject.c Wed Jan 25 23:33:27 2017 +0100 +++ b/Objects/weakrefobject.c Thu Jan 26 02:43:18 2017 +0100 @@ -461,7 +461,15 @@ proxy_checkref(PyWeakReference *proxy) WRAP_BINARY(proxy_getattr, PyObject_GetAttr) WRAP_UNARY(proxy_str, PyObject_Str) -WRAP_TERNARY(proxy_call, PyEval_CallObjectWithKeywords) + +static PyObject * +proxy_call(PyObject *proxy, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) +{ + UNWRAP(proxy); + return _PyObject_FastCallKeywords(proxy, args, nargs, kwnames); +} + static PyObject * proxy_repr(PyWeakReference *proxy) @@ -711,7 +719,7 @@ PyTypeObject &proxy_as_sequence, /* tp_as_sequence */ &proxy_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ - proxy_call, /* tp_call */ + 0, /* tp_call */ proxy_str, /* tp_str */ proxy_getattr, /* tp_getattro */ (setattrofunc)proxy_setattr, /* tp_setattro */ @@ -724,6 +732,8 @@ PyTypeObject 0, /* tp_weaklistoffset */ (getiterfunc)proxy_iter, /* tp_iter */ (iternextfunc)proxy_iternext, /* tp_iternext */ + + .tp_fastcall = (fastternaryfunc)proxy_call, }; diff -r 8feb6a5ce4c6 Tools/gdb/libpython.py --- a/Tools/gdb/libpython.py Wed Jan 25 23:33:27 2017 +0100 +++ b/Tools/gdb/libpython.py Thu Jan 26 02:43:18 2017 +0100 @@ -1504,7 +1504,7 @@ class Frame(object): if caller in ('_PyCFunction_FastCallDict', '_PyCFunction_FastCallKeywords'): - if caller == '_PyCFunction_FastCallKeywords': + if caller == '_PyCFunction_FastCallDict': arg_name = 'func_obj' else: arg_name = 'func'