diff --git a/Include/abstract.h b/Include/abstract.h index 6647be7..4942bef 100644 --- a/Include/abstract.h +++ b/Include/abstract.h @@ -177,24 +177,6 @@ PyAPI_FUNC(PyObject *) _PyStack_AsDict( PyObject **values, PyObject *kwnames); -/* Convert (args, nargs, kwargs: dict) into a (stack, nargs, kwnames: tuple). - - Return 0 on success, raise an exception and return -1 on error. - - Write the new stack into *p_stack. If *p_stack is differen than args, it - must be released by PyMem_Free(). - - The stack uses borrowed references. - - The type of keyword keys is not checked, these checks should be done - later (ex: _PyArg_ParseStackAndKeywords). */ -PyAPI_FUNC(int) _PyStack_UnpackDict( - PyObject **args, - Py_ssize_t nargs, - PyObject *kwargs, - PyObject ***p_stack, - PyObject **p_kwnames); - /* Suggested size (number of positional arguments) for arrays of PyObject* allocated on a C stack to avoid allocating memory on the heap memory. Such array is used to pass positional arguments to call functions of the @@ -254,12 +236,28 @@ PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend( PyObject *args, PyObject *kwargs); +PyAPI_FUNC(PyObject *) _PyObject_FastCall_Prepend(PyObject *func, + PyObject *obj, + PyObject **args, + Py_ssize_t nargs, + PyObject *kwnames); + PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *callable, PyObject *result, const char *where); #endif /* Py_LIMITED_API */ +#ifdef Py_BUILD_CORE +PyAPI_FUNC(PyObject *) _Py_RawFastCallDict( + PyObject *self, + fastternaryfunc fastcall, + PyObject **args, + Py_ssize_t nargs, + PyObject *kwargs); +#endif + + /* Call a callable Python object 'callable', with arguments given by the tuple 'args'. If no arguments are needed, then 'args' can be *NULL*. diff --git a/Include/methodobject.h b/Include/methodobject.h index 7370419..71b3fe1 100644 --- a/Include/methodobject.h +++ b/Include/methodobject.h @@ -16,8 +16,6 @@ PyAPI_DATA(PyTypeObject) PyCFunction_Type; #define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); -typedef PyObject *(*_PyCFunctionFast) (PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*PyNoArgsFunction)(PyObject *); @@ -96,12 +94,12 @@ typedef struct { PyObject *m_weakreflist; /* List of weak references */ } PyCFunctionObject; -PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallDict( +PyAPI_FUNC(PyObject *) _PyMethodDef_RawFastCallKeywords( PyMethodDef *method, PyObject *self, PyObject **args, Py_ssize_t nargs, - PyObject *kwargs); + PyObject *kwnames); #endif PyAPI_FUNC(int) PyCFunction_ClearFreeList(void); diff --git a/Include/object.h b/Include/object.h index 63e37b8..38c305d 100644 --- a/Include/object.h +++ b/Include/object.h @@ -339,6 +339,8 @@ typedef int (*descrsetfunc) (PyObject *, PyObject *, PyObject *); typedef int (*initproc)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*newfunc)(struct _typeobject *, PyObject *, PyObject *); typedef PyObject *(*allocfunc)(struct _typeobject *, Py_ssize_t); +typedef PyObject* (*fastternaryfunc) (PyObject *self, PyObject **stack, + Py_ssize_t nargs, PyObject *kwnames); #ifdef Py_LIMITED_API typedef struct _typeobject PyTypeObject; /* opaque */ @@ -424,6 +426,8 @@ typedef struct _typeobject { destructor tp_finalize; + fastternaryfunc tp_fastcall; + #ifdef COUNT_ALLOCS /* these must be last and never explicitly initialized */ Py_ssize_t tp_allocs; @@ -631,12 +635,12 @@ given type object has a specified feature. /* Objects support garbage collection (see objimp.h) */ #define Py_TPFLAGS_HAVE_GC (1UL << 14) -/* These two bits are preserved for Stackless Python, next after this is 17 */ +/* Bits 15 and 16 were used by Stackless Python, but are now used for tp_fastcall */ #ifdef STACKLESS -#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION (3UL << 15) -#else -#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION 0 +# error "Stackless is no more supported" #endif +/* Constant kept for backward compatibility */ +#define Py_TPFLAGS_HAVE_STACKLESS_EXTENSION 0 /* Objects support type attribute cache */ #define Py_TPFLAGS_HAVE_VERSION_TAG (1UL << 18) @@ -655,17 +659,21 @@ given type object has a specified feature. #define Py_TPFLAGS_BASE_EXC_SUBCLASS (1UL << 30) #define Py_TPFLAGS_TYPE_SUBCLASS (1UL << 31) -#define Py_TPFLAGS_DEFAULT ( \ - Py_TPFLAGS_HAVE_STACKLESS_EXTENSION | \ - Py_TPFLAGS_HAVE_VERSION_TAG | \ - 0) - /* NOTE: The following flags reuse lower bits (removed as part of the * Python 3.0 transition). */ -/* Type structure has tp_finalize member (3.4) */ +/* Type structure has tp_finalize member (3.4), it can be NULL */ #define Py_TPFLAGS_HAVE_FINALIZE (1UL << 0) +/* Type structure has tp_fastcall member (3.7), it can be NULL */ +#define Py_TPFLAGS_HAVE_FASTCALL (1UL << 15) + +#define Py_TPFLAGS_DEFAULT ( \ + Py_TPFLAGS_HAVE_VERSION_TAG | \ + Py_TPFLAGS_HAVE_FINALIZE | \ + Py_TPFLAGS_HAVE_FASTCALL | \ + 0) + #ifdef Py_LIMITED_API #define PyType_HasFeature(t,f) ((PyType_GetFlags(t) & (f)) != 0) #else diff --git a/Lib/test/test_gdb.py b/Lib/test/test_gdb.py index 557591f..08e0875 100644 --- a/Lib/test/test_gdb.py +++ b/Lib/test/test_gdb.py @@ -845,7 +845,7 @@ id(42) breakpoint='time_gmtime', cmds_after_breakpoint=['py-bt-full'], ) - self.assertIn('#1 nitems; - if (kw != NULL && !_PyArg_NoKeywords("itemgetter", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("itemgetter", kwnames)) { return NULL; - if (!PyArg_UnpackTuple(args, "itemgetter", 1, 1, &obj)) + } + if (!_PyArg_UnpackStack(args, nargs, "itemgetter", 1, 1, &obj)) { return NULL; - if (nitems == 1) + } + if (nitems == 1) { return PyObject_GetItem(obj, ig->item); + } assert(PyTuple_Check(ig->item)); assert(PyTuple_GET_SIZE(ig->item) == nitems); result = PyTuple_New(nitems); - if (result == NULL) + if (result == NULL) { return NULL; + } for (i=0 ; i < nitems ; i++) { PyObject *item, *val; @@ -1078,7 +1083,7 @@ static PyTypeObject itemgetter_type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)itemgetter_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1102,7 +1107,8 @@ static PyTypeObject itemgetter_type = { 0, /* tp_init */ 0, /* tp_alloc */ itemgetter_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)itemgetter_call, }; @@ -1278,24 +1284,30 @@ dotted_getattr(PyObject *obj, PyObject *attr) } static PyObject * -attrgetter_call(attrgetterobject *ag, PyObject *args, PyObject *kw) +attrgetter_call(attrgetterobject *ag, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { PyObject *obj, *result; Py_ssize_t i, nattrs=ag->nattrs; - if (kw != NULL && !_PyArg_NoKeywords("attrgetter", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("attrgetter", kwnames)) { return NULL; - if (!PyArg_UnpackTuple(args, "attrgetter", 1, 1, &obj)) + } + if (!_PyArg_UnpackStack(args, nargs, "attrgetter", 1, 1, &obj)) { return NULL; - if (ag->nattrs == 1) /* ag->attr is always a tuple */ + } + /* ag->attr is always a tuple */ + if (ag->nattrs == 1) { return dotted_getattr(obj, PyTuple_GET_ITEM(ag->attr, 0)); + } assert(PyTuple_Check(ag->attr)); assert(PyTuple_GET_SIZE(ag->attr) == nattrs); result = PyTuple_New(nattrs); - if (result == NULL) + if (result == NULL) { return NULL; + } for (i=0 ; i < nattrs ; i++) { PyObject *attr, *val; @@ -1310,6 +1322,7 @@ attrgetter_call(attrgetterobject *ag, PyObject *args, PyObject *kw) return result; } + static PyObject * dotjoinattr(PyObject *attr, PyObject **attrsep) { @@ -1422,7 +1435,7 @@ static PyTypeObject attrgetter_type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)attrgetter_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1446,7 +1459,8 @@ static PyTypeObject attrgetter_type = { 0, /* tp_init */ 0, /* tp_alloc */ attrgetter_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)attrgetter_call, }; @@ -1523,13 +1537,14 @@ methodcaller_traverse(methodcallerobject *mc, visitproc visit, void *arg) } static PyObject * -methodcaller_call(methodcallerobject *mc, PyObject *args, PyObject *kw) +methodcaller_call(methodcallerobject *mc, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) { PyObject *method, *obj, *result; - if (kw != NULL && !_PyArg_NoKeywords("methodcaller", kw)) + if (kwnames != NULL && !_PyArg_NoStackKeywords("methodcaller", kwnames)) return NULL; - if (!PyArg_UnpackTuple(args, "methodcaller", 1, 1, &obj)) + if (!_PyArg_UnpackStack(args, nargs, "methodcaller", 1, 1, &obj)) return NULL; method = PyObject_GetAttr(obj, mc->name); if (method == NULL) @@ -1686,7 +1701,7 @@ static PyTypeObject methodcaller_type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)methodcaller_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -1710,7 +1725,8 @@ static PyTypeObject methodcaller_type = { 0, /* tp_init */ 0, /* tp_alloc */ methodcaller_new, /* tp_new */ - 0, /* tp_free */ + + .tp_fastcall = (fastternaryfunc)methodcaller_call, }; diff --git a/Objects/abstract.c b/Objects/abstract.c index 5864032..d2bb29c 100644 --- a/Objects/abstract.c +++ b/Objects/abstract.c @@ -2249,7 +2249,17 @@ PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs) if (Py_EnterRecursiveCall(" while calling a Python object")) return NULL; - result = (*call)(callable, args, kwargs); + if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + result = _Py_RawFastCallDict(callable, + Py_TYPE(callable)->tp_fastcall, + &PyTuple_GET_ITEM(args, 0), + PyTuple_GET_SIZE(args), + kwargs); + } + else { + result = (*call)(callable, args, kwargs); + } Py_LeaveRecursiveCall(); @@ -2328,11 +2338,17 @@ _PyObject_FastCallDict(PyObject *callable, PyObject **args, Py_ssize_t nargs, else if (PyCFunction_Check(callable)) { result = _PyCFunction_FastCallDict(callable, args, nargs, kwargs); } + else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall; + + result = _Py_RawFastCallDict(callable, fastcall, args, nargs, kwargs); + } else { PyObject *tuple; /* Slow-path: build a temporary tuple */ - call = callable->ob_type->tp_call; + call = Py_TYPE(callable)->tp_call; if (call == NULL) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", callable->ob_type->tp_name); @@ -2364,17 +2380,17 @@ _PyObject_Call_Prepend(PyObject *callable, { PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; PyObject **stack; - Py_ssize_t argcount; + Py_ssize_t nargs; PyObject *result; assert(PyTuple_Check(args)); - argcount = PyTuple_GET_SIZE(args); - if (argcount + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + nargs = PyTuple_GET_SIZE(args); + if (nargs + 1 <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { stack = small_stack; } else { - stack = PyMem_Malloc((argcount + 1) * sizeof(PyObject *)); + stack = PyMem_Malloc((nargs + 1) * sizeof(PyObject *)); if (stack == NULL) { PyErr_NoMemory(); return NULL; @@ -2384,11 +2400,11 @@ _PyObject_Call_Prepend(PyObject *callable, /* use borrowed references */ stack[0] = obj; memcpy(&stack[1], - &PyTuple_GET_ITEM(args, 0), - argcount * sizeof(PyObject *)); + &PyTuple_GET_ITEM(args, 0), + nargs * sizeof(PyObject *)); result = _PyObject_FastCallDict(callable, - stack, argcount + 1, + stack, nargs + 1, kwargs); if (stack != small_stack) { PyMem_Free(stack); @@ -2397,6 +2413,45 @@ _PyObject_Call_Prepend(PyObject *callable, } PyObject * +_PyObject_FastCall_Prepend(PyObject *func, PyObject *obj, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *small_stack[_PY_FASTCALL_SMALL_STACK]; + PyObject **stack; + PyObject *result; + Py_ssize_t alloc; + + alloc = nargs + 1; + if (kwnames != NULL) { + alloc += PyTuple_GET_SIZE(kwnames); + } + + if (alloc <= (Py_ssize_t)Py_ARRAY_LENGTH(small_stack)) { + stack = small_stack; + } + else { + stack = PyMem_Malloc(alloc * sizeof(PyObject *)); + if (stack == NULL) { + PyErr_NoMemory(); + return NULL; + } + } + + /* use borrowed references */ + stack[0] = obj; + memcpy(&stack[1], args, (alloc - 1) * sizeof(PyObject *)); + + result = _PyObject_FastCallKeywords(func, + stack, nargs + 1, + kwnames); + if (stack != small_stack) { + PyMem_Free(stack); + } + + return result; +} + +PyObject * _PyStack_AsDict(PyObject **values, PyObject *kwnames) { Py_ssize_t nkwargs; @@ -2423,9 +2478,9 @@ _PyStack_AsDict(PyObject **values, PyObject *kwnames) return kwdict; } -int -_PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, - PyObject ***p_stack, PyObject **p_kwnames) +static int +stack_unpack_dict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, + PyObject ***p_stack, PyObject **p_kwnames) { PyObject **stack, **kwstack; Py_ssize_t nkwargs; @@ -2437,6 +2492,7 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, assert(kwargs == NULL || PyDict_CheckExact(kwargs)); if (kwargs == NULL || (nkwargs = PyDict_GET_SIZE(kwargs)) == 0) { + /* Note: args can be NULL */ *p_stack = args; *p_kwnames = NULL; return 0; @@ -2459,8 +2515,11 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, return -1; } - /* Copy position arguments (borrowed references) */ - memcpy(stack, args, nargs * sizeof(stack[0])); + /* args is NULL if nargs==0: don't call memcpy(stack, NULL, 0) */ + if (nargs) { + /* Copy position arguments (borrowed references) */ + memcpy(stack, args, nargs * sizeof(stack[0])); + } kwstack = stack + nargs; pos = i = 0; @@ -2481,6 +2540,30 @@ _PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, } PyObject * +_Py_RawFastCallDict(PyObject *self, fastternaryfunc fastcall, + PyObject **args, Py_ssize_t nargs, PyObject *kwargs) +{ + PyObject **stack; + PyObject *kwnames; + PyObject *result; + + assert(!PyErr_Occurred()); + + if (stack_unpack_dict(args, nargs, kwargs, &stack, &kwnames) < 0) { + return NULL; + } + + result = fastcall(self, stack, nargs, kwnames); + if (stack != args) { + PyMem_Free(stack); + } + Py_XDECREF(kwnames); + + result = _Py_CheckFunctionResult(self, result, NULL); + return result; +} + +PyObject * _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames) { @@ -2502,6 +2585,22 @@ _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t narg if (PyCFunction_Check(callable)) { return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames); } + else if (PyType_HasFeature(Py_TYPE(callable), Py_TPFLAGS_HAVE_FASTCALL) + && Py_TYPE(callable)->tp_fastcall) { + fastternaryfunc fastcall = Py_TYPE(callable)->tp_fastcall; + PyObject *result; + + if (Py_EnterRecursiveCall(" while calling a Python object")) { + return NULL; + } + + result = fastcall(callable, stack, nargs, kwnames); + result = _Py_CheckFunctionResult(callable, result, NULL); + + Py_LeaveRecursiveCall(); + + return result; + } else { /* Slow-path: build a temporary tuple for positional arguments and a temporary dictionary for keyword arguments (if any) */ @@ -2519,7 +2618,7 @@ _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t narg return NULL; } - call = callable->ob_type->tp_call; + call = Py_TYPE(callable)->tp_call; if (call == NULL) { PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", callable->ob_type->tp_name); diff --git a/Objects/classobject.c b/Objects/classobject.c index b0ed023..8172676 100644 --- a/Objects/classobject.c +++ b/Objects/classobject.c @@ -302,7 +302,8 @@ method_traverse(PyMethodObject *im, visitproc visit, void *arg) } static PyObject * -method_call(PyObject *method, PyObject *args, PyObject *kwargs) +method_call(PyObject *method, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { PyObject *self, *func; @@ -313,8 +314,7 @@ method_call(PyObject *method, PyObject *args, PyObject *kwargs) } func = PyMethod_GET_FUNCTION(method); - - return _PyObject_Call_Prepend(func, self, args, kwargs); + return _PyObject_FastCall_Prepend(func, self, args, nargs, kwnames); } static PyObject * @@ -346,7 +346,7 @@ PyTypeObject PyMethod_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ (hashfunc)method_hash, /* tp_hash */ - method_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ method_getattro, /* tp_getattro */ PyObject_GenericSetAttr, /* tp_setattro */ @@ -370,6 +370,8 @@ PyTypeObject PyMethod_Type = { 0, /* tp_init */ 0, /* tp_alloc */ method_new, /* tp_new */ + + .tp_fastcall = (fastternaryfunc)method_call, }; /* Clear out the free list */ diff --git a/Objects/descrobject.c b/Objects/descrobject.c index a254a2a..0062210 100644 --- a/Objects/descrobject.c +++ b/Objects/descrobject.c @@ -210,14 +210,12 @@ getset_set(PyGetSetDescrObject *descr, PyObject *obj, PyObject *value) } static PyObject * -methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject *kwargs) +methoddescr_call(PyMethodDescrObject *descr, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { - Py_ssize_t nargs; PyObject *self, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - nargs = PyTuple_GET_SIZE(args); if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " @@ -226,7 +224,7 @@ methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject *kwargs) PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self), (PyObject *)PyDescr_TYPE(descr))) { PyErr_Format(PyExc_TypeError, @@ -239,24 +237,21 @@ methoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyObject *kwargs) return NULL; } - result = _PyMethodDef_RawFastCallDict(descr->d_method, self, - &PyTuple_GET_ITEM(args, 1), nargs - 1, - kwargs); + result = _PyMethodDef_RawFastCallKeywords(descr->d_method, self, + args + 1, nargs - 1, + kwnames); result = _Py_CheckFunctionResult((PyObject *)descr, result, NULL); return result; } static PyObject * -classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, - PyObject *kwds) +classmethoddescr_call(PyMethodDescrObject *descr, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) { - Py_ssize_t argc; - PyObject *self, *func, *result, **stack; + PyObject *self, *func, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - argc = PyTuple_GET_SIZE(args); - if (argc < 1) { + if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " "object needs an argument", @@ -264,7 +259,7 @@ classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!PyType_Check(self)) { PyErr_Format(PyExc_TypeError, "descriptor '%V' requires a type " @@ -288,22 +283,19 @@ classmethoddescr_call(PyMethodDescrObject *descr, PyObject *args, func = PyCFunction_NewEx(descr->d_method, self, NULL); if (func == NULL) return NULL; - stack = &PyTuple_GET_ITEM(args, 1); - result = _PyObject_FastCallDict(func, stack, argc - 1, kwds); + result = _PyObject_FastCallKeywords(func, args + 1, nargs - 1, kwnames); Py_DECREF(func); return result; } static PyObject * -wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) +wrapperdescr_call(PyMethodDescrObject *descr, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { - Py_ssize_t argc; - PyObject *self, *func, *result, **stack; + PyObject *self, *func, *result; /* Make sure that the first argument is acceptable as 'self' */ - assert(PyTuple_Check(args)); - argc = PyTuple_GET_SIZE(args); - if (argc < 1) { + if (nargs < 1) { PyErr_Format(PyExc_TypeError, "descriptor '%V' of '%.100s' " "object needs an argument", @@ -311,7 +303,7 @@ wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) PyDescr_TYPE(descr)->tp_name); return NULL; } - self = PyTuple_GET_ITEM(args, 0); + self = args[0]; if (!_PyObject_RealIsSubclass((PyObject *)Py_TYPE(self), (PyObject *)PyDescr_TYPE(descr))) { PyErr_Format(PyExc_TypeError, @@ -328,8 +320,7 @@ wrapperdescr_call(PyWrapperDescrObject *descr, PyObject *args, PyObject *kwds) if (func == NULL) return NULL; - stack = &PyTuple_GET_ITEM(args, 1); - result = _PyObject_FastCallDict(func, stack, argc - 1, kwds); + result = _PyObject_FastCallKeywords(func, args + 1, nargs - 1, kwnames); Py_DECREF(func); return result; } @@ -489,7 +480,7 @@ PyTypeObject PyMethodDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)methoddescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -508,7 +499,8 @@ PyTypeObject PyMethodDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)method_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)methoddescr_call, }; /* This is for METH_CLASS in C, not for "f = classmethod(f)" in Python! */ @@ -527,12 +519,13 @@ PyTypeObject PyClassMethodDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)classmethoddescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ - Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC, /* tp_flags */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC + | Py_TPFLAGS_HAVE_FASTCALL, /* tp_flags */ 0, /* tp_doc */ descr_traverse, /* tp_traverse */ 0, /* tp_clear */ @@ -546,7 +539,8 @@ PyTypeObject PyClassMethodDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)classmethod_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)classmethoddescr_call, }; PyTypeObject PyMemberDescr_Type = { @@ -638,7 +632,7 @@ PyTypeObject PyWrapperDescr_Type = { 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ - (ternaryfunc)wrapperdescr_call, /* tp_call */ + 0, /* tp_call */ 0, /* tp_str */ PyObject_GenericGetAttr, /* tp_getattro */ 0, /* tp_setattro */ @@ -657,7 +651,8 @@ PyTypeObject PyWrapperDescr_Type = { 0, /* tp_base */ 0, /* tp_dict */ (descrgetfunc)wrapperdescr_get, /* tp_descr_get */ - 0, /* tp_descr_set */ + + .tp_fastcall = (fastternaryfunc)wrapperdescr_call, }; static PyDescrObject * @@ -1357,10 +1352,8 @@ property_dealloc(PyObject *self) static PyObject * property_descr_get(PyObject *self, PyObject *obj, PyObject *type) { - static PyObject * volatile cached_args = NULL; - PyObject *args; - PyObject *ret; propertyobject *gs = (propertyobject *)self; + PyObject *args[1]; if (obj == NULL || obj == Py_None) { Py_INCREF(self); @@ -1370,29 +1363,9 @@ property_descr_get(PyObject *self, PyObject *obj, PyObject *type) PyErr_SetString(PyExc_AttributeError, "unreadable attribute"); return NULL; } - args = cached_args; - cached_args = NULL; - if (!args) { - args = PyTuple_New(1); - if (!args) - return NULL; - _PyObject_GC_UNTRACK(args); - } - Py_INCREF(obj); - PyTuple_SET_ITEM(args, 0, obj); - ret = PyObject_Call(gs->prop_get, args, NULL); - if (cached_args == NULL && Py_REFCNT(args) == 1) { - assert(Py_SIZE(args) == 1); - assert(PyTuple_GET_ITEM(args, 0) == obj); - cached_args = args; - Py_DECREF(obj); - } - else { - assert(Py_REFCNT(args) >= 1); - _PyObject_GC_TRACK(args); - Py_DECREF(args); - } - return ret; + + args[0] = obj; + return _PyObject_FastCall(gs->prop_get, args, 1); } static int diff --git a/Objects/funcobject.c b/Objects/funcobject.c index a3af4b3..3eec9b3 100644 --- a/Objects/funcobject.c +++ b/Objects/funcobject.c @@ -623,6 +623,8 @@ PyTypeObject PyFunction_Type = { 0, /* tp_init */ 0, /* tp_alloc */ func_new, /* tp_new */ + + .tp_fastcall = _PyFunction_FastCallKeywords, }; diff --git a/Objects/methodobject.c b/Objects/methodobject.c index 35a827e..1825b34 100644 --- a/Objects/methodobject.c +++ b/Objects/methodobject.c @@ -87,26 +87,30 @@ PyCFunction_Call(PyObject *func, PyObject *args, PyObject *kwargs) } PyObject * -_PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, PyObject **args, - Py_ssize_t nargs, PyObject *kwargs) +_PyCFunction_FastCallDict(PyObject *func_obj, PyObject **args, Py_ssize_t nargs, + PyObject *kwargs) { + PyCFunctionObject *func = (PyCFunctionObject*)func_obj; PyCFunction meth; - PyObject *result; + PyObject *self, *result; int flags; PyObject *argstuple; - /* _PyMethodDef_RawFastCallDict() must not be called with an exception set, - because it can clear it (directly or indirectly) and so the + /* _PyCFunction_FastCallDict() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the caller loses its exception */ assert(!PyErr_Occurred()); - assert(method != NULL); + assert(PyCFunction_Check(func_obj)); + assert(func_obj != NULL); assert(nargs >= 0); assert(nargs == 0 || args != NULL); assert(kwargs == NULL || PyDict_Check(kwargs)); - meth = method->ml_meth; - flags = method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + func = (PyCFunctionObject*)func_obj; + meth = PyCFunction_GET_FUNCTION(func); + self = PyCFunction_GET_SELF(func); + flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST); switch (flags) { @@ -114,7 +118,7 @@ _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, PyObject **arg if (nargs != 0) { PyErr_Format(PyExc_TypeError, "%.200s() takes no arguments (%zd given)", - method->ml_name, nargs); + func->m_ml->ml_name, nargs); return NULL; } @@ -129,7 +133,7 @@ _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, PyObject **arg if (nargs != 1) { PyErr_Format(PyExc_TypeError, "%.200s() takes exactly one argument (%zd given)", - method->ml_name, nargs); + func->m_ml->ml_name, nargs); return NULL; } @@ -165,19 +169,8 @@ _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, PyObject **arg case METH_FASTCALL: { - PyObject **stack; - PyObject *kwnames; - _PyCFunctionFast fastmeth = (_PyCFunctionFast)meth; - - if (_PyStack_UnpackDict(args, nargs, kwargs, &stack, &kwnames) < 0) { - return NULL; - } - - result = (*fastmeth) (self, stack, nargs, kwnames); - if (stack != args) { - PyMem_Free(stack); - } - Py_XDECREF(kwnames); + result = _Py_RawFastCallDict(self, (fastternaryfunc)meth, + args, nargs, kwargs); break; } @@ -188,44 +181,27 @@ _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, PyObject **arg return NULL; } + result = _Py_CheckFunctionResult(func_obj, result, NULL); + return result; no_keyword_error: PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - method->ml_name, nargs); - + func->m_ml->ml_name, nargs); return NULL; } PyObject * -_PyCFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, - PyObject *kwargs) -{ - PyObject *result; - - assert(func != NULL); - assert(PyCFunction_Check(func)); - - result = _PyMethodDef_RawFastCallDict(((PyCFunctionObject*)func)->m_ml, - PyCFunction_GET_SELF(func), - args, nargs, kwargs); - result = _Py_CheckFunctionResult(func, result, NULL); - return result; -} - -PyObject * -_PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args, - Py_ssize_t nargs, PyObject *kwnames) +_PyMethodDef_RawFastCallKeywords(PyMethodDef *method, PyObject *self, + PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) { - PyCFunctionObject *func; PyCFunction meth; - PyObject *self, *result; + PyObject *result; Py_ssize_t nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); int flags; - assert(func_obj != NULL); - assert(PyCFunction_Check(func_obj)); assert(nargs >= 0); assert(kwnames == NULL || PyTuple_CheckExact(kwnames)); assert((nargs == 0 && nkwargs == 0) || args != NULL); @@ -237,10 +213,8 @@ _PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args, loses its exception */ assert(!PyErr_Occurred()); - func = (PyCFunctionObject*)func_obj; - meth = PyCFunction_GET_FUNCTION(func); - self = PyCFunction_GET_SELF(func); - flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + meth = method->ml_meth; + flags = method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); switch (flags) { @@ -248,7 +222,7 @@ _PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args, if (nargs != 0) { PyErr_Format(PyExc_TypeError, "%.200s() takes no arguments (%zd given)", - func->m_ml->ml_name, nargs); + method->ml_name, nargs); return NULL; } @@ -263,7 +237,7 @@ _PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args, if (nargs != 1) { PyErr_Format(PyExc_TypeError, "%.200s() takes exactly one argument (%zd given)", - func->m_ml->ml_name, nargs); + method->ml_name, nargs); return NULL; } @@ -276,7 +250,7 @@ _PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args, case METH_FASTCALL: /* Fast-path: avoid temporary dict to pass keyword arguments */ - result = ((_PyCFunctionFast)meth) (self, args, nargs, kwnames); + result = ((fastternaryfunc)meth) (self, args, nargs, kwnames); break; case METH_VARARGS: @@ -326,16 +300,31 @@ _PyCFunction_FastCallKeywords(PyObject *func_obj, PyObject **args, return NULL; } - result = _Py_CheckFunctionResult(func_obj, result, NULL); return result; no_keyword_error: PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", - func->m_ml->ml_name); + method->ml_name); return NULL; } +PyObject * +_PyCFunction_FastCallKeywords(PyObject *func, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *result; + + assert(func != NULL); + assert(PyCFunction_Check(func)); + + result = _PyMethodDef_RawFastCallKeywords(((PyCFunctionObject*)func)->m_ml, + PyCFunction_GET_SELF(func), + args, nargs, kwnames); + result = _Py_CheckFunctionResult(func, result, NULL); + return result; +} + /* Methods (the standard built-in methods, that is) */ static void @@ -556,8 +545,8 @@ PyTypeObject PyCFunction_Type = { meth_methods, /* tp_methods */ meth_members, /* tp_members */ meth_getsets, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ + + .tp_fastcall = _PyCFunction_FastCallKeywords, }; /* Clear out the free list */ diff --git a/Objects/typeobject.c b/Objects/typeobject.c index 0fdff31..304eac9 100644 --- a/Objects/typeobject.c +++ b/Objects/typeobject.c @@ -4672,6 +4672,38 @@ overrides_hash(PyTypeObject *type) return 0; } +/* tp_call slot which calls tp_fastcall */ +static PyObject * +fastcall_wrapper(PyObject *callable, PyObject *args_tuple, PyObject *kwargs) +{ + fastternaryfunc fastcall; + PyTypeObject *type; + PyObject *result; + + type = Py_TYPE(callable); + while (!PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL)) { + PyTypeObject *base = type->tp_base; + if (base == type || base == NULL) { + PyErr_SetString(PyExc_SystemError, + "unable to find a fastcall in a base class"); + return NULL; + } + type = base; + } + + fastcall = type->tp_fastcall; + assert(fastcall != NULL); + + result = _Py_RawFastCallDict(callable, + fastcall, + &PyTuple_GET_ITEM(args_tuple, 0), + PyTuple_GET_SIZE(args_tuple), + kwargs); + + result = _Py_CheckFunctionResult(callable, result, NULL); + return result; +} + static void inherit_slots(PyTypeObject *type, PyTypeObject *base) { @@ -4795,7 +4827,18 @@ inherit_slots(PyTypeObject *type, PyTypeObject *base) /* tp_reserved is ignored */ COPYSLOT(tp_repr); /* tp_hash see tp_richcompare */ - COPYSLOT(tp_call); + + if ((type->tp_flags & Py_TPFLAGS_HAVE_FASTCALL) + && (base->tp_flags & Py_TPFLAGS_HAVE_FASTCALL) + /* don't inherit tp_fastcall if tp_call is defined */ + && (type->tp_call == NULL || type->tp_call != fastcall_wrapper)) { + COPYSLOT(tp_fastcall); + } + + if (type->tp_call != fastcall_wrapper) { + COPYSLOT(tp_call); + } + COPYSLOT(tp_str); { /* Copy comparison-related slots only when @@ -4924,6 +4967,15 @@ PyType_Ready(PyTypeObject *type) type->tp_dict = dict; } + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL) + && type->tp_fastcall && !type->tp_call) { + /* tp_fastcall defined, but not tp_call: tp_call will use a wrapper + calling tp_fastcall. We need to define tp_call before calling + add_operators(), otherwise the __call__ descriptor is not + defined. */ + type->tp_call = fastcall_wrapper; + } + /* Add type-specific descriptors to tp_dict */ if (add_operators(type) < 0) goto error; @@ -6151,6 +6203,22 @@ slot_tp_call(PyObject *self, PyObject *args, PyObject *kwds) return res; } +static PyObject * +slot_tp_fastcall(PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) +{ + _Py_IDENTIFIER(__call__); + PyObject *meth = lookup_method(self, &PyId___call__); + PyObject *res; + + if (meth == NULL) + return NULL; + + res = _PyObject_FastCallKeywords(meth, args, nargs, kwnames); + + Py_DECREF(meth); + return res; +} + /* There are two slot dispatch functions for tp_getattro. - slot_tp_getattro() is used when __getattribute__ is overridden @@ -6985,6 +7053,11 @@ fixup_slot_dispatchers(PyTypeObject *type) init_slotdefs(); for (p = slotdefs; p->name; ) p = update_one_slot(type, p); + + if (PyType_HasFeature(type, Py_TPFLAGS_HAVE_FASTCALL) + && type->tp_call == slot_tp_call && type->tp_fastcall == NULL) { + type->tp_fastcall = slot_tp_fastcall; + } } static void diff --git a/Objects/weakrefobject.c b/Objects/weakrefobject.c index ab6b235..940d2a3 100644 --- a/Objects/weakrefobject.c +++ b/Objects/weakrefobject.c @@ -461,7 +461,15 @@ proxy_checkref(PyWeakReference *proxy) WRAP_BINARY(proxy_getattr, PyObject_GetAttr) WRAP_UNARY(proxy_str, PyObject_Str) -WRAP_TERNARY(proxy_call, PyEval_CallObjectWithKeywords) + +static PyObject * +proxy_call(PyObject *proxy, PyObject **args, Py_ssize_t nargs, + PyObject *kwnames) +{ + UNWRAP(proxy); + return _PyObject_FastCallKeywords(proxy, args, nargs, kwnames); +} + static PyObject * proxy_repr(PyWeakReference *proxy) @@ -711,7 +719,7 @@ _PyWeakref_CallableProxyType = { &proxy_as_sequence, /* tp_as_sequence */ &proxy_as_mapping, /* tp_as_mapping */ 0, /* tp_hash */ - proxy_call, /* tp_call */ + 0, /* tp_call */ proxy_str, /* tp_str */ proxy_getattr, /* tp_getattro */ (setattrofunc)proxy_setattr, /* tp_setattro */ @@ -724,6 +732,8 @@ _PyWeakref_CallableProxyType = { 0, /* tp_weaklistoffset */ (getiterfunc)proxy_iter, /* tp_iter */ (iternextfunc)proxy_iternext, /* tp_iternext */ + + .tp_fastcall = (fastternaryfunc)proxy_call, };