diff -r e06af4027546 Include/abstract.h --- a/Include/abstract.h Mon Feb 06 07:15:57 2017 -0800 +++ b/Include/abstract.h Mon Feb 06 17:57:12 2017 +0100 @@ -209,6 +209,11 @@ PyAPI_FUNC(int) _PyStack_UnpackDict( 40 bytes on the stack. */ #define _PY_FASTCALL_SMALL_STACK 5 +PyAPI_FUNC(PyObject *) _PyObject_FastCall( + PyObject *callable, + PyObject **stack, + Py_ssize_t nargs); + /* Call the callable object 'callable' with the "fast call" calling convention: args is a C array for positional arguments (nargs is the number of positional arguments), kwargs is a dictionary for keyword arguments. @@ -245,11 +250,8 @@ PyAPI_FUNC(PyObject *) _PyObject_FastCal Py_ssize_t nargs, PyObject *kwnames); -#define _PyObject_FastCall(func, args, nargs) \ - _PyObject_FastCallDict((func), (args), (nargs), NULL) - #define _PyObject_CallNoArg(func) \ - _PyObject_FastCallDict((func), NULL, 0, NULL) + _PyObject_FastCall((func), NULL, 0) PyAPI_FUNC(PyObject *) _PyObject_Call_Prepend( PyObject *callable, diff -r e06af4027546 Include/funcobject.h --- a/Include/funcobject.h Mon Feb 06 07:15:57 2017 -0800 +++ b/Include/funcobject.h Mon Feb 06 17:57:12 2017 +0100 @@ -59,6 +59,11 @@ PyAPI_FUNC(PyObject *) PyFunction_GetAnn PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyFunction_FastCall( + PyObject *func, + PyObject **args, + Py_ssize_t nargs); + PyAPI_FUNC(PyObject *) _PyFunction_FastCallDict( PyObject *func, PyObject **args, diff -r e06af4027546 Include/methodobject.h --- a/Include/methodobject.h Mon Feb 06 07:15:57 2017 -0800 +++ b/Include/methodobject.h Mon Feb 06 17:57:12 2017 +0100 @@ -40,6 +40,11 @@ PyAPI_FUNC(int) PyCFunction_GetFlags(PyO PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyCFunction_FastCall( + PyObject *func, + PyObject **args, + Py_ssize_t nargs); + PyAPI_FUNC(PyObject *) _PyCFunction_FastCallDict(PyObject *func, PyObject **args, Py_ssize_t nargs, diff -r e06af4027546 Objects/abstract.c --- a/Objects/abstract.c Mon Feb 06 07:15:57 2017 -0800 +++ b/Objects/abstract.c Mon Feb 06 17:57:12 2017 +0100 @@ -2318,6 +2318,10 @@ PyObject * assert(nargs == 0 || args != NULL); assert(kwargs == NULL || PyDict_Check(kwargs)); + if (kwargs == NULL) { + return _PyObject_FastCall(callable, args, nargs); + } + if (Py_EnterRecursiveCall(" while calling a Python object")) { return NULL; } @@ -2479,6 +2483,119 @@ int return 0; } +/* Disable inlining to reduce the stack consumption */ +static PyObject* _Py_NO_INLINE +object_fastcall(PyObject *callable, PyObject **stack, Py_ssize_t nargs) +{ + /* Slow-path: build a temporary tuple for positional arguments */ + + ternaryfunc call; + PyObject *argtuple; + PyObject *result; + + result = NULL; + assert(nargs == 0 || stack != NULL); + + if (Py_EnterRecursiveCall(" while calling a Python object")) { + return NULL; + } + + call = callable->ob_type->tp_call; + if (call == NULL) { + PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", + callable->ob_type->tp_name); + goto exit; + } + + argtuple = _PyStack_AsTuple(stack, nargs); + if (argtuple == NULL) { + goto exit; + } + + result = (*call)(callable, argtuple, NULL); + Py_DECREF(argtuple); + + result = _Py_CheckFunctionResult(callable, result, NULL); + +exit: + Py_LeaveRecursiveCall(); + return result; +} + +PyObject * +_PyObject_FastCall(PyObject *callable, PyObject **args, Py_ssize_t nargs) +{ + /* _PyObject_FastCall() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + + if (PyFunction_Check(callable)) { + return _PyFunction_FastCall(callable, args, nargs); + } + if (PyCFunction_Check(callable)) { + return _PyCFunction_FastCall(callable, args, nargs); + } + else { + return object_fastcall(callable, args, nargs); + } +} + +/* Disable inlining to reduce the stack consumption */ +static PyObject * _Py_NO_INLINE +object_fastcallkw_slow(PyObject *callable, PyObject **stack, Py_ssize_t nargs, + PyObject *kwnames) +{ + /* Slow-path: build a temporary tuple for positional arguments and a + temporary dictionary for keyword arguments (if any) */ + + ternaryfunc call; + PyObject *argtuple; + PyObject *kwdict, *result; + Py_ssize_t nkwargs; + + result = NULL; + nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); + assert((nargs == 0 && nkwargs == 0) || stack != NULL); + + if (Py_EnterRecursiveCall(" while calling a Python object")) { + return NULL; + } + + call = callable->ob_type->tp_call; + if (call == NULL) { + PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", + callable->ob_type->tp_name); + goto exit; + } + + argtuple = _PyStack_AsTuple(stack, nargs); + if (argtuple == NULL) { + goto exit; + } + + if (nkwargs > 0) { + kwdict = _PyStack_AsDict(stack + nargs, kwnames); + if (kwdict == NULL) { + Py_DECREF(argtuple); + goto exit; + } + } + else { + kwdict = NULL; + } + + result = (*call)(callable, argtuple, kwdict); + Py_DECREF(argtuple); + Py_XDECREF(kwdict); + + result = _Py_CheckFunctionResult(callable, result, NULL); + +exit: + Py_LeaveRecursiveCall(); + return result; +} + PyObject * _PyObject_FastCallKeywords(PyObject *callable, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames) @@ -2491,6 +2608,10 @@ PyObject * assert(nargs >= 0); assert(kwnames == NULL || PyTuple_CheckExact(kwnames)); + if (kwnames == NULL) { + return _PyObject_FastCall(callable, stack, nargs); + } + /* kwnames must only contains str strings, no subclass, and all keys must be unique: these checks are implemented in Python/ceval.c and _PyArg_ParseStackAndKeywords(). */ @@ -2502,54 +2623,7 @@ PyObject * return _PyCFunction_FastCallKeywords(callable, stack, nargs, kwnames); } else { - /* Slow-path: build a temporary tuple for positional arguments and a - temporary dictionary for keyword arguments (if any) */ - - ternaryfunc call; - PyObject *argtuple; - PyObject *kwdict, *result; - Py_ssize_t nkwargs; - - result = NULL; - nkwargs = (kwnames == NULL) ? 0 : PyTuple_GET_SIZE(kwnames); - assert((nargs == 0 && nkwargs == 0) || stack != NULL); - - if (Py_EnterRecursiveCall(" while calling a Python object")) { - return NULL; - } - - call = callable->ob_type->tp_call; - if (call == NULL) { - PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", - callable->ob_type->tp_name); - goto exit; - } - - argtuple = _PyStack_AsTuple(stack, nargs); - if (argtuple == NULL) { - goto exit; - } - - if (nkwargs > 0) { - kwdict = _PyStack_AsDict(stack + nargs, kwnames); - if (kwdict == NULL) { - Py_DECREF(argtuple); - goto exit; - } - } - else { - kwdict = NULL; - } - - result = (*call)(callable, argtuple, kwdict); - Py_DECREF(argtuple); - Py_XDECREF(kwdict); - - result = _Py_CheckFunctionResult(callable, result, NULL); - - exit: - Py_LeaveRecursiveCall(); - return result; + return object_fastcallkw_slow(callable, stack, nargs, kwnames); } } diff -r e06af4027546 Objects/methodobject.c --- a/Objects/methodobject.c Mon Feb 06 07:15:57 2017 -0800 +++ b/Objects/methodobject.c Mon Feb 06 17:57:12 2017 +0100 @@ -87,6 +87,83 @@ PyCFunction_Call(PyObject *func, PyObjec } PyObject * +_PyCFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs) +{ + /* _PyMethodDef_RawFastCallKeywords() must not be called with an exception set, + because it can clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + + assert(func != NULL); + assert(PyCFunction_Check(func)); + + assert(nargs >= 0); + + PyMethodDef *method = ((PyCFunctionObject*)func)->m_ml; + PyCFunction meth = method->ml_meth; + PyObject *self = PyCFunction_GET_SELF(func); + int flags = method->ml_flags & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + PyObject *result; + + switch (flags) + { + case METH_NOARGS: + if (nargs != 0) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%zd given)", + method->ml_name, nargs); + return NULL; + } + + result = (*meth) (self, NULL); + break; + + case METH_O: + if (nargs != 1) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%zd given)", + method->ml_name, nargs); + return NULL; + } + + result = (*meth) (self, args[0]); + break; + + case METH_FASTCALL: + result = ((_PyCFunctionFast)meth) (self, args, nargs, NULL); + break; + + case METH_VARARGS: + case METH_VARARGS | METH_KEYWORDS: + { + /* Slow-path: create a temporary tuple for positional arguments */ + PyObject *argtuple = _PyStack_AsTuple(args, nargs); + if (argtuple == NULL) { + return NULL; + } + + if (flags & METH_KEYWORDS) { + result = (*(PyCFunctionWithKeywords)meth) (self, argtuple, NULL); + } + else { + result = (*meth) (self, argtuple); + } + Py_DECREF(argtuple); + break; + } + + default: + PyErr_SetString(PyExc_SystemError, + "Bad call flags in _PyCFunction_FastCallKeywords. " + "METH_OLDARGS is no longer supported!"); + return NULL; + } + + result = _Py_CheckFunctionResult(func, result, NULL); + return result; +} + +PyObject * _PyMethodDef_RawFastCallDict(PyMethodDef *method, PyObject *self, PyObject **args, Py_ssize_t nargs, PyObject *kwargs) { diff -r e06af4027546 Python/ceval.c --- a/Python/ceval.c Mon Feb 06 07:15:57 2017 -0800 +++ b/Python/ceval.c Mon Feb 06 17:57:12 2017 +0100 @@ -4897,8 +4897,8 @@ call_function(PyObject ***pp_stack, Py_s */ static PyObject* _Py_HOT_FUNCTION -_PyFunction_FastCall(PyCodeObject *co, PyObject **args, Py_ssize_t nargs, - PyObject *globals) +function_fastcall_impl(PyCodeObject *co, PyObject **args, Py_ssize_t nargs, + PyObject *globals) { PyFrameObject *f; PyThreadState *tstate = PyThreadState_GET(); @@ -4937,6 +4937,55 @@ static PyObject* _Py_HOT_FUNCTION return result; } +PyObject* +_PyFunction_FastCall(PyObject *func, PyObject **args, Py_ssize_t nargs) +{ + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *kwdefs, *closure, *name, *qualname; + PyObject **d; + Py_ssize_t nd; + + assert(PyFunction_Check(func)); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + + if (co->co_kwonlyargcount == 0 && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) + { + if (argdefs == NULL && co->co_argcount == nargs) { + return function_fastcall_impl(co, args, nargs, globals); + } + else if (nargs == 0 && argdefs != NULL + && co->co_argcount == Py_SIZE(argdefs)) { + /* function called with no arguments, but all parameters have + a default value: use default values as arguments .*/ + args = &PyTuple_GET_ITEM(argdefs, 0); + return function_fastcall_impl(co, args, Py_SIZE(argdefs), globals); + } + } + + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + closure = PyFunction_GET_CLOSURE(func); + name = ((PyFunctionObject *)func) -> func_name; + qualname = ((PyFunctionObject *)func) -> func_qualname; + + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + return _PyEval_EvalCodeWithName((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + NULL, NULL, 0, 0, + d, (int)nd, kwdefs, + closure, name, qualname); +} + static PyObject * fast_function(PyObject *func, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames) @@ -4960,14 +5009,14 @@ fast_function(PyObject *func, PyObject * co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { if (argdefs == NULL && co->co_argcount == nargs) { - return _PyFunction_FastCall(co, stack, nargs, globals); + return function_fastcall_impl(co, stack, nargs, globals); } else if (nargs == 0 && argdefs != NULL && co->co_argcount == Py_SIZE(argdefs)) { /* function called with no arguments, but all parameters have a default value: use default values as arguments .*/ stack = &PyTuple_GET_ITEM(argdefs, 0); - return _PyFunction_FastCall(co, stack, Py_SIZE(argdefs), globals); + return function_fastcall_impl(co, stack, Py_SIZE(argdefs), globals); } } @@ -5024,14 +5073,14 @@ PyObject * { /* Fast paths */ if (argdefs == NULL && co->co_argcount == nargs) { - return _PyFunction_FastCall(co, args, nargs, globals); + return function_fastcall_impl(co, args, nargs, globals); } else if (nargs == 0 && argdefs != NULL && co->co_argcount == Py_SIZE(argdefs)) { /* function called with no arguments, but all parameters have a default value: use default values as arguments .*/ args = &PyTuple_GET_ITEM(argdefs, 0); - return _PyFunction_FastCall(co, args, Py_SIZE(argdefs), globals); + return function_fastcall_impl(co, args, Py_SIZE(argdefs), globals); } }