diff -r 05120447f2c6 Include/abstract.h --- a/Include/abstract.h Thu Aug 11 11:01:45 2016 -0400 +++ b/Include/abstract.h Thu Aug 11 22:37:36 2016 +0300 @@ -267,10 +267,22 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx PyObject *args, PyObject *kw); #ifndef Py_LIMITED_API + /* Call the callable object func with the "fast call" calling convention: + args is a C array for positional parameters (nargs is the number of + positional paramater). + + If nargs is equal to zero, args can be NULL. nargs must be greater + or equal to zero. + + Return the result on success. Raise an exception on return NULL on + error. */ + PyAPI_FUNC(PyObject *) _PyObject_FastCall(PyObject *func, + PyObject * const *args, int nargs); + PyAPI_FUNC(PyObject *) _Py_CheckFunctionResult(PyObject *func, PyObject *result, const char *where); -#endif +#endif /* Py_LIMITED_API */ /* Call a callable Python object, callable_object, with diff -r 05120447f2c6 Include/funcobject.h --- a/Include/funcobject.h Thu Aug 11 11:01:45 2016 -0400 +++ b/Include/funcobject.h Thu Aug 11 22:37:36 2016 +0300 @@ -58,6 +58,12 @@ PyAPI_FUNC(int) PyFunction_SetClosure(Py PyAPI_FUNC(PyObject *) PyFunction_GetAnnotations(PyObject *); PyAPI_FUNC(int) PyFunction_SetAnnotations(PyObject *, PyObject *); +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyFunction_FastCall( + PyObject *func, + PyObject * const *args, int nargs); +#endif + /* Macros for direct access to these values. Type checks are *not* done, so use with care. */ #define PyFunction_GET_CODE(func) \ diff -r 05120447f2c6 Include/methodobject.h --- a/Include/methodobject.h Thu Aug 11 11:01:45 2016 -0400 +++ b/Include/methodobject.h Thu Aug 11 22:37:36 2016 +0300 @@ -37,6 +37,11 @@ PyAPI_FUNC(int) PyCFunction_GetFlags(PyO #endif PyAPI_FUNC(PyObject *) PyCFunction_Call(PyObject *, PyObject *, PyObject *); +#ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject *) _PyCFunction_FastCall(PyObject *func, + PyObject * const *args, int nargs); +#endif + struct PyMethodDef { const char *ml_name; /* The name of the built-in function/method */ PyCFunction ml_meth; /* The C function that implements it */ diff -r 05120447f2c6 Include/tupleobject.h --- a/Include/tupleobject.h Thu Aug 11 11:01:45 2016 -0400 +++ b/Include/tupleobject.h Thu Aug 11 22:37:36 2016 +0300 @@ -46,6 +46,7 @@ PyAPI_FUNC(PyObject *) PyTuple_GetItem(P PyAPI_FUNC(int) PyTuple_SetItem(PyObject *, Py_ssize_t, PyObject *); PyAPI_FUNC(PyObject *) PyTuple_GetSlice(PyObject *, Py_ssize_t, Py_ssize_t); #ifndef Py_LIMITED_API +PyAPI_FUNC(PyObject*) _PyTuple_FromArray(PyObject * const *, Py_ssize_t); PyAPI_FUNC(int) _PyTuple_Resize(PyObject **, Py_ssize_t); #endif PyAPI_FUNC(PyObject *) PyTuple_Pack(Py_ssize_t, ...); diff -r 05120447f2c6 Objects/abstract.c --- a/Objects/abstract.c Thu Aug 11 11:01:45 2016 -0400 +++ b/Objects/abstract.c Thu Aug 11 22:37:36 2016 +0300 @@ -2110,9 +2110,15 @@ PyMapping_Values(PyObject *o) /* XXX PyCallable_Check() is in object.c */ PyObject * -PyObject_CallObject(PyObject *o, PyObject *a) +PyObject_CallObject(PyObject *func, PyObject *args) { - return PyEval_CallObjectWithKeywords(o, a, NULL); + if (args != NULL) { + assert(PyTuple_Check(args)); + return PyObject_Call(func, args, NULL); + } + else { + return PyObject_CallFunctionObjArgs(func, NULL); + } } PyObject* @@ -2193,30 +2199,76 @@ PyObject_Call(PyObject *func, PyObject * return _Py_CheckFunctionResult(func, result, NULL); } +PyObject * +_PyObject_FastCall(PyObject *func, PyObject * const *args, int nargs) +{ + ternaryfunc call; + PyObject *result = NULL; + + /* _PyObject_FastCall() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + + assert(func != NULL); + assert(nargs >= 0); + assert(nargs == 0 || args != NULL); + + if (Py_EnterRecursiveCall(" while calling a Python object")) { + return NULL; + } + + if (PyFunction_Check(func)) { + result = _PyFunction_FastCall(func, args, nargs); + } + else if (PyCFunction_Check(func)) { + result = _PyCFunction_FastCall(func, args, nargs); + } + else { + PyObject *tuple; + + /* Slow-path: build a temporary tuple */ + call = func->ob_type->tp_call; + if (call == NULL) { + PyErr_Format(PyExc_TypeError, "'%.200s' object is not callable", + func->ob_type->tp_name); + goto exit; + } + + tuple = _PyTuple_FromArray(args, nargs); + if (tuple == NULL) { + goto exit; + } + + result = (*call)(func, tuple, NULL); + Py_DECREF(tuple); + } + + result = _Py_CheckFunctionResult(func, result, NULL); + +exit: + Py_LeaveRecursiveCall(); + + return result; +} + static PyObject* call_function_tail(PyObject *callable, PyObject *args) { - PyObject *retval; + PyObject *result; if (args == NULL) return NULL; - if (!PyTuple_Check(args)) { - PyObject *a; - - a = PyTuple_New(1); - if (a == NULL) { - Py_DECREF(args); - return NULL; - } - PyTuple_SET_ITEM(a, 0, args); - args = a; + if (PyTuple_Check(args)) { + result = PyObject_Call(callable, args, NULL); } - retval = PyObject_Call(callable, args, NULL); + else { + result = PyObject_CallFunctionObjArgs(callable, args, NULL); + } Py_DECREF(args); - - return retval; + return result; } PyObject * @@ -2372,32 +2424,52 @@ PyObject * return retval; } +#define STATIC_ARGS 8 + static PyObject * -objargs_mktuple(va_list va) +callfunction(PyObject *callable, va_list va) { + PyObject *args, *arg, *result; + va_list countva; int i, n = 0; - va_list countva; - PyObject *result, *tmp; - - Py_VA_COPY(countva, va); - + PyObject *static_args[STATIC_ARGS]; + + for (n = 0; n < STATIC_ARGS; ++n) { + arg = (PyObject *)va_arg(va, PyObject *); + if (arg == NULL) { + return _PyObject_FastCall(callable, static_args, n); + } + static_args[n] = arg; + } + /* count the args */ + Py_VA_COPY(countva, va); while (((PyObject *)va_arg(countva, PyObject *)) != NULL) ++n; - result = PyTuple_New(n); - if (result != NULL && n > 0) { - for (i = 0; i < n; ++i) { - tmp = (PyObject *)va_arg(va, PyObject *); - PyTuple_SET_ITEM(result, i, tmp); - Py_INCREF(tmp); - } + + args = PyTuple_New(n); + if (args == NULL) { + return NULL; } + for (i = 0; i < STATIC_ARGS; ++i) { + arg = static_args[i]; + Py_INCREF(arg); + PyTuple_SET_ITEM(args, i, arg); + } + for (; i < n; ++i) { + arg = (PyObject *)va_arg(va, PyObject *); + Py_INCREF(arg); + PyTuple_SET_ITEM(args, i, arg); + } + result = PyObject_Call(callable, args, NULL); + Py_DECREF(args); + return result; } PyObject * PyObject_CallMethodObjArgs(PyObject *callable, PyObject *name, ...) { - PyObject *args, *tmp; + PyObject *result; va_list vargs; if (callable == NULL || name == NULL) @@ -2407,26 +2479,19 @@ PyObject_CallMethodObjArgs(PyObject *cal if (callable == NULL) return NULL; - /* count the args */ va_start(vargs, name); - args = objargs_mktuple(vargs); + result = callfunction(callable, vargs); va_end(vargs); - if (args == NULL) { - Py_DECREF(callable); - return NULL; - } - tmp = PyObject_Call(callable, args, NULL); - Py_DECREF(args); Py_DECREF(callable); - return tmp; + return result; } PyObject * _PyObject_CallMethodIdObjArgs(PyObject *callable, struct _Py_Identifier *name, ...) { - PyObject *args, *tmp; + PyObject *result; va_list vargs; if (callable == NULL || name == NULL) @@ -2436,40 +2501,28 @@ PyObject * if (callable == NULL) return NULL; - /* count the args */ va_start(vargs, name); - args = objargs_mktuple(vargs); + result = callfunction(callable, vargs); va_end(vargs); - if (args == NULL) { - Py_DECREF(callable); - return NULL; - } - tmp = PyObject_Call(callable, args, NULL); - Py_DECREF(args); Py_DECREF(callable); - return tmp; + return result; } PyObject * PyObject_CallFunctionObjArgs(PyObject *callable, ...) { - PyObject *args, *tmp; + PyObject *result; va_list vargs; if (callable == NULL) return null_error(); - /* count the args */ va_start(vargs, callable); - args = objargs_mktuple(vargs); + result = callfunction(callable, vargs); va_end(vargs); - if (args == NULL) - return NULL; - tmp = PyObject_Call(callable, args, NULL); - Py_DECREF(args); - - return tmp; + + return result; } diff -r 05120447f2c6 Objects/methodobject.c --- a/Objects/methodobject.c Thu Aug 11 11:01:45 2016 -0400 +++ b/Objects/methodobject.c Thu Aug 11 22:37:36 2016 +0300 @@ -145,6 +145,77 @@ PyCFunction_Call(PyObject *func, PyObjec return _Py_CheckFunctionResult(func, res, NULL); } +PyObject * +_PyCFunction_FastCall(PyObject *func_obj, PyObject * const *args, int nargs) +{ + PyCFunctionObject* func = (PyCFunctionObject*)func_obj; + PyCFunction meth = PyCFunction_GET_FUNCTION(func); + PyObject *self = PyCFunction_GET_SELF(func); + PyObject *result; + int flags; + + /* _PyCFunction_FastCall() must not be called with an exception set, + because it may clear it (directly or indirectly) and so the + caller loses its exception */ + assert(!PyErr_Occurred()); + + flags = PyCFunction_GET_FLAGS(func) & ~(METH_CLASS | METH_STATIC | METH_COEXIST); + + switch (flags) + { + case METH_NOARGS: + if (nargs != 0) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes no arguments (%zd given)", + func->m_ml->ml_name, nargs); + return NULL; + } + + result = (*meth) (self, NULL); + break; + + case METH_O: + if (nargs != 1) { + PyErr_Format(PyExc_TypeError, + "%.200s() takes exactly one argument (%zd given)", + func->m_ml->ml_name, nargs); + return NULL; + } + + result = (*meth) (self, args[0]); + break; + + case METH_VARARGS: + case METH_VARARGS | METH_KEYWORDS: + { + /* Slow-path: create a temporary tuple */ + PyObject *tuple = _PyTuple_FromArray(args, nargs); + if (tuple == NULL) { + return NULL; + } + + if (flags & METH_KEYWORDS) { + result = (*(PyCFunctionWithKeywords)meth) (self, tuple, NULL); + } + else { + result = (*meth) (self, tuple); + } + Py_DECREF(tuple); + break; + } + + default: + PyErr_SetString(PyExc_SystemError, + "Bad call flags in PyCFunction_Call. " + "METH_OLDARGS is no longer supported!"); + return NULL; + } + + result = _Py_CheckFunctionResult(func_obj, result, NULL); + + return result; +} + /* Methods (the standard built-in methods, that is) */ static void diff -r 05120447f2c6 Objects/tupleobject.c --- a/Objects/tupleobject.c Thu Aug 11 11:01:45 2016 -0400 +++ b/Objects/tupleobject.c Thu Aug 11 22:37:36 2016 +0300 @@ -82,8 +82,7 @@ PyTuple_New(Py_ssize_t size) return NULL; } #if PyTuple_MAXSAVESIZE > 0 - if (size == 0 && free_list[0]) { - op = free_list[0]; + if (size == 0 && (op = free_list[size]) != NULL) { Py_INCREF(op); #ifdef COUNT_ALLOCS tuple_zero_allocs++; @@ -831,6 +830,26 @@ PyTypeObject PyTuple_Type = { PyObject_GC_Del, /* tp_free */ }; +PyObject * +_PyTuple_FromArray(PyObject * const *data, Py_ssize_t size) +{ + PyObject *result; + Py_ssize_t i; + + result = PyTuple_New(size); + if (result == NULL) { + return NULL; + } + + for (i = 0; i < size; i++) { + PyObject *item = data[i]; + Py_INCREF(item); + PyTuple_SET_ITEM(result, i, item); + } + + return result; +} + /* The following function breaks the notion that tuples are immutable: it changes the size of a tuple. We get away with this only if there is only one module referencing the object. You can also think of it diff -r 05120447f2c6 Python/ceval.c --- a/Python/ceval.c Thu Aug 11 11:01:45 2016 -0400 +++ b/Python/ceval.c Thu Aug 11 22:37:36 2016 +0300 @@ -113,7 +113,7 @@ static PyObject * call_function(PyObject #else static PyObject * call_function(PyObject ***, int); #endif -static PyObject * fast_function(PyObject *, PyObject ***, int, int, int); +static PyObject * fast_function(PyObject *, PyObject **, int, int, int); static PyObject * do_call(PyObject *, PyObject ***, int, int); static PyObject * ext_do_call(PyObject *, PyObject ***, int, int, int); static PyObject * update_keyword_args(PyObject *, int, PyObject ***, @@ -3779,81 +3779,60 @@ too_many_positional(PyCodeObject *co, in Py_DECREF(kwonly_sig); } + /* This is gonna seem *real weird*, but if you put some other code between PyEval_EvalFrame() and PyEval_EvalCodeEx() you will need to adjust the test in the if statements in Misc/gdbinit (pystack and pystackv). */ -static PyObject * -_PyEval_EvalCodeWithName(PyObject *_co, PyObject *globals, PyObject *locals, - PyObject **args, int argcount, PyObject **kws, int kwcount, - PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure, - PyObject *name, PyObject *qualname) +static int +fastlocals_dict(PyObject **fastlocals, PyCodeObject *co, + PyObject *kwdict, PyObject *kwargs) { - PyCodeObject* co = (PyCodeObject*)_co; - PyFrameObject *f; - PyObject *retval = NULL; - PyObject **fastlocals, **freevars; - PyThreadState *tstate = PyThreadState_GET(); - PyObject *x, *u; - int total_args = co->co_argcount + co->co_kwonlyargcount; - int i; - int n = argcount; - PyObject *kwdict = NULL; - - if (globals == NULL) { - PyErr_SetString(PyExc_SystemError, - "PyEval_EvalCodeEx: NULL globals"); - return NULL; + const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; + _Py_IDENTIFIER(items); + PyObject *items, *iter; + PyObject *item = NULL; + + assert(PyDict_Check(kwargs)); + + items = _PyObject_CallMethodId((PyObject *)kwargs, &PyId_items, NULL); + if (items == NULL) { + return -1; } - assert(tstate != NULL); - assert(globals != NULL); - f = PyFrame_New(tstate, co, globals, locals); - if (f == NULL) - return NULL; - - fastlocals = f->f_localsplus; - freevars = f->f_localsplus + co->co_nlocals; - - /* Parse arguments. */ - if (co->co_flags & CO_VARKEYWORDS) { - kwdict = PyDict_New(); - if (kwdict == NULL) - goto fail; - i = total_args; - if (co->co_flags & CO_VARARGS) - i++; - SETLOCAL(i, kwdict); + iter = PyObject_GetIter(items); + Py_DECREF(items); + if (iter == NULL) { + goto fail; } - if (argcount > co->co_argcount) - n = co->co_argcount; - for (i = 0; i < n; i++) { - x = args[i]; - Py_INCREF(x); - SETLOCAL(i, x); - } - if (co->co_flags & CO_VARARGS) { - u = PyTuple_New(argcount - n); - if (u == NULL) - goto fail; - SETLOCAL(total_args, u); - for (i = n; i < argcount; i++) { - x = args[i]; - Py_INCREF(x); - PyTuple_SET_ITEM(u, i-n, x); - } - } - for (i = 0; i < kwcount; i++) { + + do { PyObject **co_varnames; - PyObject *keyword = kws[2*i]; - PyObject *value = kws[2*i + 1]; + PyObject *keyword, *value; int j; + + /* Get first item */ + item = PyIter_Next(iter); + if (item == NULL) { + if (PyErr_Occurred()) { + goto fail; + } + + /* nothing more to add */ + break; + } + assert(PyTuple_CheckExact(item) && PyTuple_GET_SIZE(item) == 2); + + keyword = PyTuple_GET_ITEM(item, 0); + value = PyTuple_GET_ITEM(item, 1); + if (keyword == NULL || !PyUnicode_Check(keyword)) { PyErr_Format(PyExc_TypeError, "%U() keywords must be strings", co->co_name); goto fail; } + /* Speed hack: do raw pointer compares. As names are normally interned this should almost always hit. */ co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; @@ -3862,6 +3841,7 @@ static PyObject * if (nm == keyword) goto kw_found; } + /* Slow fallback, just in case */ for (j = 0; j < total_args; j++) { PyObject *nm = co_varnames[j]; @@ -3872,6 +3852,7 @@ static PyObject * else if (cmp < 0) goto fail; } + if (j >= total_args && kwdict == NULL) { PyErr_Format(PyExc_TypeError, "%U() got an unexpected " @@ -3880,10 +3861,13 @@ static PyObject * keyword); goto fail; } + if (PyDict_SetItem(kwdict, keyword, value) == -1) { goto fail; } + Py_DECREF(item); continue; + kw_found: if (GETLOCAL(j) != NULL) { PyErr_Format(PyExc_TypeError, @@ -3895,11 +3879,172 @@ static PyObject * } Py_INCREF(value); SETLOCAL(j, value); + + Py_CLEAR(item); + } while (1); + + Py_DECREF(iter); + return 0; + +fail: + Py_DECREF(iter); + Py_XDECREF(item); + return -1; +} + +static PyObject * +_PyEval_EvalCode(PyObject *_co, PyObject *globals, PyObject *locals, + PyObject * const *args, int argcount, + PyObject **kws, int kwcount, PyObject *kwargs, + PyObject **defs, int defcount, + PyObject *kwdefs, PyObject *closure, + PyObject *name, PyObject *qualname) +{ + PyCodeObject* co = (PyCodeObject*)_co; + PyFrameObject *f; + PyObject *retval = NULL; + PyObject **fastlocals, **freevars; + PyThreadState *tstate; + PyObject *x, *u; + const Py_ssize_t total_args = co->co_argcount + co->co_kwonlyargcount; + Py_ssize_t i, n; + PyObject *kwdict; + + assert((kwcount == 0) || (kws != NULL)); + + if (globals == NULL) { + PyErr_SetString(PyExc_SystemError, + "PyEval_EvalCodeEx: NULL globals"); + return NULL; } + + /* Create the frame */ + tstate = PyThreadState_GET(); + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, locals); + if (f == NULL) { + return NULL; + } + fastlocals = f->f_localsplus; + freevars = f->f_localsplus + co->co_nlocals; + + /* Create a dictionary for keyword parameters (**kwags) */ + if (co->co_flags & CO_VARKEYWORDS) { + kwdict = PyDict_New(); + if (kwdict == NULL) + goto fail; + i = total_args; + if (co->co_flags & CO_VARARGS) { + i++; + } + SETLOCAL(i, kwdict); + } + else { + kwdict = NULL; + } + + /* Copy positional arguments into local variables */ + if (argcount > co->co_argcount) { + n = co->co_argcount; + } + else { + n = argcount; + } + for (i = 0; i < n; i++) { + x = args[i]; + Py_INCREF(x); + SETLOCAL(i, x); + } + + /* Pack other positional arguments into the *args argument */ + if (co->co_flags & CO_VARARGS) { + u = PyTuple_New(argcount - n); + if (u == NULL) { + goto fail; + } + SETLOCAL(total_args, u); + for (i = n; i < argcount; i++) { + x = args[i]; + Py_INCREF(x); + PyTuple_SET_ITEM(u, i-n, x); + } + } + + /* Handle keyword arguments (passed as an array of (key, value)) */ + for (i = 0; i < kwcount; i++) { + PyObject **co_varnames; + PyObject *keyword = kws[2*i]; + PyObject *value = kws[2*i + 1]; + int j; + + if (keyword == NULL || !PyUnicode_Check(keyword)) { + PyErr_Format(PyExc_TypeError, + "%U() keywords must be strings", + co->co_name); + goto fail; + } + + /* Speed hack: do raw pointer compares. As names are + normally interned this should almost always hit. */ + co_varnames = ((PyTupleObject *)(co->co_varnames))->ob_item; + for (j = 0; j < total_args; j++) { + PyObject *nm = co_varnames[j]; + if (nm == keyword) + goto kw_found; + } + + /* Slow fallback, just in case */ + for (j = 0; j < total_args; j++) { + PyObject *nm = co_varnames[j]; + int cmp = PyObject_RichCompareBool( + keyword, nm, Py_EQ); + if (cmp > 0) + goto kw_found; + else if (cmp < 0) + goto fail; + } + + if (j >= total_args && kwdict == NULL) { + PyErr_Format(PyExc_TypeError, + "%U() got an unexpected " + "keyword argument '%S'", + co->co_name, + keyword); + goto fail; + } + + if (PyDict_SetItem(kwdict, keyword, value) == -1) { + goto fail; + } + continue; + + kw_found: + if (GETLOCAL(j) != NULL) { + PyErr_Format(PyExc_TypeError, + "%U() got multiple " + "values for argument '%S'", + co->co_name, + keyword); + goto fail; + } + Py_INCREF(value); + SETLOCAL(j, value); + } + + /* Handle keyword arguments (passed as a dictionary) */ + if (kwargs != NULL) { + if (fastlocals_dict(fastlocals, co, kwdict, kwargs) < 0) { + goto fail; + } + } + + /* Check the number of positional arguments */ if (argcount > co->co_argcount && !(co->co_flags & CO_VARARGS)) { too_many_positional(co, argcount, defcount, fastlocals); goto fail; } + + /* Add missing positional arguments (copy default values from defs) */ if (argcount < co->co_argcount) { int m = co->co_argcount - defcount; int missing = 0; @@ -3922,6 +4067,8 @@ static PyObject * } } } + + /* Add missing keyword arguments (copy default values from kwdefs) */ if (co->co_kwonlyargcount > 0) { int missing = 0; for (i = co->co_argcount; i < total_args; i++) { @@ -3964,12 +4111,15 @@ static PyObject * goto fail; SETLOCAL(co->co_nlocals + i, c); } + + /* Copy closure variables to free variables */ for (i = 0; i < PyTuple_GET_SIZE(co->co_freevars); ++i) { PyObject *o = PyTuple_GET_ITEM(closure, i); Py_INCREF(o); freevars[PyTuple_GET_SIZE(co->co_cellvars) + i] = o; } + /* Handle generator/coroutine */ if (co->co_flags & (CO_GENERATOR | CO_COROUTINE)) { PyObject *gen; PyObject *coro_wrapper = tstate->coroutine_wrapper; @@ -4033,10 +4183,10 @@ PyEval_EvalCodeEx(PyObject *_co, PyObjec PyObject **args, int argcount, PyObject **kws, int kwcount, PyObject **defs, int defcount, PyObject *kwdefs, PyObject *closure) { - return _PyEval_EvalCodeWithName(_co, globals, locals, - args, argcount, kws, kwcount, - defs, defcount, kwdefs, closure, - NULL, NULL); + return _PyEval_EvalCode(_co, globals, locals, + args, argcount, kws, kwcount, NULL, + defs, defcount, kwdefs, closure, + NULL, NULL); } static PyObject * @@ -4555,6 +4705,10 @@ PyEval_CallObjectWithKeywords(PyObject * #endif if (arg == NULL) { + if (kw == NULL) { + return PyObject_CallFunctionObjArgs(func, NULL); + } + arg = PyTuple_New(0); if (arg == NULL) return NULL; @@ -4723,10 +4877,12 @@ call_function(PyObject ***pp_stack, int } else Py_INCREF(func); READ_TIMESTAMP(*pintr0); - if (PyFunction_Check(func)) - x = fast_function(func, pp_stack, n, na, nk); - else + if (PyFunction_Check(func)) { + x = fast_function(func, (*pp_stack) - n, n, na, nk); + } + else { x = do_call(func, pp_stack, na, nk); + } READ_TIMESTAMP(*pintr1); Py_DECREF(func); @@ -4756,62 +4912,121 @@ call_function(PyObject ***pp_stack, int done before evaluating the frame. */ +static PyObject* +_PyFunction_FastCallNoKw(PyObject * const *args, Py_ssize_t na, + PyCodeObject *co, PyObject *globals) +{ + PyFrameObject *f; + PyThreadState *tstate = PyThreadState_GET(); + PyObject **fastlocals; + Py_ssize_t i; + PyObject *result; + + PCALL(PCALL_FASTER_FUNCTION); + assert(globals != NULL); + /* XXX Perhaps we should create a specialized + PyFrame_New() that doesn't take locals, but does + take builtins without sanity checking them. + */ + assert(tstate != NULL); + f = PyFrame_New(tstate, co, globals, NULL); + if (f == NULL) { + return NULL; + } + + fastlocals = f->f_localsplus; + + for (i = 0; i < na; i++) { + Py_INCREF(*args); + fastlocals[i] = *args++; + } + result = PyEval_EvalFrameEx(f,0); + + ++tstate->recursion_depth; + Py_DECREF(f); + --tstate->recursion_depth; + + return result; +} + static PyObject * -fast_function(PyObject *func, PyObject ***pp_stack, int n, int na, int nk) +fast_function(PyObject *func, PyObject **stack, int n, int na, int nk) { PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); PyObject *globals = PyFunction_GET_GLOBALS(func); PyObject *argdefs = PyFunction_GET_DEFAULTS(func); - PyObject *kwdefs = PyFunction_GET_KW_DEFAULTS(func); - PyObject *name = ((PyFunctionObject *)func) -> func_name; - PyObject *qualname = ((PyFunctionObject *)func) -> func_qualname; - PyObject **d = NULL; - int nd = 0; + PyObject *kwdefs, *name, *qualname; + PyObject **d; + int nd; PCALL(PCALL_FUNCTION); PCALL(PCALL_FAST_FUNCTION); - if (argdefs == NULL && co->co_argcount == n && - co->co_kwonlyargcount == 0 && nk==0 && - co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) { - PyFrameObject *f; - PyObject *retval = NULL; - PyThreadState *tstate = PyThreadState_GET(); - PyObject **fastlocals, **stack; - int i; - - PCALL(PCALL_FASTER_FUNCTION); - assert(globals != NULL); - /* XXX Perhaps we should create a specialized - PyFrame_New() that doesn't take locals, but does - take builtins without sanity checking them. - */ - assert(tstate != NULL); - f = PyFrame_New(tstate, co, globals, NULL); - if (f == NULL) - return NULL; - - fastlocals = f->f_localsplus; - stack = (*pp_stack) - n; - - for (i = 0; i < n; i++) { - Py_INCREF(*stack); - fastlocals[i] = *stack++; - } - retval = PyEval_EvalFrameEx(f,0); - ++tstate->recursion_depth; - Py_DECREF(f); - --tstate->recursion_depth; - return retval; + + if (argdefs == NULL && co->co_argcount == na && + co->co_kwonlyargcount == 0 && nk == 0 && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) + { + return _PyFunction_FastCallNoKw(stack, na, co, globals); } + + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + name = ((PyFunctionObject *)func) -> func_name; + qualname = ((PyFunctionObject *)func) -> func_qualname; + if (argdefs != NULL) { d = &PyTuple_GET_ITEM(argdefs, 0); nd = Py_SIZE(argdefs); } - return _PyEval_EvalCodeWithName((PyObject*)co, globals, - (PyObject *)NULL, (*pp_stack)-n, na, - (*pp_stack)-2*nk, nk, d, nd, kwdefs, - PyFunction_GET_CLOSURE(func), - name, qualname); + else { + d = NULL; + nd = 0; + } + return _PyEval_EvalCode((PyObject*)co, globals, (PyObject *)NULL, + stack, na, + stack + na, nk, NULL, + d, nd, kwdefs, + PyFunction_GET_CLOSURE(func), + name, qualname); +} + +PyObject * +_PyFunction_FastCall(PyObject *func, PyObject * const *args, int nargs) +{ + PyCodeObject *co = (PyCodeObject *)PyFunction_GET_CODE(func); + PyObject *globals = PyFunction_GET_GLOBALS(func); + PyObject *argdefs = PyFunction_GET_DEFAULTS(func); + PyObject *kwdefs, *name, *qualname; + PyObject **d; + int nd; + + PCALL(PCALL_FUNCTION); + PCALL(PCALL_FAST_FUNCTION); + + if (argdefs == NULL && co->co_argcount == nargs && + co->co_kwonlyargcount == 0 && + co->co_flags == (CO_OPTIMIZED | CO_NEWLOCALS | CO_NOFREE)) + { + return _PyFunction_FastCallNoKw(args, nargs, co, globals); + } + + kwdefs = PyFunction_GET_KW_DEFAULTS(func); + name = ((PyFunctionObject *)func) -> func_name; + qualname = ((PyFunctionObject *)func) -> func_qualname; + + if (argdefs != NULL) { + d = &PyTuple_GET_ITEM(argdefs, 0); + nd = Py_SIZE(argdefs); + } + else { + d = NULL; + nd = 0; + } + return _PyEval_EvalCode((PyObject*)co, globals, (PyObject *)NULL, + args, nargs, + NULL, 0, NULL, + d, nd, kwdefs, + PyFunction_GET_CLOSURE(func), + name, qualname); } static PyObject *