diff -r d622dbd71f2b -r 74abb8ddf7f2 Include/abstract.h --- a/Include/abstract.h Fri Sep 09 17:37:37 2016 -0700 +++ b/Include/abstract.h Fri Sep 09 17:40:38 2016 -0700 @@ -277,6 +277,22 @@ xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx PyObject *kwnames, PyObject *func); + /* Convert (args, nargs, kwargs) into a (stack, nargs, kwnames). + + Return a new stack which should be released by PyMem_Free(), or return + args unchanged if kwargs is NULL or an empty dictionary. + + The stack uses borrowed references. + + Keyword keys must be Unicode strings, but no check is done in this + function. */ + PyAPI_FUNC(PyObject **) _PyStack_UnpackDict( + PyObject **args, + Py_ssize_t nargs, + PyObject *kwargs, + PyObject **kwnames, + PyObject *func); + /* Call the callable object func with the "fast call" calling convention: args is a C array for positional arguments (nargs is the number of positional arguments), kwargs is a dictionary for keyword arguments. diff -r d622dbd71f2b -r 74abb8ddf7f2 Include/methodobject.h --- a/Include/methodobject.h Fri Sep 09 17:37:37 2016 -0700 +++ b/Include/methodobject.h Fri Sep 09 17:40:38 2016 -0700 @@ -16,6 +16,8 @@ PyAPI_DATA(PyTypeObject) PyCFunction_Typ #define PyCFunction_Check(op) (Py_TYPE(op) == &PyCFunction_Type) typedef PyObject *(*PyCFunction)(PyObject *, PyObject *); +typedef PyObject *(*_PyCFunctionFast) (PyObject *self, PyObject **args, + Py_ssize_t nargs, PyObject *kwnames); typedef PyObject *(*PyCFunctionWithKeywords)(PyObject *, PyObject *, PyObject *); typedef PyObject *(*PyNoArgsFunction)(PyObject *); @@ -83,6 +85,8 @@ PyAPI_FUNC(PyObject *) PyCFunction_NewEx #define METH_COEXIST 0x0040 +#define METH_FASTCALL 0x0080 + #ifndef Py_LIMITED_API typedef struct { PyObject_HEAD diff -r d622dbd71f2b -r 74abb8ddf7f2 Include/modsupport.h --- a/Include/modsupport.h Fri Sep 09 17:37:37 2016 -0700 +++ b/Include/modsupport.h Fri Sep 09 17:40:38 2016 -0700 @@ -58,10 +58,13 @@ typedef struct _PyArg_Parser { } _PyArg_Parser; #ifdef PY_SSIZE_T_CLEAN #define _PyArg_ParseTupleAndKeywordsFast _PyArg_ParseTupleAndKeywordsFast_SizeT +#define _PyArg_ParseStack _PyArg_ParseStack_SizeT #define _PyArg_VaParseTupleAndKeywordsFast _PyArg_VaParseTupleAndKeywordsFast_SizeT #endif PyAPI_FUNC(int) _PyArg_ParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, ...); +PyAPI_FUNC(int) _PyArg_ParseStack(PyObject **args, Py_ssize_t nargs, PyObject *kwnames, + struct _PyArg_Parser *, ...); PyAPI_FUNC(int) _PyArg_VaParseTupleAndKeywordsFast(PyObject *, PyObject *, struct _PyArg_Parser *, va_list); void _PyArg_Fini(void); diff -r d622dbd71f2b -r 74abb8ddf7f2 Objects/abstract.c --- a/Objects/abstract.c Fri Sep 09 17:37:37 2016 -0700 +++ b/Objects/abstract.c Fri Sep 09 17:40:38 2016 -0700 @@ -2403,6 +2403,62 @@ PyObject * return kwdict; } +PyObject ** +_PyStack_UnpackDict(PyObject **args, Py_ssize_t nargs, PyObject *kwargs, + PyObject **p_kwnames, PyObject *func) +{ + PyObject **stack, **kwstack; + Py_ssize_t nkwargs; + Py_ssize_t pos, i; + PyObject *key, *value; + PyObject *kwnames; + + assert(nargs >= 0); + assert(kwargs == NULL || PyDict_CheckExact(kwargs)); + + nkwargs = (kwargs != NULL) ? PyDict_Size(kwargs) : 0; + if (!nkwargs) { + *p_kwnames = NULL; + return args; + } + + if ((size_t)nargs > PY_SSIZE_T_MAX / sizeof(stack[0]) - (size_t)nkwargs) { + PyErr_NoMemory(); + return NULL; + } + + stack = PyMem_Malloc((nargs + nkwargs) * sizeof(stack[0])); + if (stack == NULL) { + PyErr_NoMemory(); + return NULL; + } + + kwnames = PyTuple_New(nkwargs); + if (kwnames == NULL) { + PyMem_Free(stack); + return NULL; + } + + /* Copy position arguments (borrowed references) */ + Py_MEMCPY(stack, args, nargs * sizeof(stack[0])); + + kwstack = stack + nargs; + pos = i = 0; + /* This loop doesn't support lookup function mutating the dictionary + to change its size. It's a deliberate choice for speed, this function is + called in the performance critical hot code. */ + while (PyDict_Next(kwargs, &pos, &key, &value)) { + Py_INCREF(key); + PyTuple_SET_ITEM(kwnames, i, key); + /* The stack contains borrowed references */ + kwstack[i] = value; + i++; + } + + *p_kwnames = kwnames; + return stack; +} + PyObject * _PyObject_FastCallKeywords(PyObject *func, PyObject **stack, Py_ssize_t nargs, PyObject *kwnames) diff -r d622dbd71f2b -r 74abb8ddf7f2 Objects/methodobject.c --- a/Objects/methodobject.c Fri Sep 09 17:37:37 2016 -0700 +++ b/Objects/methodobject.c Fri Sep 09 17:40:38 2016 -0700 @@ -97,6 +97,11 @@ PyCFunction_Call(PyObject *func, PyObjec if (flags == (METH_VARARGS | METH_KEYWORDS)) { res = (*(PyCFunctionWithKeywords)meth)(self, args, kwds); } + else if (flags == METH_FASTCALL) { + PyObject **stack = &PyTuple_GET_ITEM(args, 0); + Py_ssize_t nargs = PyTuple_GET_SIZE(args); + res = _PyCFunction_FastCallDict(func, stack, nargs, kwds); + } else { if (kwds != NULL && PyDict_Size(kwds) != 0) { PyErr_Format(PyExc_TypeError, "%.200s() takes no keyword arguments", @@ -232,6 +237,22 @@ PyObject * break; } + case METH_FASTCALL: + { + PyObject **stack; + PyObject *kwnames; + _PyCFunctionFast fastmeth = (_PyCFunctionFast)meth; + + stack = _PyStack_UnpackDict(args, nargs, kwargs, &kwnames, func_obj); + + result = (*fastmeth) (self, stack, nargs, kwnames); + if (stack != args) { + PyMem_Free(stack); + } + Py_XDECREF(kwnames); + break; + } + default: PyErr_SetString(PyExc_SystemError, "Bad call flags in PyCFunction_Call. " diff -r d622dbd71f2b -r 74abb8ddf7f2 Python/getargs.c --- a/Python/getargs.c Fri Sep 09 17:37:37 2016 -0700 +++ b/Python/getargs.c Fri Sep 09 17:40:38 2016 -0700 @@ -79,6 +79,10 @@ static int vgetargskeywords(PyObject *, const char *, char **, va_list *, int); static int vgetargskeywordsfast(PyObject *, PyObject *, struct _PyArg_Parser *, va_list *, int); +static int vgetargskeywordsfast_impl(PyObject **args, Py_ssize_t nargs, + PyObject *keywords, PyObject *kwnames, + struct _PyArg_Parser *parser, + va_list *p_va, int flags); static const char *skipitem(const char **, va_list *, int); int @@ -1469,6 +1473,46 @@ int return retval; } +int +_PyArg_ParseStack(PyObject **args, Py_ssize_t nargs, PyObject *kwnames, + struct _PyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + if ((kwnames != NULL && !PyTuple_Check(kwnames)) || + parser == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + va_start(va, parser); + retval = vgetargskeywordsfast_impl(args, nargs, NULL, kwnames, parser, &va, 0); + va_end(va); + return retval; +} + +int +_PyArg_ParseStack_SizeT(PyObject **args, Py_ssize_t nargs, PyObject *kwnames, + struct _PyArg_Parser *parser, ...) +{ + int retval; + va_list va; + + if ((kwnames != NULL && !PyTuple_Check(kwnames)) || + parser == NULL) + { + PyErr_BadInternalCall(); + return 0; + } + + va_start(va, parser); + retval = vgetargskeywordsfast_impl(args, nargs, NULL, kwnames, parser, &va, FLAG_SIZE_T); + va_end(va); + return retval; +} + int _PyArg_VaParseTupleAndKeywordsFast(PyObject *args, PyObject *keywords, @@ -1899,10 +1943,29 @@ parser_clear(struct _PyArg_Parser *parse Py_CLEAR(parser->kwtuple); } +static PyObject* +find_keyword(PyObject *kwnames, PyObject **kwstack, PyObject *key) +{ + Py_ssize_t i, nkwargs; + + nkwargs = PyTuple_GET_SIZE(kwnames); + for (i=0; i < nkwargs; i++) { + PyObject *kwname = PyTuple_GET_ITEM(kwnames, i); + + /* ptr==ptr should match in most cases since keyword keys + should be interned strings */ + if (kwname == key || _PyUnicode_EQ(kwname, key)) { + return kwstack[i]; + } + } + return NULL; +} + static int -vgetargskeywordsfast(PyObject *args, PyObject *keywords, - struct _PyArg_Parser *parser, - va_list *p_va, int flags) +vgetargskeywordsfast_impl(PyObject **args, Py_ssize_t nargs, + PyObject *keywords, PyObject *kwnames, + struct _PyArg_Parser *parser, + va_list *p_va, int flags) { PyObject *kwtuple; char msgbuf[512]; @@ -1911,17 +1974,20 @@ vgetargskeywordsfast(PyObject *args, PyO const char *msg; PyObject *keyword; int i, pos, len; - Py_ssize_t nargs, nkeywords; + Py_ssize_t nkeywords; PyObject *current_arg; freelistentry_t static_entries[STATIC_FREELIST_ENTRIES]; freelist_t freelist; + PyObject **kwstack = NULL; freelist.entries = static_entries; freelist.first_available = 0; freelist.entries_malloced = 0; - assert(args != NULL && PyTuple_Check(args)); assert(keywords == NULL || PyDict_Check(keywords)); + assert(kwnames == NULL || PyTuple_Check(kwnames)); + assert((keywords != NULL || kwnames != NULL) + || (keywords == NULL && kwnames == NULL)); assert(parser != NULL); assert(p_va != NULL); @@ -1942,8 +2008,16 @@ vgetargskeywordsfast(PyObject *args, PyO freelist.entries_malloced = 1; } - nargs = PyTuple_GET_SIZE(args); - nkeywords = (keywords == NULL) ? 0 : PyDict_Size(keywords); + if (keywords != NULL) { + nkeywords = PyDict_Size(keywords); + } + else if (kwnames != NULL) { + nkeywords = PyTuple_GET_SIZE(kwnames); + kwstack = args + nargs; + } + else { + nkeywords = 0; + } if (nargs + nkeywords > len) { PyErr_Format(PyExc_TypeError, "%s%s takes at most %d argument%s (%zd given)", @@ -1976,9 +2050,14 @@ vgetargskeywordsfast(PyObject *args, PyO current_arg = NULL; if (nkeywords && i >= pos) { - current_arg = PyDict_GetItem(keywords, keyword); - if (!current_arg && PyErr_Occurred()) { - return cleanreturn(0, &freelist); + if (keywords != NULL) { + current_arg = PyDict_GetItem(keywords, keyword); + if (!current_arg && PyErr_Occurred()) { + return cleanreturn(0, &freelist); + } + } + else { + current_arg = find_keyword(kwnames, kwstack, keyword); } } if (current_arg) { @@ -1993,7 +2072,7 @@ vgetargskeywordsfast(PyObject *args, PyO } } else if (i < nargs) - current_arg = PyTuple_GET_ITEM(args, i); + current_arg = args[i]; if (current_arg) { msg = convertitem(current_arg, &format, p_va, flags, @@ -2039,24 +2118,52 @@ vgetargskeywordsfast(PyObject *args, PyO /* make sure there are no extraneous keyword arguments */ if (nkeywords > 0) { - PyObject *key, *value; - Py_ssize_t pos = 0; - while (PyDict_Next(keywords, &pos, &key, &value)) { - int match; - if (!PyUnicode_Check(key)) { - PyErr_SetString(PyExc_TypeError, - "keywords must be strings"); - return cleanreturn(0, &freelist); + if (keywords != NULL) { + PyObject *key, *value; + Py_ssize_t pos = 0; + while (PyDict_Next(keywords, &pos, &key, &value)) { + int match; + if (!PyUnicode_Check(key)) { + PyErr_SetString(PyExc_TypeError, + "keywords must be strings"); + return cleanreturn(0, &freelist); + } + match = PySequence_Contains(kwtuple, key); + if (match <= 0) { + if (!match) { + PyErr_Format(PyExc_TypeError, + "'%U' is an invalid keyword " + "argument for this function", + key); + } + return cleanreturn(0, &freelist); + } } - match = PySequence_Contains(kwtuple, key); - if (match <= 0) { - if (!match) { - PyErr_Format(PyExc_TypeError, - "'%U' is an invalid keyword " - "argument for this function", - key); + } + else { + Py_ssize_t j, nkwargs; + + nkwargs = PyTuple_GET_SIZE(kwnames); + for (j=0; j < nkwargs; j++) { + PyObject *key = PyTuple_GET_ITEM(kwnames, j); + int match; + + if (!PyUnicode_Check(key)) { + PyErr_SetString(PyExc_TypeError, + "keywords must be strings"); + return cleanreturn(0, &freelist); } - return cleanreturn(0, &freelist); + + match = PySequence_Contains(kwtuple, key); + if (match <= 0) { + if (!match) { + PyErr_Format(PyExc_TypeError, + "'%U' is an invalid keyword " + "argument for this function", + key); + } + return cleanreturn(0, &freelist); + } } } } @@ -2064,6 +2171,21 @@ vgetargskeywordsfast(PyObject *args, PyO return cleanreturn(1, &freelist); } +static int +vgetargskeywordsfast(PyObject *args, PyObject *keywords, + struct _PyArg_Parser *parser, va_list *p_va, int flags) +{ + PyObject **stack; + Py_ssize_t nargs; + + assert(args != NULL && PyTuple_Check(args)); + + stack = &PyTuple_GET_ITEM(args, 0); + nargs = PyTuple_GET_SIZE(args); + return vgetargskeywordsfast_impl(stack, nargs, keywords, NULL, + parser, p_va, flags); +} + static const char * skipitem(const char **p_format, va_list *p_va, int flags) diff -r d622dbd71f2b -r 74abb8ddf7f2 Tools/clinic/clinic.py --- a/Tools/clinic/clinic.py Fri Sep 09 17:37:37 2016 -0700 +++ b/Tools/clinic/clinic.py Fri Sep 09 17:40:38 2016 -0700 @@ -705,6 +705,11 @@ class CLanguage(Language): {c_basename}({self_type}{self_name}, PyObject *args, PyObject *kwargs) """) + parser_prototype_fastcall = normalize_snippet(""" + static PyObject * + {c_basename}({self_type}{self_name}, PyObject **args, Py_ssize_t nargs, PyObject *kwnames) + """) + parser_prototype_varargs = normalize_snippet(""" static PyObject * {c_basename}({self_type}{self_name}, PyObject *args) @@ -845,6 +850,19 @@ class CLanguage(Language): }} """, indent=4)) + elif not new_or_init: + flags = "METH_FASTCALL" + + parser_prototype = parser_prototype_fastcall + + body = normalize_snippet(""" + if (!_PyArg_ParseStack(args, nargs, kwnames, &_parser, + {parse_arguments})) {{ + goto exit; + }} + """, indent=4) + parser_definition = parser_body(parser_prototype, body) + parser_definition = insert_keywords(parser_definition) else: # positional-or-keyword arguments flags = "METH_VARARGS|METH_KEYWORDS"