diff -r e5ff10873d0f Modules/_elementtree.c --- a/Modules/_elementtree.c Fri May 08 14:15:11 2015 +0300 +++ b/Modules/_elementtree.c Fri May 08 05:30:30 2015 -0700 @@ -3206,14 +3206,14 @@ html: object = NULL target: object = NULL - encoding: str(accept={str, NoneType}) = NULL + encoding: str(accept|={NoneType}) = NULL [clinic start generated code]*/ static int _elementtree_XMLParser___init___impl(XMLParserObject *self, PyObject *html, PyObject *target, const char *encoding) -/*[clinic end generated code: output=d6a16c63dda54441 input=155bc5695baafffd]*/ +/*[clinic end generated code: output=d6a16c63dda54441 input=4bb005d8bf2597d0]*/ { self->entity = PyDict_New(); if (!self->entity) diff -r e5ff10873d0f Modules/_io/_iomodule.c --- a/Modules/_io/_iomodule.c Fri May 08 14:15:11 2015 +0300 +++ b/Modules/_io/_iomodule.c Fri May 08 05:30:30 2015 -0700 @@ -100,9 +100,9 @@ file: object mode: str = "r" buffering: int = -1 - encoding: str(accept={str, NoneType}) = NULL - errors: str(accept={str, NoneType}) = NULL - newline: str(accept={str, NoneType}) = NULL + encoding: str(accept|={NoneType}) = NULL + errors: str(accept|={NoneType}) = NULL + newline: str(accept|={NoneType}) = NULL closefd: int(c_default="1") = True opener: object = None @@ -230,7 +230,7 @@ _io_open_impl(PyModuleDef *module, PyObject *file, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd, PyObject *opener) -/*[clinic end generated code: output=7615d0d746eb14d2 input=f4e1ca75223987bc]*/ +/*[clinic end generated code: output=7615d0d746eb14d2 input=ab5c58c8dade8d8c]*/ { unsigned i; diff -r e5ff10873d0f Modules/_io/textio.c --- a/Modules/_io/textio.c Fri May 08 14:15:11 2015 +0300 +++ b/Modules/_io/textio.c Fri May 08 05:30:30 2015 -0700 @@ -789,9 +789,9 @@ /*[clinic input] _io.TextIOWrapper.__init__ buffer: object - encoding: str(accept={str, NoneType}) = NULL - errors: str(accept={str, NoneType}) = NULL - newline: str(accept={str, NoneType}) = NULL + encoding: str(accept|={NoneType}) = NULL + errors: str(accept|={NoneType}) = NULL + newline: str(accept|={NoneType}) = NULL line_buffering: int(c_default="0") = False write_through: int(c_default="0") = False @@ -830,7 +830,7 @@ const char *encoding, const char *errors, const char *newline, int line_buffering, int write_through) -/*[clinic end generated code: output=56a83402ce2a8381 input=3126cb3101a2c99b]*/ +/*[clinic end generated code: output=56a83402ce2a8381 input=98e258da174c528f]*/ { PyObject *raw, *codec_info = NULL; _PyIO_State *state = NULL; diff -r e5ff10873d0f Modules/_tkinter.c --- a/Modules/_tkinter.c Fri May 08 14:15:11 2015 +0300 +++ b/Modules/_tkinter.c Fri May 08 05:30:30 2015 -0700 @@ -3119,7 +3119,7 @@ /*[clinic input] _tkinter.create - screenName: str(accept={str, NoneType}) = NULL + screenName: str(accept|={NoneType}) = NULL baseName: str = NULL className: str = "Tk" interactive: int(c_default="0") = False @@ -3128,7 +3128,7 @@ if false, then Tk_Init() doesn't get called sync: int(c_default="0") = False if true, then pass -sync to wish - use: str(accept={str, NoneType}) = NULL + use: str(accept|={NoneType}) = NULL if not None, then pass -use to wish / @@ -3139,7 +3139,7 @@ const char *baseName, const char *className, int interactive, int wantobjects, int wantTk, int sync, const char *use) -/*[clinic end generated code: output=b8847800fc3b27eb input=0d522aad1cb0ca0e]*/ +/*[clinic end generated code: output=b8847800fc3b27eb input=b77b925efdec0b35]*/ { /* XXX baseName is not used anymore; * try getting rid of it. */ diff -r e5ff10873d0f Modules/cjkcodecs/multibytecodec.c --- a/Modules/cjkcodecs/multibytecodec.c Fri May 08 14:15:11 2015 +0300 +++ b/Modules/cjkcodecs/multibytecodec.c Fri May 08 05:30:30 2015 -0700 @@ -544,7 +544,7 @@ _multibytecodec.MultibyteCodec.encode input: object - errors: str(accept={str, NoneType}) = NULL + errors: str(accept|={NoneType}) = NULL Return an encoded string version of `input'. @@ -558,7 +558,7 @@ _multibytecodec_MultibyteCodec_encode_impl(MultibyteCodecObject *self, PyObject *input, const char *errors) -/*[clinic end generated code: output=7b26652045ba56a9 input=05f6ced3c8dd0582]*/ +/*[clinic end generated code: output=7b26652045ba56a9 input=70442938f26646e8]*/ { MultibyteCodec_State state; PyObject *errorcb, *r, *ucvt; @@ -613,7 +613,7 @@ _multibytecodec.MultibyteCodec.decode input: Py_buffer - errors: str(accept={str, NoneType}) = NULL + errors: str(accept|={NoneType}) = NULL Decodes 'input'. @@ -627,7 +627,7 @@ _multibytecodec_MultibyteCodec_decode_impl(MultibyteCodecObject *self, Py_buffer *input, const char *errors) -/*[clinic end generated code: output=ff419f65bad6cc77 input=a7d45f87f75e5e02]*/ +/*[clinic end generated code: output=ff419f65bad6cc77 input=98fca61567daa6eb]*/ { MultibyteCodec_State state; MultibyteDecodeBuffer buf; diff -r e5ff10873d0f Modules/pyexpat.c --- a/Modules/pyexpat.c Fri May 08 14:15:11 2015 +0300 +++ b/Modules/pyexpat.c Fri May 08 05:30:30 2015 -0700 @@ -911,7 +911,7 @@ /*[clinic input] pyexpat.xmlparser.ExternalEntityParserCreate - context: str(accept={str, NoneType}) + context: str(accept|={NoneType}) encoding: str = NULL / @@ -922,7 +922,7 @@ pyexpat_xmlparser_ExternalEntityParserCreate_impl(xmlparseobject *self, const char *context, const char *encoding) -/*[clinic end generated code: output=535cda9d7a0fbcd6 input=b906714cc122c322]*/ +/*[clinic end generated code: output=535cda9d7a0fbcd6 input=e20d18fa1ac9c537]*/ { xmlparseobject *new_parser; int i; @@ -1546,8 +1546,8 @@ /*[clinic input] pyexpat.ParserCreate - encoding: str(accept={str, NoneType}) = NULL - namespace_separator: str(accept={str, NoneType}) = NULL + encoding: str(accept|={NoneType}) = NULL + namespace_separator: str(accept|={NoneType}) = NULL intern: object = NULL Return a new XML parser object. @@ -1556,7 +1556,7 @@ static PyObject * pyexpat_ParserCreate_impl(PyModuleDef *module, const char *encoding, const char *namespace_separator, PyObject *intern) -/*[clinic end generated code: output=81fccd233e1743a8 input=23d29704acad385d]*/ +/*[clinic end generated code: output=81fccd233e1743a8 input=8af2e7ddc61fb7b3]*/ { PyObject *result; int intern_decref = 0; diff -r e5ff10873d0f Tools/clinic/clinic.py --- a/Tools/clinic/clinic.py Fri May 08 14:15:11 2015 +0300 +++ b/Tools/clinic/clinic.py Fri May 08 05:30:30 2015 -0700 @@ -25,6 +25,8 @@ import sys import tempfile import textwrap +import token +import tokenize import traceback import types import uuid @@ -3159,6 +3161,89 @@ return line[indent:] +def _line_generator(b): + """ + Takes a bytes object containing lines of text + separated by newlines. + + Returns a generator yielding individual lines. + + Designed to be used by tokenize.tokenize(). + """ + lines = list(b.split(b'\n')) + lines.reverse() + def closure(): + if lines: + return lines.pop() + b"\n" + raise StopIteration() + return closure + + +def _strip_or_equals(s): + """ + Modifies a snippet of Python syntax, + converting text that looks like + |= + into + = + (where "" is an identifier). + + Returns a tuple: + (ids, stripped) + where "ids" is a list of identifiers that used to + be followed by '|=' tokens, and "stripped" is s with + all such '|='' tokens converted to '='' tokens. + Also, any trailing whitespace will be stripped. + (Uses the tokenize module--does a proper job.) + + s can be either bytes or string. If string, + it must be ASCII. + + Examples: + b"a = b" => ([], b"a = b") + b"x |= y" => (['x'], b"x = y") + + """ + + if isinstance(s, (bytes, bytearray)): + b = bytes(s) + elif isinstance(s, str): + b = s.encode('ascii') + else: + raise ValueError('strip_plus_equals: s must be either bytes or str') + + b = b'# -*- coding: ascii -*-\n' + b + g = _line_generator(b) + + ids = [] + tokens = [] + + previous_id = None + for t in tokenize.tokenize(g): + if t.type == token.NAME: + previous_id = t.string + elif (t.type == token.OP and + t.string == '|=' and + previous_id): + # found one! + ids.append(previous_id) + end = (t.end[0], t.end[1] - 1) + t = tokenize.TokenInfo(token.OP, '=', t.start, t.end, t.line) + else: + previous_id = None + tokens.append(t) + + stripped = tokenize.untokenize(tokens) + lines = stripped.split(b'\n') + lines.pop(0) # throw away encoding line + stripped = b'\n'.join(lines).rstrip() + + if isinstance(s, str): + stripped = stripped.decode('ascii') + + return (ids, stripped) + + class DSLParser: def __init__(self, clinic): self.clinic = clinic @@ -3661,15 +3746,17 @@ module = None try: - ast_input = "def x({}): pass".format(base) + or_equals_ids, stripped = _strip_or_equals(base) + ast_input = "def x({}): pass".format(stripped) module = ast.parse(ast_input) - except SyntaxError: + except (SyntaxError, tokenize.TokenError): try: # the last = was probably inside a function call, like # c: int(accept={str}) # so assume there was no actual default value. default = None - ast_input = "def x({}): pass".format(line) + or_equals_ids, stripped = _strip_or_equals(line) + ast_input = "def x({}): pass".format(stripped) module = ast.parse(ast_input) except SyntaxError: pass @@ -3812,7 +3899,17 @@ fail('{} is not a valid {}converter'.format(name, legacy_str)) # if you use a c_name for the parameter, we just give that name to the converter # but the parameter object gets the python name - converter = dict[name](c_name or parameter_name, parameter_name, self.function, value, **kwargs) + converter_class = dict[name] + if or_equals_ids: + p = inspect.signature(converter_class.converter_init).parameters + for id in or_equals_ids: + if id not in kwargs: + continue + if id not in p: + fail("Argument " + repr(id) + " passed in to " + repr(name) + " but is not a keyword argument") + kwargs[id] |= p[id].default + + converter = converter_class(c_name or parameter_name, parameter_name, self.function, value, **kwargs) kind = inspect.Parameter.KEYWORD_ONLY if self.keyword_only else inspect.Parameter.POSITIONAL_OR_KEYWORD