diff -r 41fb7fd04b5d Doc/howto/clinic.rst --- a/Doc/howto/clinic.rst Mon May 04 06:59:46 2015 -0700 +++ b/Doc/howto/clinic.rst Mon May 04 08:05:01 2015 -0700 @@ -758,6 +758,14 @@ In addition, some converters accept additional arguments. Here is a list of these arguments, along with their meanings: + ``accept`` + A set of Python types (and possibly pseudo-types); + this restricts the allowable Python argument to values of these types. + (This is not a general-purpose facility; as a rule it only supports + specific lists of types as shown in the legacy converter table.) + + To accept ``None``, add ``NoneType`` to this set. + ``bitwise`` Only supported for unsigned integers. The native integer value of this Python argument will be written to the parameter without any range checking, @@ -777,25 +785,15 @@ string be passed in to the impl function, just after the string parameter, in a parameter named ``_length``. - ``nullable`` - Only supported for strings. If true, this parameter may also be set to - ``None``, in which case the C parameter will be set to ``NULL``. - ``subclass_of`` Only supported for the ``object`` converter. Requires that the Python value be a subclass of a Python type, as expressed in C. - ``types`` - Only supported for the ``object`` (and ``self``) converter. Specifies + ``type`` + Only supported for the ``object`` and ``self`` converters. Specifies the C type that will be used to declare the variable. Default value is ``"PyObject *"``. - ``types`` - A string containing a list of Python types (and possibly pseudo-types); - this restricts the allowable Python argument to values of these types. - (This is not a general-purpose facility; as a rule it only supports - specific lists of types as shown in the legacy converter table.) - ``zeroes`` Only supported for strings. If true, embedded NUL bytes (``'\\0'``) are permitted inside the value. @@ -803,8 +801,8 @@ Please note, not every possible combination of arguments will work. Often these arguments are implemented internally by specific ``PyArg_ParseTuple`` *format units*, with specific behavior. For example, currently you cannot -call ``str`` and pass in ``zeroes=True`` without also specifying an ``encoding``; -although it's perfectly reasonable to think this would work, these semantics don't +call ``str`` and pass in ``length=True`` without also specifying ``zeroes=True``. +Although it's perfectly reasonable to think this would work, these semantics don't map to any existing format unit. So Argument Clinic doesn't support it. (Or, at least, not yet.) @@ -816,13 +814,13 @@ ``'B'`` ``unsigned_char(bitwise=True)`` ``'b'`` ``unsigned_char`` ``'c'`` ``char`` -``'C'`` ``int(types='str')`` +``'C'`` ``int(accept={str})`` ``'d'`` ``double`` ``'D'`` ``Py_complex`` +``'es'`` ``str(encoding='name_of_encoding')`` ``'es#'`` ``str(encoding='name_of_encoding', length=True, zeroes=True)`` -``'es'`` ``str(encoding='name_of_encoding')`` -``'et#'`` ``str(encoding='name_of_encoding', types='bytes bytearray str', length=True)`` -``'et'`` ``str(encoding='name_of_encoding', types='bytes bytearray str')`` +``'et'`` ``str(encoding='name_of_encoding', accept={bytes, bytearray, str})`` +``'et#'`` ``str(encoding='name_of_encoding', length=True, zeroes=True, accept={bytes, bytearray, str})`` ``'f'`` ``float`` ``'h'`` ``short`` ``'H'`` ``unsigned_short(bitwise=True)`` @@ -832,27 +830,27 @@ ``'K'`` ``unsigned_PY_LONG_LONG(bitwise=True)`` ``'L'`` ``PY_LONG_LONG`` ``'n'`` ``Py_ssize_t`` +``'O'`` ``object`` ``'O!'`` ``object(subclass_of='&PySomething_Type')`` ``'O&'`` ``object(converter='name_of_c_function')`` -``'O'`` ``object`` ``'p'`` ``bool`` -``'s#'`` ``str(length=True)`` ``'S'`` ``PyBytesObject`` ``'s'`` ``str`` -``'s*'`` ``Py_buffer(types='str bytes bytearray buffer')`` +``'s#'`` ``str(length=True)`` +``'s*'`` ``Py_buffer(accept={buffer, str})`` +``'U'`` ``unicode`` +``'u'`` ``Py_UNICODE`` ``'u#'`` ``Py_UNICODE(length=True)`` -``'u'`` ``Py_UNICODE`` -``'U'`` ``unicode`` -``'w*'`` ``Py_buffer(types='bytearray rwbuffer')`` -``'y#'`` ``str(types='bytes', length=True)`` +``'w*'`` ``Py_buffer(accept={rwbuffer})`` ``'Y'`` ``PyByteArrayObject`` -``'y'`` ``str(types='bytes')`` +``'y'`` ``str(accept={bytes})`` +``'y#'`` ``str(accept={robuffer}, length=True, zeroes=True)`` ``'y*'`` ``Py_buffer`` -``'Z#'`` ``Py_UNICODE(nullable=True, length=True)`` -``'z#'`` ``str(nullable=True, length=True)`` -``'Z'`` ``Py_UNICODE(nullable=True)`` -``'z'`` ``str(nullable=True)`` -``'z*'`` ``Py_buffer(types='str bytes bytearray buffer', nullable=True)`` +``'Z'`` ``Py_UNICODE(accept={str, NoneType})`` +``'Z#'`` ``Py_UNICODE(accept={str, NoneType}, length=True, zeroes=True)`` +``'z'`` ``str(accept={str, NoneType})`` +``'z#'`` ``str(accept={str, NoneType}, length=True, zeroes=True)`` +``'z*'`` ``Py_buffer(accept={buffer, str, NoneType})`` ========= ================================================================================= As an example, here's our sample ``pickle.Pickler.dump`` using the proper diff -r 41fb7fd04b5d Modules/_dbmmodule.c --- a/Modules/_dbmmodule.c Mon May 04 06:59:46 2015 -0700 +++ b/Modules/_dbmmodule.c Mon May 04 08:05:01 2015 -0700 @@ -285,6 +285,7 @@ _dbm_dbm_get_impl(dbmobject *self, const char *key, Py_ssize_clean_t key_length, PyObject *default_value) /*[clinic end generated code: output=b44f95eba8203d93 input=3c7c1afd9c508457]*/ +/*[clinic end generated code: output=4f5c0e523eaf1251 input=9402c0af8582dc69]*/ { datum dbm_key, val; diff -r 41fb7fd04b5d Modules/arraymodule.c --- a/Modules/arraymodule.c Mon May 04 06:59:46 2015 -0700 +++ b/Modules/arraymodule.c Mon May 04 08:05:01 2015 -0700 @@ -1673,7 +1673,7 @@ /*[clinic input] array.array.fromunicode - ustr: Py_UNICODE(length=True) + ustr: Py_UNICODE(length=True, zeroes=True) / Extends this array with data from the unicode string ustr. @@ -1686,7 +1686,7 @@ static PyObject * array_array_fromunicode_impl(arrayobject *self, Py_UNICODE *ustr, Py_ssize_clean_t ustr_length) -/*[clinic end generated code: output=ebb72fc16975e06d input=56bcedb5ef70139f]*/ +/*[clinic end generated code: output=ebb72fc16975e06d input=3b49c84343c30c05]*/ { char typecode; diff -r 41fb7fd04b5d Tools/clinic/clinic.py --- a/Tools/clinic/clinic.py Mon May 04 06:59:46 2015 -0700 +++ b/Tools/clinic/clinic.py Mon May 04 08:05:01 2015 -0700 @@ -2644,64 +2644,88 @@ class rwbuffer: pass class robuffer: pass -@add_legacy_c_converter('s#', accept={str, robuffer}, length=True) -@add_legacy_c_converter('y', accept={robuffer}) -@add_legacy_c_converter('y#', accept={robuffer}, length=True) -@add_legacy_c_converter('z', accept={str, NoneType}) -@add_legacy_c_converter('z#', accept={str, NoneType}, length=True) -# add_legacy_c_converter not supported for es, es#, et, et# -# because of their extra encoding argument +def str_converter_key(types, encoding, length): + return (frozenset(types), bool(encoding), bool(length)) + +str_converter_argument_map = {} + class str_converter(CConverter): type = 'const char *' default_type = (str, Null, NoneType) format_unit = 's' - def converter_init(self, *, encoding=None, accept={str}, length=False, zeroes=False): - + def converter_init(self, *, accept={str}, + encoding=None, length=False, zeroes=False): + + key = str_converter_key(accept, encoding, length) + format_unit = str_converter_argument_map.get(key) + if not format_unit: + fail("str_converter: illegal combination of arguments", key) + + self.format_unit = format_unit self.length = bool(length) - - is_b_or_ba = accept == {bytes, bytearray} - is_b_or_ba_or_none = accept == {bytes, bytearray, NoneType} - is_str = accept == {str} - is_str_or_none = accept == {str, NoneType} - is_robuffer = accept == {robuffer} - is_str_or_robuffer = accept == {str, robuffer} - is_str_or_robuffer_or_none = accept == {str, robuffer, NoneType} - - format_unit = None - if encoding: + if self.default not in (Null, None, unspecified): + fail("str_converter: Argument Clinic doesn't support default values for encoded strings") self.encoding = encoding - - if is_str and not length and not zeroes: - format_unit = 'es' - elif is_str_or_none and length and zeroes: - format_unit = 'es#' - elif is_b_or_ba and not length and not zeroes: - format_unit = 'et' - elif is_b_or_ba_or_none and length and zeroes: - format_unit = 'et#' - - else: - if zeroes: - fail("str_converter: illegal combination of arguments (zeroes is only legal with an encoding)") - - if is_str and not length: - format_unit = 's' - elif is_str_or_none and not length: - format_unit = 'z' - elif is_robuffer and not length: - format_unit = 'y' - elif is_robuffer and length: - format_unit = 'y#' - elif is_str_or_robuffer and length: - format_unit = 's#' - elif is_str_or_robuffer_or_none and length: - format_unit = 'z#' - - if not format_unit: - fail("str_converter: illegal combination of arguments") - self.format_unit = format_unit + self.type = 'char *' + # sorry, clinic can't support preallocated buffers + # for es# and et# + self.c_default = "NULL" + + def cleanup(self): + if self.encoding: + name = ensure_legal_c_identifier(self.name) + return "".join(["if (", name, ")\n PyMem_FREE(", name, ");\n"]) + +# +# This is the fourth or fifth rewrite of registering all the +# crazy string converter format units. Previous approaches hid +# bugs--generally mismatches between the semantics of the format +# unit and the arguments necessary to represent those semantics +# properly. Hopefully with this approach we'll get it 100% right. +# +# The r() function (short for "register") both registers the +# mapping from arguments to format unit *and* registers the +# legacy C converter for that format unit. +# +def r(format_unit, *, accept, encoding=False, length=False, zeroes=False): + if not encoding and format_unit != 's': + # add the legacy c converters here too. + # + # note: add_legacy_c_converter can't work for + # es, es#, et, or et# + # because of their extra encoding argument + # + # also don't add the converter for 's' because + # the metaclass for CConverter adds it for us. + kwargs = {} + if accept != {str}: + kwargs['accept'] = accept + if length: + kwargs['length'] = True + if zeroes: + kwargs['zeroes'] = True + added_f = functools.partial(str_converter, **kwargs) + legacy_converters[format_unit] = added_f + + d = str_converter_argument_map + key = str_converter_key(accept, encoding, length) + if key in d: + sys.exit("Duplicate keys specified for str_converter_argument_map!") + d[key] = format_unit + +r('es', encoding=True, accept={str}) +r('es#', encoding=True, length=True, zeroes=True, accept={str}) +r('et', encoding=True, accept={bytes, bytearray, str}) +r('et#', encoding=True, length=True, zeroes=True, accept={bytes, bytearray, str}) +r('s', accept={str}) +r('s#', length=True, zeroes=True, accept={robuffer, str}) +r('y', accept={robuffer}) +r('y#', length=True, zeroes=True, accept={robuffer}) +r('z', accept={str, NoneType}) +r('z#', length=True, zeroes=True, accept={robuffer, str, NoneType}) +del r class PyBytesObject_converter(CConverter): @@ -2719,17 +2743,18 @@ default_type = (str, Null, NoneType) format_unit = 'U' -@add_legacy_c_converter('u#', length=True) +@add_legacy_c_converter('u#', length=True, zeroes=True) @add_legacy_c_converter('Z', accept={str, NoneType}) -@add_legacy_c_converter('Z#', accept={str, NoneType}, length=True) +@add_legacy_c_converter('Z#', accept={str, NoneType}, length=True, zeroes=True) class Py_UNICODE_converter(CConverter): type = 'Py_UNICODE *' default_type = (str, Null, NoneType) format_unit = 'u' - def converter_init(self, *, accept={str}, length=False): + def converter_init(self, *, accept={str}, length=False, zeroes=False): format_unit = 'Z' if accept=={str, NoneType} else 'u' - if length: + if length or zeroes: + assert length and zeroes, "Py_UNICODE: length and zeroes must be the same value" format_unit += '#' self.length = True self.format_unit = format_unit