diff --git a/Include/pyport.h b/Include/pyport.h --- a/Include/pyport.h +++ b/Include/pyport.h @@ -188,6 +188,13 @@ #define SIZEOF_PY_UHASH_T SIZEOF_SIZE_T typedef size_t Py_uhash_t; +/* Only used for compatibility with code that may not be PY_SSIZE_T_CLEAN. */ +#ifdef PY_SSIZE_T_CLEAN +typedef Py_ssize_t Py_ssize_clean_t; +#else +typedef int Py_ssize_clean_t; +#endif + /* Largest possible value of size_t. SIZE_MAX is part of C99, so it might be defined on some platforms. If it is not defined, (size_t)-1 is a portable diff --git a/Modules/_dbmmodule.c b/Modules/_dbmmodule.c --- a/Modules/_dbmmodule.c +++ b/Modules/_dbmmodule.c @@ -43,6 +43,12 @@ static PyObject *DbmError; +/*[clinic] +module dbm +class dbm.dbm +[clinic]*/ +/*[clinic checksum: da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ + static PyObject * newdbmobject(const char *file, int flags, int mode) { @@ -248,27 +254,76 @@ 0, /* sq_inplace_repeat */ }; +/*[clinic] + +dbm.dbm.get + + key: str(length=True) + [ + default: object + ] + / + +Return the value for key if present, otherwise default. +[clinic]*/ + +PyDoc_STRVAR(dbm_dbm_get__doc__, +"Return the value for key if present, otherwise default.\n" +"\n" +"dbm.dbm.get(key, [default])"); + +#define DBM_DBM_GET_METHODDEF \ + {"get", (PyCFunction)dbm_dbm_get, METH_VARARGS, dbm_dbm_get__doc__}, + static PyObject * -dbm_get(dbmobject *dp, PyObject *args) +dbm_dbm_get_impl(PyObject *self, const char *key, Py_ssize_clean_t key_length, int group_right_1, PyObject *default_value); + +static PyObject * +dbm_dbm_get(PyObject *self, PyObject *args) { - datum key, val; - PyObject *defvalue = Py_None; - char *tmp_ptr; - Py_ssize_t tmp_size; + PyObject *return_value = NULL; + const char *key; + Py_ssize_clean_t key_length; + int group_right_1 = 0; + PyObject *default_value = NULL; - if (!PyArg_ParseTuple(args, "s#|O:get", - &tmp_ptr, &tmp_size, &defvalue)) - return NULL; - key.dptr = tmp_ptr; - key.dsize = tmp_size; + switch (PyTuple_Size(args)) { + case 1: + if (!PyArg_ParseTuple(args, "s#:get", &key, &key_length)) + return NULL; + break; + case 2: + if (!PyArg_ParseTuple(args, "s#O:get", &key, &key_length, &default_value)) + return NULL; + group_right_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "dbm.dbm.get requires 1 to 2 arguments"); + return NULL; + } + return_value = dbm_dbm_get_impl(self, key, key_length, group_right_1, default_value); + + return return_value; +} + +static PyObject * +dbm_dbm_get_impl(PyObject *self, const char *key, Py_ssize_clean_t key_length, int group_right_1, PyObject *default_value) +/*[clinic checksum: 4d05efa43d74bd8b7dcf8de2cc16dc03dd383e93]*/ +{ + dbmobject *dp = (dbmobject *)self; + datum dbm_key, val; + + if (!group_right_1) + default_value = Py_None; + dbm_key.dptr = (char *)key; + dbm_key.dsize = key_length; check_dbmobject_open(dp); - val = dbm_fetch(dp->di_dbm, key); + val = dbm_fetch(dp->di_dbm, dbm_key); if (val.dptr != NULL) return PyBytes_FromStringAndSize(val.dptr, val.dsize); - else { - Py_INCREF(defvalue); - return defvalue; - } + + Py_INCREF(default_value); + return default_value; } static PyObject * @@ -333,9 +388,7 @@ "close()\nClose the database."}, {"keys", (PyCFunction)dbm_keys, METH_NOARGS, "keys() -> list\nReturn a list of all keys in the database."}, - {"get", (PyCFunction)dbm_get, METH_VARARGS, - "get(key[, default]) -> value\n" - "Return the value for key if present, otherwise default."}, + DBM_DBM_GET_METHODDEF {"setdefault", (PyCFunction)dbm_setdefault, METH_VARARGS, "setdefault(key[, default]) -> value\n" "Return the value for key if present, otherwise default. If key\n" @@ -379,7 +432,6 @@ /* ----------------------------------------------------------------- */ /*[clinic] -module dbm dbm.open as dbmopen diff --git a/Modules/zlibmodule.c b/Modules/zlibmodule.c --- a/Modules/zlibmodule.c +++ b/Modules/zlibmodule.c @@ -81,6 +81,13 @@ PyErr_Format(ZlibError, "Error %d %s: %.200s", err, msg, zmsg); } +/*[clinic] +module zlib +class zlib.Compress +class zlib.Decompress +[clinic]*/ +/*[clinic checksum: da39a3ee5e6b4b0d3255bfef95601890afd80709]*/ + PyDoc_STRVAR(compressobj__doc__, "compressobj(level=-1, method=DEFLATED, wbits=15, memlevel=8,\n" " strategy=Z_DEFAULT_STRATEGY[, zdict])\n" @@ -157,32 +164,86 @@ PyMem_RawFree(ptr); } -PyDoc_STRVAR(compress__doc__, -"compress(string[, level]) -- Returned compressed string.\n" +/*[clinic] +zlib.compress + bytes: Py_buffer + Binary data to be compressed. + [ + level: int + Compression level, in 0-9. + ] + / + +Returns compressed string. + +[clinic]*/ + +PyDoc_STRVAR(zlib_compress__doc__, +"Returns compressed string.\n" "\n" -"Optional arg level is the compression level, in 0-9."); +"zlib.compress(bytes, [level])\n" +" bytes\n" +" Binary data to be compressed.\n" +" level\n" +" Compression level, in 0-9."); + +#define ZLIB_COMPRESS_METHODDEF \ + {"compress", (PyCFunction)zlib_compress, METH_VARARGS, zlib_compress__doc__}, static PyObject * -PyZlib_compress(PyObject *self, PyObject *args) +zlib_compress_impl(PyObject *module, Py_buffer *bytes, int group_right_1, int level); + +static PyObject * +zlib_compress(PyObject *module, PyObject *args) +{ + PyObject *return_value = NULL; + Py_buffer bytes; + int group_right_1 = 0; + int level = 0; + + switch (PyTuple_Size(args)) { + case 1: + if (!PyArg_ParseTuple(args, "y*:compress", &bytes)) + return NULL; + break; + case 2: + if (!PyArg_ParseTuple(args, "y*i:compress", &bytes, &level)) + return NULL; + group_right_1 = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "zlib.compress requires 1 to 2 arguments"); + return NULL; + } + return_value = zlib_compress_impl(module, &bytes, group_right_1, level); + + /* Cleanup for bytes */ + if (bytes.buf) + PyBuffer_Release(&bytes); + + return return_value; +} + +static PyObject * +zlib_compress_impl(PyObject *module, Py_buffer *bytes, int group_right_1, int level) +/*[clinic checksum: 95933a3d1491114b9bb5a530784cd256428c46d0]*/ { PyObject *ReturnVal = NULL; - Py_buffer pinput; Byte *input, *output = NULL; unsigned int length; - int level=Z_DEFAULT_COMPRESSION, err; + int err; z_stream zst; - /* require Python string object, optional 'level' arg */ - if (!PyArg_ParseTuple(args, "y*|i:compress", &pinput, &level)) - return NULL; + if (!group_right_1) + level = Z_DEFAULT_COMPRESSION; - if ((size_t)pinput.len > UINT_MAX) { + if ((size_t)bytes->len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "Size does not fit in an unsigned int"); goto error; } - input = pinput.buf; - length = (unsigned int)pinput.len; + input = bytes->buf; + length = (unsigned int)bytes->len; zst.avail_out = length + length/1000 + 12 + 1; @@ -239,7 +300,6 @@ zlib_error(zst, err, "while finishing compression"); error: - PyBuffer_Release(&pinput); PyMem_Free(output); return ReturnVal; @@ -682,10 +742,6 @@ } /*[clinic] - -module zlib -class zlib.Decompress - zlib.Decompress.decompress data: Py_buffer @@ -739,14 +795,15 @@ exit: /* Cleanup for data */ - PyBuffer_Release(&data); + if (data.buf) + PyBuffer_Release(&data); return return_value; } static PyObject * zlib_Decompress_decompress_impl(PyObject *self, Py_buffer *data, unsigned int max_length) -/*[clinic checksum: 76ca9259e3f5ca86bae9da3d0e75637b5d492234]*/ +/*[clinic checksum: f83e91728d327462d7ccbee95299514f26b92253]*/ { compobject *zself = (compobject *)self; int err; @@ -966,8 +1023,6 @@ #ifdef HAVE_ZLIB_COPY /*[clinic] - -class zlib.Compress zlib.Compress.copy Return a copy of the compression object. @@ -1295,8 +1350,7 @@ { {"adler32", (PyCFunction)PyZlib_adler32, METH_VARARGS, adler32__doc__}, - {"compress", (PyCFunction)PyZlib_compress, METH_VARARGS, - compress__doc__}, + ZLIB_COMPRESS_METHODDEF {"compressobj", (PyCFunction)PyZlib_compressobj, METH_VARARGS|METH_KEYWORDS, compressobj__doc__}, {"crc32", (PyCFunction)PyZlib_crc32, METH_VARARGS, diff --git a/Tools/clinic/clinic.py b/Tools/clinic/clinic.py --- a/Tools/clinic/clinic.py +++ b/Tools/clinic/clinic.py @@ -23,7 +23,6 @@ import tempfile import textwrap - # TODO: # converters for # @@ -52,6 +51,8 @@ # is too new for us. # +version = '1' + _empty = inspect._empty _void = inspect._void @@ -195,6 +196,46 @@ return output()[:-1] +def version_splitter(s): + """Splits a version string into a tuple of integers. + + The following ASCII characters are allowed, and employ + the following conversions: + a -> -3 + b -> -2 + c -> -1 + (This permits Python-style version strings such as "1.4b3".) + """ + version = [] + accumulator = [] + def flush(): + if not accumulator: + raise ValueError('Malformed version string: ' + repr(s)) + version.append(int(''.join(accumulator))) + accumulator.clear() + + for c in s: + if c.isdigit(): + accumulator.append(c) + elif c == '.': + flush() + elif c in 'abc': + flush() + version.append('abc'.index(c) - 3) + else: + raise ValueError('Illegal character ' + repr(c) + ' in version string ' + repr(s)) + flush() + return tuple(version) + +def version_comparitor(version1, version2): + iterator = itertools.zip_longest(version_splitter(version1), version_splitter(version2), fillvalue=0) + for i, (a, b) in enumerate(iterator): + if a < b: + return -1 + if a > b: + return 1 + return 0 + class CRenderData: def __init__(self): @@ -1307,6 +1348,7 @@ # The C converter *function* to be used, if any. # (If this is not None, format_unit must be 'O&'.) converter = None + encoding = None impl_by_reference = False parse_by_reference = True @@ -1354,6 +1396,8 @@ # impl_arguments s = ("&" if self.impl_by_reference else "") + name data.impl_arguments.append(s) + if self.length: + data.impl_arguments.append(self.length_name()) # keywords data.keywords.append(name) @@ -1370,12 +1414,20 @@ # impl_parameters data.impl_parameters.append(self.simple_declaration(by_reference=self.impl_by_reference)) + if self.length: + data.impl_parameters.append("Py_ssize_clean_t " + self.length_name()) # cleanup cleanup = self.cleanup() if cleanup: data.cleanup.append('/* Cleanup for ' + name + ' */\n' + cleanup.rstrip() + "\n") + def length_name(self): + """Computes the name of the associated "length" variable.""" + if not self.length: + return None + return ensure_legal_c_identifier(self.name) + "_length" + # Why is this one broken out separately? # For "positional-only" function parsing, # which generates a bunch of PyArg_ParseTuple calls. @@ -1388,9 +1440,13 @@ if self.encoding: list.append(self.encoding) - s = ("&" if self.parse_by_reference else "") + ensure_legal_c_identifier(self.name) + legal_name = ensure_legal_c_identifier(self.name) + s = ("&" if self.parse_by_reference else "") + legal_name list.append(s) + if self.length: + list.append("&" + self.length_name()) + # # All the functions after here are intended as extension points. # @@ -1421,6 +1477,10 @@ declaration.append(" = ") declaration.append(default) declaration.append(";") + if self.length: + declaration.append('\nPy_ssize_clean_t ') + declaration.append(self.length_name()) + declaration.append(';') return "".join(declaration) def initialize(self): @@ -1462,7 +1522,7 @@ def converter_init(self, *, bitwise=False): if bitwise: - format_unit = 'B' + self.format_unit = 'B' class short_converter(CConverter): type = 'short' @@ -1478,15 +1538,17 @@ if not bitwise: fail("Unsigned shorts must be bitwise (for now).") -@add_legacy_c_converter('C', from_str=True) +@add_legacy_c_converter('C', types='str') class int_converter(CConverter): type = 'int' format_unit = 'i' c_ignored_default = "0" - def converter_init(self, *, from_str=False): - if from_str: - format_unit = 'C' + def converter_init(self, *, types='int'): + if types == 'str': + self.format_unit = 'C' + elif types != 'int': + fail("int_converter: illegal 'types' argument") class unsigned_int_converter(CConverter): type = 'unsigned int' @@ -1568,18 +1630,66 @@ self.encoding = type -@add_legacy_c_converter('y', from_bytes=True) +@add_legacy_c_converter('s#', length=True) +@add_legacy_c_converter('y', type="bytes") +@add_legacy_c_converter('y#', type="bytes", length=True) @add_legacy_c_converter('z', nullable=True) +@add_legacy_c_converter('z#', nullable=True, length=True) class str_converter(CConverter): type = 'const char *' format_unit = 's' - def converter_init(self, *, nullable=False, from_bytes=False): - if from_bytes: - assert not nullable - format_unit = 'y' - if nullable: - format_unit = 'z' + def converter_init(self, *, encoding=None, types="str", + length=False, nullable=False, zeroes=False): + + types = set(types.strip().split()) + bytes_type = set(("bytes",)) + str_type = set(("str",)) + all_3_type = set(("bytearray",)) | bytes_type | str_type + is_bytes = types == bytes_type + is_str = types == str_type + is_all_3 = types == all_3_type + + self.length = bool(length) + format_unit = None + + if encoding: + self.encoding = encoding + + if is_str and not (length or zeroes or nullable): + format_unit = 'es' + elif is_all_3 and not (length or zeroes or nullable): + format_unit = 'et' + elif is_str and length and zeroes and not nullable: + format_unit = 'es#' + elif is_all_3 and length and not (nullable or zeroes): + format_unit = 'et#' + + if format_unit.endswith('#'): + # TODO set pointer to NULL + # TODO add cleanup for buffer + pass + + else: + if zeroes: + fail("str_converter: illegal combination of arguments (zeroes is only legal with an encoding)") + + if is_bytes and not (nullable or length): + format_unit = 'y' + elif is_bytes and length and not nullable: + format_unit = 'y#' + elif is_str and not (nullable or length): + format_unit = 's' + elif is_str and length and not nullable: + format_unit = 's#' + elif is_str and nullable and not length: + format_unit = 'z' + elif is_str and nullable and length: + format_unit = 'z#' + + if not format_unit: + fail("str_converter: illegal combination of arguments") + self.format_unit = format_unit class PyBytesObject_converter(CConverter): @@ -1594,36 +1704,63 @@ type = 'PyObject *' format_unit = 'U' +@add_legacy_c_converter('u#', length=True) @add_legacy_c_converter('Z', nullable=True) +@add_legacy_c_converter('Z#', nullable=True, length=True) class Py_UNICODE_converter(CConverter): type = 'Py_UNICODE *' format_unit = 'u' - def converter_init(self, *, nullable=False): - if nullable: - format_unit = 'Z' - -@add_legacy_c_converter('s*', zeroes=True) -@add_legacy_c_converter('w*', read_write=True) -@add_legacy_c_converter('z*', zeroes=True, nullable=True) + def converter_init(self, *, nullable=False, length=False): + format_unit = 'Z' if nullable else 'u' + if length: + format_unit += '#' + self.length = True + self.format_unit = format_unit + +# +# We define three string conventions for buffer types in the 'types' argument: +# 'buffer' : any object supporting the buffer interface +# 'rwbuffer': any object supporting the buffer interface, but must be writeable +# 'robuffer': any object supporting the buffer interface, but must not be writeable +# +@add_legacy_c_converter('s*', types='str bytes bytearray buffer') +@add_legacy_c_converter('z*', types='str bytes bytearray buffer', nullable=True) +@add_legacy_c_converter('w*', types='bytearray rwbuffer') class Py_buffer_converter(CConverter): type = 'Py_buffer' format_unit = 'y*' impl_by_reference = True c_ignored_default = "{NULL, NULL, 0, 0, 0, 0, NULL, NULL, NULL, NULL, NULL}" - def converter_init(self, *, str=False, zeroes=False, nullable=False, read_write=False): - if not str: - assert not (zeroes or nullable or read_write) - elif read_write: - assert not (zeroes or nullable) - self.format_unit = 'w*' + def converter_init(self, *, types='bytes bytearray buffer', nullable=False): + types = set(types.strip().split()) + bytes_type = set(('bytes',)) + bytearray_type = set(('bytearray',)) + buffer_type = set(('buffer',)) + rwbuffer_type = set(('rwbuffer',)) + robuffer_type = set(('robuffer',)) + str_type = set(('str',)) + bytes_bytearray_buffer_type = bytes_type | bytearray_type | buffer_type + + format_unit = None + if types == (str_type | bytes_bytearray_buffer_type): + format_unit = 's*' if not nullable else 'z*' else: - assert zeroes - self.format_unit = 'z*' if nullable else 's*' + if nullable: + fail('Py_buffer_converter: illegal combination of arguments (nullable=True)') + elif types == (bytes_bytearray_buffer_type): + format_unit = 'y*' + elif types == (bytearray_type | rwuffer_type): + format_unit = 'w*' + if not format_unit: + fail("Py_buffer_converter: illegal combination of arguments") + + self.format_unit = format_unit def cleanup(self): - return "PyBuffer_Release(&" + ensure_legal_c_identifier(self.name) + ");\n" + name = ensure_legal_c_identifier(self.name) + return "".join(["if (", name, ".buf)\n PyBuffer_Release(&", name, ");\n"]) def add_c_return_converter(f, name=None): @@ -1830,6 +1967,11 @@ self.kind = CALLABLE self.coexist = False + def directive_version(self, required): + global version + if version_comparitor(version, required) < 0: + fail("Insufficient Clinic version!\n Version: " + version + "\n Required: " + required) + def directive_module(self, name): fields = name.split('.') new = fields.pop()