diff -r 026b9f5dd97f Modules/_cursesmodule.c --- a/Modules/_cursesmodule.c Fri Dec 07 10:18:22 2012 -0800 +++ b/Modules/_cursesmodule.c Fri Dec 07 12:25:36 2012 -0800 @@ -553,68 +553,133 @@ /* Addch, Addstr, Addnstr */ +/*[clinic] +curses.window.addch +positional-only + + int x; + group=coordinates + X-coordinate. + + int y; + group=coordinates + Y-coordinate. + + PyObject *ch; + Character to add. + + long attr; + group=attr + Attributes for the character. + +Paint character ch at (y, x) with attributes attr, +overwriting any character previously painter at that location. +By default, the character position and attributes are the +current settings for the window object. +[clinic]*/ + +PyDoc_STRVAR(curses_window_addch__doc__, +"curses.window.addch([x, y,] ch[, attr])\n" +"\n" +" x\n" +" X-coordinate.\n" +"\n" +" y\n" +" Y-coordinate.\n" +"\n" +" ch\n" +" Character to add.\n" +"\n" +" attr\n" +" Attributes for the character.\n" +"\n" +"Paint character ch at (y, x) with attributes attr,\n" +"overwriting any character previously painter at that location.\n" +"By default, the character position and attributes are the\n" +"current settings for the window object.\n"); + +#define CURSES_WINDOW_ADDCH_METHODDEF \ + {"addch", (PyCFunction)curses_window_addch, METH_VARARGS, curses_window_addch__doc__} + static PyObject * -PyCursesWindow_AddCh(PyCursesWindowObject *self, PyObject *args) +curses_window_addch_impl(PyObject *self, int coordinates_group, int x, int y, PyObject *ch, int attr_group, long attr); + +static PyObject * +curses_window_addch(PyObject *self, PyObject *args) { - int rtn, x, y, use_xy = FALSE; - PyObject *chobj; + int coordinates_group = 0; + int x; + int y; + PyObject *ch; + int attr_group = 0; + long attr; + + switch (PyTuple_Size(args)) { + case 1: + if (!PyArg_ParseTuple(args, "O:addch", &ch)) + return NULL; + break; + case 2: + if (!PyArg_ParseTuple(args, "Ol:addch", &ch, &attr)) + return NULL; + attr_group = 1; + break; + case 3: + if (!PyArg_ParseTuple(args, "iiO:addch", &x, &y, &ch)) + return NULL; + coordinates_group = 1; + break; + case 4: + if (!PyArg_ParseTuple(args, "iiOl:addch", &x, &y, &ch, &attr)) + return NULL; + coordinates_group = 1; + attr_group = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "window.addch requires 1 to 4 arguments"); + return NULL; + } + + return curses_window_addch_impl(self, coordinates_group, x, y, ch, attr_group, attr); +} + +static PyObject * +curses_window_addch_impl(PyObject *self, int coordinates_group, int x, int y, PyObject *ch, int attr_group, long attr) +/*[clinic end:355e50ee04fc6675476cbe9baa1a06ff14450c03]*/ +{ + PyCursesWindowObject *cwself = (PyCursesWindowObject *)self; + int rtn; int type; - chtype ch; + chtype cch; #ifdef HAVE_NCURSESW cchar_t wch; #endif - attr_t attr = A_NORMAL; - long lattr; const char *funcname; - switch (PyTuple_Size(args)) { - case 1: - if (!PyArg_ParseTuple(args, "O;ch or int", &chobj)) - return NULL; - break; - case 2: - if (!PyArg_ParseTuple(args, "Ol;ch or int,attr", &chobj, &lattr)) - return NULL; - attr = lattr; - break; - case 3: - if (!PyArg_ParseTuple(args,"iiO;y,x,ch or int", &y, &x, &chobj)) - return NULL; - use_xy = TRUE; - break; - case 4: - if (!PyArg_ParseTuple(args,"iiOl;y,x,ch or int, attr", - &y, &x, &chobj, &lattr)) - return NULL; - attr = lattr; - use_xy = TRUE; - break; - default: - PyErr_SetString(PyExc_TypeError, "addch requires 1 to 4 arguments"); - return NULL; - } + if (!attr_group) + attr = A_NORMAL; #ifdef HAVE_NCURSESW - type = PyCurses_ConvertToCchar_t(self, chobj, &ch, &wch); + type = PyCurses_ConvertToCchar_t(cwself, ch, &cch, &wch); if (type == 2) { funcname = "add_wch"; wch.attr = attr; - if (use_xy == TRUE) - rtn = mvwadd_wch(self->win,y,x, &wch); + if (coordinates_group) + rtn = mvwadd_wch(cwself->win,y,x, &wch); else { - rtn = wadd_wch(self->win, &wch); + rtn = wadd_wch(cwself->win, &wch); } } else #else - type = PyCurses_ConvertToCchar_t(self, chobj, &ch); + type = PyCurses_ConvertToCchar_t(cwself, chobj, &cch); #endif if (type == 1) { funcname = "addch"; - if (use_xy == TRUE) - rtn = mvwaddch(self->win,y,x, ch | attr); + if (coordinates_group) + rtn = mvwaddch(cwself->win,y,x, cch | attr); else { - rtn = waddch(self->win, ch | attr); + rtn = waddch(cwself->win, cch | attr); } } else { @@ -1950,7 +2015,7 @@ static PyMethodDef PyCursesWindow_Methods[] = { - {"addch", (PyCFunction)PyCursesWindow_AddCh, METH_VARARGS}, + CURSES_WINDOW_ADDCH_METHODDEF, {"addnstr", (PyCFunction)PyCursesWindow_AddNStr, METH_VARARGS}, {"addstr", (PyCFunction)PyCursesWindow_AddStr, METH_VARARGS}, {"attroff", (PyCFunction)PyCursesWindow_AttrOff, METH_VARARGS}, diff -r 026b9f5dd97f Modules/_dbmmodule.c --- a/Modules/_dbmmodule.c Fri Dec 07 10:18:22 2012 -0800 +++ b/Modules/_dbmmodule.c Fri Dec 07 12:25:36 2012 -0800 @@ -44,7 +44,7 @@ static PyObject *DbmError; static PyObject * -newdbmobject(char *file, int flags, int mode) +newdbmobject(const char *file, int flags, int mode) { dbmobject *dp; @@ -361,16 +361,69 @@ /* ----------------------------------------------------------------- */ +/*[clinic] +dbm.open -> mapping +basename=dbmopen + + const char *filename; + The filename to open. + + const char *flags="r"; + How to open the file. "r" for reading, "w" for writing, etc. + + int mode=0666; + default=0o666 + If creating a new file, the mode bits for the new file + (e.g. os.O_RDWR). + +Returns a database object. + +[clinic]*/ + +PyDoc_STRVAR(dbmopen__doc__, +"dbm.open(filename[, flags=\'r\'[, mode=0o666]]) -> mapping\n" +"\n" +" filename\n" +" The filename to open.\n" +"\n" +" flags\n" +" How to open the file. \"r\" for reading, \"w\" for writing, etc.\n" +"\n" +" mode\n" +" If creating a new file, the mode bits for the new file\n" +" (e.g. os.O_RDWR).\n" +"\n" +"Returns a database object.\n" +"\n"); + +#define DBMOPEN_METHODDEF \ + {"open", (PyCFunction)dbmopen, METH_VARARGS | METH_KEYWORDS, dbmopen__doc__} + static PyObject * -dbmopen(PyObject *self, PyObject *args) +dbmopen_impl(PyObject *self, const char *filename, const char *flags, int mode); + +static PyObject * +dbmopen(PyObject *self, PyObject *args, PyObject *kwargs) { - char *name; - char *flags = "r"; + const char *filename; + const char *flags = "r"; + int mode = 0666; + static char *_keywords[] = {"filename", "flags", "mode", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "s|si:open", _keywords, + &filename, &flags, &mode)) + return NULL; + + return dbmopen_impl(self, filename, flags, mode); +} + +static PyObject * +dbmopen_impl(PyObject *self, const char *filename, const char *flags, int mode) +/*[clinic end:e9fbed95f33e75b6955f97a464ce47252def3624]*/ +{ int iflags; - int mode = 0666; - if ( !PyArg_ParseTuple(args, "s|si:open", &name, &flags, &mode) ) - return NULL; if ( strcmp(flags, "r") == 0 ) iflags = O_RDONLY; else if ( strcmp(flags, "w") == 0 ) @@ -386,13 +439,11 @@ "arg 2 to open should be 'r', 'w', 'c', or 'n'"); return NULL; } - return newdbmobject(name, iflags, mode); + return newdbmobject(filename, iflags, mode); } static PyMethodDef dbmmodule_methods[] = { - { "open", (PyCFunction)dbmopen, METH_VARARGS, - "open(path[, flag[, mode]]) -> mapping\n" - "Return a database object."}, + DBMOPEN_METHODDEF, { 0, 0 }, }; diff -r 026b9f5dd97f Modules/posixmodule.c --- a/Modules/posixmodule.c Fri Dec 07 10:18:22 2012 -0800 +++ b/Modules/posixmodule.c Fri Dec 07 12:25:36 2012 -0800 @@ -462,7 +462,7 @@ * path.function_name * If non-NULL, path_converter will use that as the name * of the function in error messages. - * (If path.argument_name is NULL it omits the function name.) + * (If path.function_name is NULL it omits the function name.) * path.argument_name * If non-NULL, path_converter will use that as the name * of the parameter in error messages. @@ -526,6 +526,9 @@ PyObject *cleanup; } path_t; +#define PATH_T_INITIALIZE(function_name, nullable, allow_fd) \ + {function_name, NULL, nullable, allow_fd, NULL, NULL, 0, 0, NULL, NULL} + static void path_cleanup(path_t *path) { if (path->cleanup) { @@ -2086,44 +2089,101 @@ return _pystat_fromstructstat(&st); } -PyDoc_STRVAR(posix_stat__doc__, -"stat(path, *, dir_fd=None, follow_symlinks=True) -> stat result\n\n\ -Perform a stat system call on the given path.\n\ -\n\ -path may be specified as either a string or as an open file descriptor.\n\ -\n\ -If dir_fd is not None, it should be a file descriptor open to a directory,\n\ - and path should be relative; path will then be relative to that directory.\n\ - dir_fd may not be supported on your platform; if it is unavailable, using\n\ - it will raise a NotImplementedError.\n\ -If follow_symlinks is False, and the last element of the path is a symbolic\n\ - link, stat will examine the symbolic link itself instead of the file the\n\ - link points to.\n\ -It is an error to use dir_fd or follow_symlinks when specifying path as\n\ - an open file descriptor."); - -static PyObject * -posix_stat(PyObject *self, PyObject *args, PyObject *kwargs) -{ - static char *keywords[] = {"path", "dir_fd", "follow_symlinks", NULL}; - path_t path; +#ifdef HAVE_FSTATAT + #define OS_STAT_DIR_FD_CONVERTER dir_fd_converter +#else + #define OS_STAT_DIR_FD_CONVERTER dir_fd_unavailable +#endif + +/*[clinic] +os.stat -> stat result + + path_t path = PATH_T_INITIALIZE("stat", 0, 1); + required + converter=path_converter + Path to be examined; can be string, bytes, or open-file-descriptor int. + + int dir_fd = DEFAULT_DIR_FD; + default=None + converter=OS_STAT_DIR_FD_CONVERTER + keyword-only + If not None, it should be a file descriptor open to a directory, + and path should be a relative string; path will then be relative to + that directory. + + int follow_symlinks = 1; + default=True + types=bool + If False, and the last element of the path is a symbolic link, + stat will examine the symbolic link itself instead of the file + the link points to. + +Perform a stat system call on the given path. + + {arguments} + +dir_fd and follow_symlinks may not be implemented + on your platform. If they are unavailable, using them will raise a + NotImplementedError. + +It's an error to use dir_fd or follow_symlinks when specifying path as + an open file descriptor. + +[clinic]*/ + +PyDoc_STRVAR(os_stat__doc__, +"os.stat(path, *[, dir_fd=None[, follow_symlinks=True]]) -> stat result\n" +"\n" +"Perform a stat system call on the given path.\n" +"\n" +" path\n" +" Path to be examined; can be string, bytes, or open-file-descriptor int.\n" +"\n" +" dir_fd\n" +" If not None, it should be a file descriptor open to a directory,\n" +" and path should be a relative string; path will then be relative to\n" +" that directory.\n" +"\n" +" follow_symlinks\n" +" If False, and the last element of the path is a symbolic link,\n" +" stat will examine the symbolic link itself instead of the file\n" +" the link points to.\n" +"\n" +"dir_fd and follow_symlinks may not be implemented\n" +" on your platform. If they are unavailable, using them will raise a\n" +" NotImplementedError.\n" +"\n" +"It's an error to use dir_fd or follow_symlinks when specifying path as\n" +" an open file descriptor.\n" +"\n"); + +#define OS_STAT_METHODDEF \ + {"stat", (PyCFunction)os_stat, METH_VARARGS | METH_KEYWORDS, os_stat__doc__} + +static PyObject * +os_stat_impl(PyObject *self, path_t path, int dir_fd, int follow_symlinks); + +static PyObject * +os_stat(PyObject *self, PyObject *args, PyObject *kwargs) +{ + path_t path = PATH_T_INITIALIZE("stat", 0, 1); int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - PyObject *return_value; - - memset(&path, 0, sizeof(path)); - path.function_name = "stat"; - path.allow_fd = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|$O&p:stat", keywords, - path_converter, &path, -#ifdef HAVE_FSTATAT - dir_fd_converter, &dir_fd, -#else - dir_fd_unavailable, &dir_fd, -#endif - &follow_symlinks)) - return NULL; - return_value = posix_do_stat("stat", &path, dir_fd, follow_symlinks); + static char *_keywords[] = {"path", "dir_fd", "follow_symlinks", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O&|$O&p:stat", _keywords, + path_converter, &path, OS_STAT_DIR_FD_CONVERTER, &dir_fd, &follow_symlinks)) + return NULL; + + return os_stat_impl(self, path, dir_fd, follow_symlinks); +} + +static PyObject * +os_stat_impl(PyObject *self, path_t path, int dir_fd, int follow_symlinks) +/*[clinic end:f3e6b328245207c240825782d908d3ff3a9c11c0]*/ +{ + PyObject *return_value = posix_do_stat("stat", &path, dir_fd, follow_symlinks); path_cleanup(&path); return return_value; } @@ -2153,44 +2213,129 @@ #endif )) return NULL; - return_value = posix_do_stat("stat", &path, dir_fd, follow_symlinks); + return_value = posix_do_stat("lstat", &path, dir_fd, follow_symlinks); path_cleanup(&path); return return_value; } -PyDoc_STRVAR(posix_access__doc__, -"access(path, mode, *, dir_fd=None, effective_ids=False,\ - follow_symlinks=True)\n\n\ -Use the real uid/gid to test for access to a path. Returns True if granted,\n\ -False otherwise.\n\ -\n\ -If dir_fd is not None, it should be a file descriptor open to a directory,\n\ - and path should be relative; path will then be relative to that directory.\n\ -If effective_ids is True, access will use the effective uid/gid instead of\n\ - the real uid/gid.\n\ -If follow_symlinks is False, and the last element of the path is a symbolic\n\ - link, access will examine the symbolic link itself instead of the file the\n\ - link points to.\n\ -dir_fd, effective_ids, and follow_symlinks may not be implemented\n\ - on your platform. If they are unavailable, using them will raise a\n\ - NotImplementedError.\n\ -\n\ -Note that most operations will use the effective uid/gid, therefore this\n\ - routine can be used in a suid/sgid environment to test if the invoking user\n\ - has the specified access to the path.\n\ -The mode argument can be F_OK to test existence, or the inclusive-OR\n\ - of R_OK, W_OK, and X_OK."); - -static PyObject * -posix_access(PyObject *self, PyObject *args, PyObject *kwargs) -{ - static char *keywords[] = {"path", "mode", "dir_fd", "effective_ids", - "follow_symlinks", NULL}; - path_t path; + +#ifdef HAVE_FACCESSAT + #define OS_ACCESS_DIR_FD_CONVERTER dir_fd_converter +#else + #define OS_ACCESS_DIR_FD_CONVERTER dir_fd_unavailable +#endif +/*[clinic] +os.access + + path_t path = PATH_T_INITIALIZE("access", 0, 1); + required + converter=path_converter + Path to be tested; can be string, bytes, or open-file-descriptor int. + + int mode; + Operating-system mode bitfield. Can be F_OK to test existence, + or the inclusive-OR of R_OK, W_OK, and X_OK. + + int dir_fd = DEFAULT_DIR_FD; + default=None + converter=OS_ACCESS_DIR_FD_CONVERTER + keyword-only + If not None, it should be a file descriptor open to a directory, + and path should be relative; path will then be relative to that + directory. + + int effective_ids = 0; + default=False + types=bool + If True, access will use the effective uid/gid instead of + the real uid/gid. + + int follow_symlinks = 1; + default=True + types=bool + If False, and the last element of the path is a symbolic link, + access will examine the symbolic link itself instead of the file + the link points to. + +Use the real uid/gid to test for access to a path. +Returns True if granted, False otherwise. + + {arguments} + +dir_fd, effective_ids, and follow_symlinks may not be implemented + on your platform. If they are unavailable, using them will raise a + NotImplementedError. + +Note that most operations will use the effective uid/gid, therefore this + routine can be used in a suid/sgid environment to test if the invoking user + has the specified access to the path. + +[clinic]*/ + +PyDoc_STRVAR(os_access__doc__, +"os.access(path, mode, *[, dir_fd=None[, effective_ids=False[, follow_symlinks=True]]])\n" +"\n" +"Use the real uid/gid to test for access to a path.\n" +"Returns True if granted, False otherwise.\n" +"\n" +" path\n" +" Path to be tested; can be string, bytes, or open-file-descriptor int.\n" +"\n" +" mode\n" +" Operating-system mode bitfield. Can be F_OK to test existence,\n" +" or the inclusive-OR of R_OK, W_OK, and X_OK.\n" +"\n" +" dir_fd\n" +" If not None, it should be a file descriptor open to a directory,\n" +" and path should be relative; path will then be relative to that\n" +" directory.\n" +"\n" +" effective_ids\n" +" If True, access will use the effective uid/gid instead of\n" +" the real uid/gid.\n" +"\n" +" follow_symlinks\n" +" If False, and the last element of the path is a symbolic link,\n" +" access will examine the symbolic link itself instead of the file\n" +" the link points to.\n" +"\n" +"dir_fd, effective_ids, and follow_symlinks may not be implemented\n" +" on your platform. If they are unavailable, using them will raise a\n" +" NotImplementedError.\n" +"\n" +"Note that most operations will use the effective uid/gid, therefore this\n" +" routine can be used in a suid/sgid environment to test if the invoking user\n" +" has the specified access to the path.\n" +"\n"); + +#define OS_ACCESS_METHODDEF \ + {"access", (PyCFunction)os_access, METH_VARARGS | METH_KEYWORDS, os_access__doc__} + +static PyObject * +os_access_impl(PyObject *self, path_t path, int mode, int dir_fd, int effective_ids, int follow_symlinks); + +static PyObject * +os_access(PyObject *self, PyObject *args, PyObject *kwargs) +{ + path_t path = PATH_T_INITIALIZE("access", 0, 1); int mode; int dir_fd = DEFAULT_DIR_FD; int effective_ids = 0; int follow_symlinks = 1; + static char *_keywords[] = {"path", "mode", "dir_fd", "effective_ids", "follow_symlinks", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O&i|$O&pp:access", _keywords, + path_converter, &path, &mode, OS_ACCESS_DIR_FD_CONVERTER, &dir_fd, &effective_ids, &follow_symlinks)) + return NULL; + + return os_access_impl(self, path, mode, dir_fd, effective_ids, follow_symlinks); +} + +static PyObject * +os_access_impl(PyObject *self, path_t path, int mode, int dir_fd, int effective_ids, int follow_symlinks) +/*[clinic end:fc0d31ac4162e46966d2493cae19efdea34f583e]*/ +{ PyObject *return_value = NULL; #ifdef MS_WINDOWS @@ -2199,17 +2344,6 @@ int result; #endif - memset(&path, 0, sizeof(path)); - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&i|$O&pp:access", keywords, - path_converter, &path, &mode, -#ifdef HAVE_FACCESSAT - dir_fd_converter, &dir_fd, -#else - dir_fd_unavailable, &dir_fd, -#endif - &effective_ids, &follow_symlinks)) - return NULL; - #ifndef HAVE_FACCESSAT if (follow_symlinks_specified("access", follow_symlinks)) goto exit; @@ -10200,9 +10334,9 @@ static PyMethodDef posix_methods[] = { - {"access", (PyCFunction)posix_access, - METH_VARARGS | METH_KEYWORDS, - posix_access__doc__}, + + OS_ACCESS_METHODDEF, + #ifdef HAVE_TTYNAME {"ttyname", posix_ttyname, METH_VARARGS, posix_ttyname__doc__}, #endif @@ -10291,9 +10425,7 @@ {"rmdir", (PyCFunction)posix_rmdir, METH_VARARGS | METH_KEYWORDS, posix_rmdir__doc__}, - {"stat", (PyCFunction)posix_stat, - METH_VARARGS | METH_KEYWORDS, - posix_stat__doc__}, + OS_STAT_METHODDEF, {"stat_float_times", stat_float_times, METH_VARARGS, stat_float_times__doc__}, #if defined(HAVE_SYMLINK) {"symlink", (PyCFunction)posix_symlink, diff -r 026b9f5dd97f Modules/zlibmodule.c --- a/Modules/zlibmodule.c Fri Dec 07 10:18:22 2012 -0800 +++ b/Modules/zlibmodule.c Fri Dec 07 12:25:36 2012 -0800 @@ -604,38 +604,80 @@ return 0; } -PyDoc_STRVAR(decomp_decompress__doc__, -"decompress(data, max_length) -- Return a string containing the decompressed\n" -"version of the data.\n" +/*[clinic] + +zlib.decompress + Py_buffer data + types="bytes bytearray buffer" + The binary data to decompress. + int max_length = 0 + The maximum allowable length of the decompressed data. + Unconsumed input data will be stored in + the unconsumed_tail attribute. + +Return a string containing the decompressed version of the data. + +After calling this function, some of the input data may still be stored in +internal buffers for later processing. +Call the flush() method to clear these buffers. +[clinic]*/ + +PyDoc_STRVAR(zlib_decompress__doc__, +"zlib.decompress(data[, max_length=0])\n" +"\n" +" data\n" +" The binary data to decompress.\n" +" max_length\n" +" The maximum allowable length of the decompressed data.\n" +" Unconsumed input data will be stored in\n" +" the unconsumed_tail attribute.\n" +"\n" +"Return a string containing the decompressed version of the data.\n" "\n" "After calling this function, some of the input data may still be stored in\n" "internal buffers for later processing.\n" -"Call the flush() method to clear these buffers.\n" -"If the max_length parameter is specified then the return value will be\n" -"no longer than max_length. Unconsumed input data will be stored in\n" -"the unconsumed_tail attribute."); +"Call the flush() method to clear these buffers.\n"); + +#define ZLIB_DECOMPRESS_METHODDEF \ + {"decompress", (PyCFunction)zlib_decompress, METH_VARARGS | METH_KEYWORDS, zlib_decompress__doc__} static PyObject * -PyZlib_objdecompress(compobject *self, PyObject *args) +zlib_decompress_impl(PyObject *self, Py_buffer data, int max_length); + +static PyObject * +zlib_decompress(PyObject *self, PyObject *args, PyObject *kwargs) { - int err, max_length = 0; + Py_buffer data; + int max_length = 0; + static char *_keywords[] = {"data", "max_length", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "y*|i:decompress", _keywords, + &data, &max_length)) + return NULL; + + return zlib_decompress_impl(self, data, max_length); +} + +static PyObject * +zlib_decompress_impl(PyObject *self, Py_buffer data, int max_length) +/*[clinic end:34d5ee75a33ac2c40a8c72f44fdfa5cc13bfd010]*/ +{ + compobject *zself = (compobject *)self; + int err; unsigned int inplen; Py_ssize_t old_length, length = DEFAULTALLOC; PyObject *RetVal = NULL; - Py_buffer pinput; Byte *input; unsigned long start_total_out; - if (!PyArg_ParseTuple(args, "y*|i:decompress", &pinput, - &max_length)) - return NULL; - if (pinput.len > UINT_MAX) { + if (data.len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "Size does not fit in an unsigned int"); goto error_outer; } - input = pinput.buf; - inplen = pinput.len; + input = data.buf; + inplen = data.len; if (max_length < 0) { PyErr_SetString(PyExc_ValueError, "max_length must be greater than zero"); @@ -648,43 +690,43 @@ if (!(RetVal = PyBytes_FromStringAndSize(NULL, length))) goto error_outer; - ENTER_ZLIB(self); + ENTER_ZLIB(zself); - start_total_out = self->zst.total_out; - self->zst.avail_in = inplen; - self->zst.next_in = input; - self->zst.avail_out = length; - self->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal); + start_total_out = zself->zst.total_out; + zself->zst.avail_in = inplen; + zself->zst.next_in = input; + zself->zst.avail_out = length; + zself->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal); Py_BEGIN_ALLOW_THREADS - err = inflate(&(self->zst), Z_SYNC_FLUSH); + err = inflate(&(zself->zst), Z_SYNC_FLUSH); Py_END_ALLOW_THREADS - if (err == Z_NEED_DICT && self->zdict != NULL) { + if (err == Z_NEED_DICT && zself->zdict != NULL) { Py_buffer zdict_buf; - if (PyObject_GetBuffer(self->zdict, &zdict_buf, PyBUF_SIMPLE) == -1) { + if (PyObject_GetBuffer(zself->zdict, &zdict_buf, PyBUF_SIMPLE) == -1) { Py_DECREF(RetVal); RetVal = NULL; goto error; } - err = inflateSetDictionary(&(self->zst), zdict_buf.buf, zdict_buf.len); + err = inflateSetDictionary(&(zself->zst), zdict_buf.buf, zdict_buf.len); PyBuffer_Release(&zdict_buf); if (err != Z_OK) { - zlib_error(self->zst, err, "while decompressing data"); + zlib_error(zself->zst, err, "while decompressing data"); Py_DECREF(RetVal); RetVal = NULL; goto error; } /* Repeat the call to inflate. */ Py_BEGIN_ALLOW_THREADS - err = inflate(&(self->zst), Z_SYNC_FLUSH); + err = inflate(&(zself->zst), Z_SYNC_FLUSH); Py_END_ALLOW_THREADS } /* While Z_OK and the output buffer is full, there might be more output. So extend the output buffer and try again. */ - while (err == Z_OK && self->zst.avail_out == 0) { + while (err == Z_OK && zself->zst.avail_out == 0) { /* If max_length set, don't continue decompressing if we've already reached the limit. */ @@ -702,12 +744,12 @@ RetVal = NULL; goto error; } - self->zst.next_out = + zself->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal) + old_length; - self->zst.avail_out = length - old_length; + zself->zst.avail_out = length - old_length; Py_BEGIN_ALLOW_THREADS - err = inflate(&(self->zst), Z_SYNC_FLUSH); + err = inflate(&(zself->zst), Z_SYNC_FLUSH); Py_END_ALLOW_THREADS } @@ -720,27 +762,27 @@ if (err == Z_STREAM_END) { /* This is the logical place to call inflateEnd, but the old behaviour of only calling it on flush() is preserved. */ - self->eof = 1; + zself->eof = 1; } else if (err != Z_OK && err != Z_BUF_ERROR) { /* We will only get Z_BUF_ERROR if the output buffer was full but there wasn't more output when we tried again, so it is not an error condition. */ - zlib_error(self->zst, err, "while decompressing data"); + zlib_error(zself->zst, err, "while decompressing data"); Py_DECREF(RetVal); RetVal = NULL; goto error; } - if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) { + if (_PyBytes_Resize(&RetVal, zself->zst.total_out - start_total_out) < 0) { Py_DECREF(RetVal); RetVal = NULL; } error: - LEAVE_ZLIB(self); + LEAVE_ZLIB(zself); error_outer: - PyBuffer_Release(&pinput); + PyBuffer_Release(&data); return RetVal; } @@ -1047,8 +1089,7 @@ static PyMethodDef Decomp_methods[] = { - {"decompress", (binaryfunc)PyZlib_objdecompress, METH_VARARGS, - decomp_decompress__doc__}, + ZLIB_DECOMPRESS_METHODDEF, {"flush", (binaryfunc)PyZlib_unflush, METH_VARARGS, decomp_flush__doc__}, #ifdef HAVE_ZLIB_COPY diff -r 026b9f5dd97f clinic.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/clinic.py Fri Dec 07 12:25:36 2012 -0800 @@ -0,0 +1,1210 @@ +#!/usr/bin/env python3 +# +# Argument Clinic (clinic.py) +# +# DSL for argument preprocessing +# +# Copyright 2012 by Larry Hastings. +# Licensed to the PSF under a Contributor Agreement. +# + + +import atexit +import builtins +import collections +import hashlib +import os +import re +import shlex +import sys +import tempfile +import textwrap + + +tempfiles = [] +def remove_tempfiles(): + for filename in tempfiles: + try: + os.unlink(filename) + except BaseException: + pass + +atexit.register(remove_tempfiles) + + + +NULL = object() + +class Unspecified: + def __repr__(self): + return '' + +unspecified = Unspecified() + + + +# This "type_map" thing is just for the Clinic prototype. +# It maps Clinic type declarations to PyArg_Parse format units. +# I expect to replace it before this ever ships in a Python release, +# hopefully something based on a new more granular +# argument parsing library. +# +# Syntax of type_map: +# = +# Declarations are one per line. +# +# is first and can contain spaces. (In truth, anything the +# parser doesn't recognize is appended to the C type.) +# +# options are in the middle, and can be any / all / none of the following: +# +# bitwise +# converter +# encoding +# immutable +# length +# nullable +# zeroes +# ( one or more python types, between parentheses, separated by whitespace ) +# +# with the exception of stuff inside parentheses, all these options +# are boolean flags and default to false. +# ("converter" and "encoding" just indicate whether or not the user +# specified a converter / encoding. for the purposes of the type map +# we don't care what the converter or encoding actually is.) +# +# The line ends with +# = +# which is the format unit used with PyArg_ParseTuple +# +# Note: for convenience's sakes, "const char *" by default +# maps to strings, not bytes. +# +# +# Unimplemented so far: +# w* + +type_map = """ +PyObject * = O +void * converter = O& + +Py_UNICODE * = u +Py_UNICODE * length = u# +Py_UNICODE * nullable = Z +Py_UNICODE * length nullable = Z# + +const char * = s +const char * nullable = z + +char * encoding = es +char * encoding (str bytes bytearray) = et +char * encoding length = es# +char * encoding length (str bytes bytearray) = et# + +const char * (bytes) = y +Py_buffer (bytes bytearray buffer) = y* +const char * length (bytes buffer) = y# + +Py_buffer * zeroes (str bytes bytearray buffer) = s* +Py_buffer * nullable zeroes (str bytes bytearray buffer) = z* +const char * length zeroes (str bytes buffer) = s# +const char * length nullable zeroes (str bytes buffer) = z# + +PyUnicodeObject * = U + +PyBytesObject * = S + +PyBytesArrayObject * = Y + +unsigned char = b +unsigned char bitwise = B +short = h +short int = h +unsigned short bitwise = H +unsigned short int bitwise = H +int = i +unsigned int bitwise = I +long = l +long int = l +unsigned long bitwise = k +unsigned long int bitwise = k +PY_LONG_LONG = L +unsigned PY_LONG_LONG bitwise = K +Py_ssize_t = n +char (bytes bytearray) = c +char = C +int (bool) = p + +float = f +double = d +Py_complex * = D + +""" + +format_unit_map = {} + +options = collections.OrderedDict() +for key in "bitwise converter encoding immutable length nullable zeroes".split(): + options[key] = False + +def get_format_unit(c_type, + bitwise=False, converter=False, encoding=False, immutable=False, + length=False, nullable=False, zeroes=False, py_types=()): + if converter: + c_type = "void *" + keys = (c_type, bitwise, converter, encoding, immutable, length, nullable, zeroes, py_types) + keystr = "c_type, bitwise, converter, encoding, immutable, length, nullable, zeroes, py_types".split() + d = format_unit_map + for key, name in zip(keys, keystr): + before_d = d + d = d.get(key, None) + if not d: + break + return d + +def set_format_unit(format_unit, c_type, + bitwise=False, converter=False, encoding=False, immutable=False, + length=False, nullable=False, zeroes=False, py_types=()): + if converter: + c_type = "void *" + keys, last_key = (c_type, bitwise, converter, encoding, immutable, length, nullable, zeroes), py_types + d = format_unit_map + for key in keys: + d = d.setdefault(key, {}) + d[last_key] = format_unit + +# conversion: + +for line in type_map.split("\n"): + line = line.strip() + if not line: + continue + fields, _, format_unit = line.rpartition('=') + format_unit = format_unit.strip() + bitwise = converter = encoding = immutable = False + length = nullable = zeroes = False + c_type = [] + types = [] + in_types = False + line_options = options.copy() + for field in fields.split(): + if not field: + continue + + if in_types: + if ')' in field: + field, _, should_be_empty = field.partition(')') + assert not should_be_empty, "don't abuse the lame type_map parser" + in_types = False + if field: + types.append(field) + continue + if field in line_options: + if line_options[field]: + sys.exit("Error: Can't specify " + field + " twice.\n\t" + line) + line_options[field] = True + continue + if field.startswith('('): + assert not in_types + field = field[1:].strip() + if ')' in field: + field, _, should_be_empty = field.partition(')') + assert not should_be_empty, "don't abuse the lame type_map parser" + else: + in_types = True + if field: + types.append(field) + continue + c_type.append(field) + assert c_type, "no c_type specified for type_map initializer line" + c_type = " ".join(c_type) + types = tuple(sorted(types)) + splat = tuple(line_options.values()) + (types,) + set_format_unit(format_unit, c_type, *splat) + + +value_to_cvalue_map = { + NULL: "NULL", + None: "Py_None", +} + +def value_to_cvalue(v): + return value_to_cvalue_map.get(v, v) + + + +class CVariable: + """ + An individual C variable required by an Argument. + (Some Arguments require more than one.) + """ + name = "" + c_type = "" + default = unspecified + + def __init__(self, c_type, name, default=unspecified): + self.c_type = c_type + self.name = name + self.default = default + + def impl_argument(self): + """ + The C code to pass this argument in to the impl function. + Returns an iterable. + """ + assert self.name and self.c_type + return (self.name,) + + def keyword(self): + """ + The value to add to the 'keywords' array for PyArg_ParseTupleAndKeywords. + Returns an iterable. + """ + assert self.name and self.c_type + return ('"' + self.name + '"',) + + def parse_argument(self): + """ + The C code to pass this argument in to PyArg_ParseTuple &c. + Returns an iterable. + """ + assert self.name and self.c_type + return ("&" + self.name,) + + def prototype(self): + """ + The C code to define this variable as an argument to a function. + Returns an iterable. + """ + assert self.name and self.c_type + prototype = [self.c_type] + if not self.c_type.endswith('*'): + prototype.append(" ") + prototype.append(self.name) + return ("".join(prototype),) + + def declaration(self): + """ + The C statement to declare this variable. + Returns an iterable. + """ + declaration = list(self.prototype()) + if self.default != unspecified: + declaration.append(" = {}".format(self.default)) + declaration.append(";") + return ("".join(declaration),) + + +class CHardCodedParserArgument(CVariable): + def __init__(self, argument): + self._parse_argument = (argument,) + + def parse_argument(self): + return self._parse_argument + + def impl_argument(self): + return () + + prototype = declaration = keyword = impl_argument + + +class CGroupBoolean(CVariable): + def __init__(self, name): + super().__init__("int", name, 0) + + def parse_argument(self): + return () + + keyword = parse_argument + + +class ArgumentBase: + + def __init__(self): + self.variables = [] + self.flags = {} + + def flag(self, key, default=False): + return self.flags.get(key, default) + + def required(): + return (self.default == unspecified) or self.flag('required') + + def _finalize(self): + pass + + def _from_variables(self, name): + self._finalize() + l = [] + for v in self.variables: + l.extend(getattr(v, name)()) + return l + + def is_optional(self): + return False + + def impl_argument(self): + return self._from_variables("impl_argument") + + def keyword(self): + return self._from_variables("keyword") + + def parse_argument(self): + return self._from_variables("parse_argument") + + def prototype(self): + return self._from_variables("prototype") + + def declaration(self): + return self._from_variables("declaration") + + def format_unit(self): + return '' + + + +class Optional(ArgumentBase): + def __init__(self): + super().__init__() + self.name = '* optional *' + + def format_unit(self): + return "|" + + def docstring(self): + return () + + +class KeywordOnly(ArgumentBase): + def __init__(self): + super().__init__() + self.name = '* keyword-only *' + + def format_unit(self): + return "$" + + def docstring(self): + return ('*',) + + +class Argument(ArgumentBase): + def __init__(self, c_type, name, line, default=unspecified): + super().__init__() + self.c_type = c_type + self.name = name + self.line = line + self.default = default + self.docstrings = [] + + def is_optional(self): + return (self.default != unspecified) and not self.flag('required') + + def _finalize(self): + if self.variables: + return + + if self.flag('group-first'): + group = self.flag('group', None) + if group is not None: + group_boolean = CGroupBoolean(group + "_group") + self.variables.append(group_boolean) + + converter = self.flag('converter', '') + encoding = self.flag('encoding', '') + assert not (converter and encoding) + if encoding: + v = CHardCodedParserArgument('"' + encoding + '"') + self.variables.append(v) + elif converter: + v = CHardCodedParserArgument(converter) + self.variables.append(v) + + v = CVariable(self.c_type, self.name, self.default) + self.variables.append(v) + + if self.flag('length'): + length_name = self.name + "_length" + length = CVariable('Py_ssize_t', length_name, -1) + self.variables.append(length) + + def docstring(self): + l = [self.name] + if self.is_optional(): + l.append('=') + value = self.flag('default', unspecified) + if value is unspecified: + try: + value = repr(eval(self.default)) + except ValueError: + value = self.default + l.append(value) + return l + + def format_unit(self): + """ + Render argument "format unit" in PyArg_ParseTupleAndKeywords format string. + Returns iterable. + """ + types = tuple(sorted(self.flag('types', '').strip().split())) + flag_tuple = tuple((bool(self.flag(flag)) for flag in options)) + (types,) + format_unit = get_format_unit(self.c_type, *flag_tuple) + if not format_unit: + sys.exit("Sorry, I don't have a format unit for " + repr(self.line)) + return format_unit + + +re_argument = re.compile(r"^(?P[A-Za-z0-9_\s]+)((?P\s*\*\s*)|(\s+))(?P\w+)(\s*=\s*(?P\S.*))?$") + +class FauxPrint: + def __init__(self): + self.output = [] + + def print(self, *args, sep=' ', end='\n', file=sys.stdout, flush=False): + line = sep.join(str(a) for a in args) + end + invalid_lines = ( + Clinic.python_begin_string, + Clinic.dsl_begin_string, + ) + if line.startswith(invalid_lines): + error = ["Invalid line in output! You can't use any of the following"] + error.append("in the output of a Python section:") + error.extend([" " + invalid for invalid in invalid_lines]) + sys.exit("\n".join(error)) + if file == sys.stdout: + self.output.append(line) + else: + file.write(line) + if flush: + file.flush() + + +class Clinic: + def __init__(self, *, verify=True): + self.state = self.state_reset + self.verify = verify + + def rewrite(self, filename): + self.read(filename) + self.write(filename) + + def read(self, filename): + self.filename = filename + + self.input = collections.deque() + self.output = [] + with open(self.filename, "rt") as f: + self.input = collections.deque(f.readlines()) + + def write(self, filename): + if not self.output: + self._rewrite() + + self.filename = filename + + + body = "\n".join(self.output) + # always end .c files with a blank line + if not body.endswith("\n"): + body += "\n" + body = body.encode("utf-8") + directory = os.path.dirname(filename) + f, name = tempfile.mkstemp(text=True, dir=directory) + tempfiles.append(name) + os.write(f, body) + os.close(f) + os.rename(name, self.filename) + tempfiles.remove(name) + + python_begin_string = "/*[python]" + python_end_string = "[python]*/" + python_end_output_prefix = "/*[python end:" + python_end_output_suffix = "]*/" + + dsl_begin_string = "/*[clinic]" + dsl_end_string = "[clinic]*/" + dsl_end_output_prefix = "/*[clinic end:" + dsl_end_output_suffix = "]*/" + + def _rewrite(self): + while self.input: + line = self.input.popleft().rstrip() + # Always append the line to the output here. + # When we throw away lines from the input, + # we do so from a separate input-reading spot. + self.output.append(line) + self.state(line) + assert self.state == self.state_idle + self.next(self.state_reset) + + @staticmethod + def ignore_line(line): + # ignore comment-only lines + if line.lstrip().startswith('#'): + return True + + # Ignore empty lines too + # (but not in docstring sections!) + if not line.strip(): + return True + + return False + + @staticmethod + def tab_nanny(line): + if '\t' in line: + sys.exit('Error: you used a tab in the DSL.\n\t' + repr(line)) + + @staticmethod + def calculate_indent(line): + return len(line) - len(line.strip()) + + def flag(self, name, default=False): + return self.flags.get(name, default) + + def found_dsl_end(self, line): + if line == self.dsl_end_string: + self.next(self.state_dsl_end, line) + return True + return False + + @staticmethod + def parse_flags_line(d, line): + line = line.strip() + if not line: + return + + if '#' in line: + # we want to support # line comments on flag lines. + # but there could be a # inside a quoted string! what to do? + # + # shlex.split is smart about complaining about unterminated quotes. + # so: try clipping the line at every instance of a '#' + # and use the first one where shlex.split doesn't complain. + subline = "" + for segment in line.split('#'): + subline += segment + try: + attempt = shlex.split(subline) + line = subline + break + except ValueError: + subline += '#' + continue + for field in shlex.split(line): + if '=' in field: + name, _, value = field.partition('=') + name = name.strip() + value = value.strip() + else: + name = field + value = True + if name in d: + sys.exit("Error: Can't specify " + repr(name) + " twice.\n\t" + line) + d[name] = value + + def next(self, state, line=None): + # print(self.state.__name__, "->", state.__name__, ", line=", line) + self.state = state + if line is not None: + self.state(line) + + def find_terminator(self, terminator_prefix, begin_string): + # read ahead until we find the end marker + saved = [] + + while self.input: + line = self.input.popleft().rstrip() + if line == begin_string: + break + if line.startswith(terminator_prefix): + if self.verify: + # verify checksum + checksum_and_comment_end = line[len(terminator_prefix):] + stored_checksum, comment, eol = checksum_and_comment_end.partition(']*/') + if eol or not comment: + sys.exit('Invalid marker line found:\n ' + line + '\nGiving up.') + data = "\n".join(saved).encode('utf-8') + b'\n' + checksum = hashlib.sha1(data).hexdigest() + if checksum != stored_checksum: + sys.exit('Checksum doesn\'t match for marker line:\n ' + line + '\nGiving up.') + return + saved.append(line) + + # if we got here, we didn't find a valid end marker for this block. + # restore the entire remainder of the file. + # (we'll automatically insert the marker with the template) + self.input.extendleft(reversed(saved)) + + def state_reset(self, line=None): + # dsl + self.module_name = self.name = None + self.return_annotation = None + self.flags = {} + self.indent = self.argument_indent = None + self.arguments = [] + self.is_optional = self.is_keyword_only = False + self.docstring = [] + self.groups = set() + + # python + self.code = [] + + self.next(self.state_idle, line) + + def state_idle(self, line): + if line == self.dsl_begin_string: + self.next(self.state_dsl_start) + if line == self.python_begin_string: + self.next(self.state_python_start) + + def state_python_start(self, line): + self.next(self.state_python_body, line) + + def state_python_body(self, line): + if line == self.python_end_string: + self.next(self.state_python_end, line) + return + self.code.append(line) + + def state_python_end(self, line): + self.find_terminator(self.python_end_output_prefix, self.python_begin_string) + code = "\n".join(self.code) + + faux = FauxPrint() + + old_print = builtins.print + builtins.print = faux.print + exec(code) + builtins.print = old_print + + output = "".join(faux.output) + if output[-1] == '\n': + output = output[:-1] + self.output.append(output) + data = output.encode('utf-8') + b'\n' + line = "".join([ + self.python_end_output_prefix, + hashlib.sha1(data).hexdigest(), + self.python_end_output_suffix, + ]) + self.output.append(line) + + self.next(self.state_reset) + + def state_dsl_start(self, line): + self.next(self.state_modulename_name, line) + + def state_modulename_name(self, line): + # looking for declaration at left column + # line should be + # modulename.fnname -> return annotation + # everything after here ^ + # is optional. + + if self.ignore_line(line) or self.found_dsl_end(line): + return + assert not line[0].isspace(), "first line of clinic dsl should not start with whitespace:" + repr(line) + self.tab_nanny(line) + line, arrow, return_annotation = line.partition('->') + self.return_annotation = return_annotation.strip() + self.module_name, period, self.name = line.strip().partition('.') + self.next(self.state_function_flags) + + def state_function_flags(self, line): + if self.ignore_line(line) or self.found_dsl_end(line): + return + + if line[0].isspace(): + return self.next(self.state_argument, line) + + self.tab_nanny(line) + self.parse_flags_line(self.flags, line) + + + # in the arguments section. + # the rule is: + # * the first line must be an argument declaration. + # * this first line establishes the indent for arguments. + # * thenceforth: + # * one or more lines immediately after an argument declaration + # at the same indent level are flags lines. + # * lines indented further are docstrings for the previous arg. + # * the section ends if we hit the function docstring. + # * the first line of the function docstring must be at column 0. + # * it's illegal to have something outdented but not at column 0 + # (until you get to the function docstring). + + def state_argument(self, line): + if self.ignore_line(line) or self.found_dsl_end(line): + return + + self.tab_nanny(line) + + if self.argument_indent == None: + self.argument_indent = self.calculate_indent(line) + + # be nice: allow ";" but clip it here + original_line = line + if line.endswith(';'): + line = line[:-1].strip() + + match = re_argument.match(line) + d = match.groupdict() + ctype = d['ctype'].strip() + if d.get('splat'): + ctype += ' *' + aname = d['name'] + # can't use d.get for the default argument, + # it's always set to None if it wasn't found. + # (thanks, re module!) + default = d["default"] + if default == None: + default = unspecified + elif self.flag('positional-only'): + sys.exit("Error: Can't use default values for arguments with positional-only functions (see " + self.module_name + "." + self.name + ")") + + a = Argument(ctype, aname, line, default) + a.original_line = original_line + self.arguments.append(a) + + self.next(self.state_argument_flags) + + def state_argument_flags(self, line): + if self.ignore_line(line) or self.found_dsl_end(line): + return + + self.tab_nanny(line) + + assert self.arguments + a = self.arguments[-1] + + indent = self.calculate_indent(line) + if indent == self.argument_indent: + self.parse_flags_line(a.flags, line) + return + + # transition to next state, + # this is a good time for a whole bunch o' processing + keyword_only = a.flag('keyword-only') + optional = (a.default != unspecified) and (not a.flag('required')) + + if keyword_only and not optional: + sys.exit("Error: You can't have a parameter which is keyword-only and required:\n\t" + a.original_line) + + if optional: + if not self.is_optional: + self.is_optional = True + self.arguments[-1] = Optional() + self.arguments.append(a) + elif self.is_optional: + sys.exit("Error: required parameter:\n\t" + a.original_line + "\nspecified after optional parameter.") + + if keyword_only and not self.is_keyword_only: + self.is_keyword_only = True + self.arguments[-1] = KeywordOnly() + self.arguments.append(a) + + group = a.flag('group') + if group and group not in self.groups: + self.groups.add(group) + a.flags['group-first'] = True + + return self.next(self.state_arguments_docstring, line) + + + def state_arguments_docstring(self, line): + if line.lstrip().startswith('#'): + return + + self.tab_nanny(line) + + assert self.arguments + a = self.arguments[-1] + + if not line.strip(): + a.docstrings.append('') + return + + indent = self.calculate_indent(line) + + if indent == self.argument_indent: + return self.next(self.state_argument, line) + elif indent < self.argument_indent: + assert not indent, "The first line of the function docstring must be at column 0." + return self.next(self.state_docstring, line) + + a.docstrings.append(line[self.argument_indent:]) + + def state_docstring(self, line): + if line.lstrip().startswith('#'): + return + + if self.found_dsl_end(line): + return + + self.tab_nanny(line) + + self.docstring.append(line) + + + def argument_list(self, name): + l = [] + for a in self.arguments: + l.extend(getattr(a, name)()) + return l + + def state_dsl_end(self, line): + self.find_terminator(self.dsl_end_output_prefix, self.dsl_begin_string) + + # used by .format(**locals()) + module_name = self.module_name + name = self.name + # for names which are themselves dotted + last_name = name.rpartition('.')[2] + + is_optional = False + is_keyword_only = False + using_groups = False + + # ensure that optional and keyword-only stuff is sane + for a in self.arguments: + + if isinstance(a, Optional): + assert not is_optional + is_optional = True + continue + + if isinstance(a, KeywordOnly): + assert is_optional + assert not is_keyword_only + is_keyword_only = True + continue + + using_groups = using_groups or a.flag('group') + + if using_groups and not self.flag('positional-only'): + sys.exit("Error: can't use groups without positional-only (see " + repr(module_name + '.' + name) + ")") + + module_name_upper = module_name.upper() + name_upper = name.upper() + prototype_name = "{module_name}_{name}".format(**locals()).replace('.', '_') + prototype_name = self.flag('basename') or prototype_name + prototype_name_upper = prototype_name.upper() + + debug = builtins.print + + faux = FauxPrint() + print = faux.print + + ## + ## compute stuff that's the same whether or not we're doing positional-only + ## + + # compute square brackets for docstring first line + if self.flag('positional-only'): + # count how many arguments there are in each group + groups = [] + counter = collections.Counter() + group_to_arguments = {} + left_groups = [] + right_groups = [] + append_to = left_groups + last = unspecified + for a in self.arguments: + group = a.flag('group', None) + if group != last: + last = group + if group in groups: + sys.exit("Non-contiguous use of group " + repr(group)) + groups.append(group) + group_to_arguments[group] = [] + if group is None: + append_to = right_groups + append_to.append(group) + counter[group] += 1 + group_to_arguments[group].append(a) + left_groups = list(reversed(left_groups)) + + right_bracket_counts = {None:0} + # skip None group (which is first in right_groups) + for group_list in (left_groups, right_groups[1:]): + right_bracket_counts.update({group: i+1 for (i, group) in enumerate(group_list)}) + last = unspecified + for a in self.arguments: + group = a.flag('group', None) + a.right_bracket_count = right_bracket_counts[group] + else: + # not positional-only + # all optional arguments are ungrouped and separately right-bracketed + # but: all keyword-only arguments don't get brackets! + i = 0 + is_keyword_only = False + for a in self.arguments: + if 0 and isinstance(a, KeywordOnly): + is_keyword_only = True + i = 0 + elif not is_keyword_only and a.is_optional(): + i += 1 + a.right_bracket_count = i + + # the docstring! + docstring_first_line = 'PyDoc_STRVAR({prototype_name}__doc__,\n"{module_name}.{name}('.format(**locals()) + + first_line = [] + docstring_arguments = [] + right_bracket_count = 0 + def fix_right_bracket_count(desired): + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + for a in self.arguments: + s = "".join(a.docstring()) + if s: + s = fix_right_bracket_count(a.right_bracket_count) + s + for old, new in ( + ('"', '\\"'), + ("'", "\\'"), + ): + s = s.replace(old, new) + first_line.append(s) + if isinstance(a, Argument): + docstring_arguments.append(a.name) + for line in a.docstrings: + docstring_arguments.append((' ' + line).rstrip()) + + docstring_first_line += ', '.join(first_line) + fix_right_bracket_count(0) + ')' + + # now fix up the places where the brackets look wrong + # and quote the strings + for old, new in ( + (', ]', ',] '), + (', [', '[, '), + ): + docstring_first_line = docstring_first_line.replace(old, new) + + if self.return_annotation: + annotation = self.return_annotation + docstring_first_line += ' -> ' + annotation + docstring_first_line += '\\n"\n"\\n"\n' + + # the contents of the {arguments} string for the docstring + arguments_docstring = "\n".join(docstring_arguments) + + docstring_end = '\n'.join(self.docstring) + if docstring_end: + if '{arguments}' not in docstring_end: + docstring_end = ' {arguments}\n\n' + docstring_end + arguments_docstring = arguments_docstring.rstrip() + final = [] + for line in docstring_end.split('\n'): + indent, is_arguments_line, _ = line.partition('{arguments}') + if is_arguments_line: + final.append(textwrap.indent(arguments_docstring, indent)) + else: + final.append(line) + docstring_end = "\n".join(final) + docstring_end = '"' + docstring_end.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n"\n"') + '\\n"' + + docstring = docstring_first_line + docstring_end + ');' + + print() + print(docstring) + print() + + # the methoddef define! + meth_keywords = " | METH_KEYWORDS" if not self.flag('positional-only') else '' + print('#define {prototype_name_upper}_METHODDEF \\'.format(**locals())) + print(' {{"{last_name}", (PyCFunction){prototype_name}, METH_VARARGS{meth_keywords}, {prototype_name}__doc__}}'.format(**locals())) + print() + + # the prototype! + prototype_arguments = ", ".join(self.argument_list('prototype')) + impl = prototype_name + "_impl" + impl_prototype = "static PyObject *\n{impl}(PyObject *self, {prototype_arguments})".format(**locals()) + print(impl_prototype + ';') + print() + + print("static PyObject *") + kwargs = ", PyObject *kwargs" if not self.flag('positional-only') else '' + print("{prototype_name}(PyObject *self, PyObject *args{kwargs})".format(**locals())) + print("{") + + # declarations, with defaults + for a in self.arguments: + for line in a.declaration(): + print(" ", line) + + if not self.flag('positional-only'): + # not positional only, aka uses PyArg_ParseTupleAndKeywords + + format_units = '' + parse_arguments = [] + for a in self.arguments: + format_units += a.format_unit() + parse_arguments.extend(a.parse_argument()) + format_units += ':' + last_name + + # "keywords" array for PyArg_ParseTupleAndKeywords + quoted_names = self.argument_list('keyword') + quoted_names.append("NULL") + keywords = " static char *_keywords[] = {" + ", ".join(quoted_names) + "};" + print(keywords) + + print() + + print(" if (!PyArg_ParseTupleAndKeywords(args, kwargs,") + print(' "' + format_units + '", _keywords,') + print(' ' + ', '.join(parse_arguments) + '))') + print(' return NULL;') + + else: + # positional only, grouped, optional arguments! + # can be optional on the left or right. + # here's an example: + # + # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] + # + # Here group D are required, and all other groups are optional. + # (Group D's "group" is actually None.) + # We can figure out which sets of arguments we have based on + # how many arguments are in the tuple. + # + # Note that you need to count up on both sides. For example, + # you could have groups C and D, or C & D & E, or C & D & E & F. + # + # What if the number of arguments leads us to an ambiguous result? + # Why, then, you're a *loser* my dear! Clinic will exit with + # an error message. + + left_bundles = [()] + l = [] + for group in left_groups: + l.append(group) + left_bundles.append(tuple(l)) + + count_min = 10000 + count_max = -1 + print() + print(" switch (PyTuple_Size(args)) {") + bundle_counts = {} + for b in left_bundles: + l = list(b) + for group in right_groups: + l.append(group) + bundle = tuple(l) + count = sum(counter[g] for g in bundle) + count_min = min(count_min, count) + count_max = max(count_max, count) + if count in bundle_counts: + sys.exit("Can't process arguments for " + self.module_name + "." + self.name + ": two bundles with same argument counts, " + repr(bundle) + " and " + repr(bundle_counts[count])) + bundle_counts[bundle] = count + + print(" case " + str(count) + ":") + print(' if (!PyArg_ParseTuple(args, "', end='') + parse_arguments = [] + group_booleans = [] + for group in bundle: + if group is not None: + group_booleans.append(group + "_group") + for a in group_to_arguments[group]: + print(a.format_unit(), end='') + parse_arguments.extend(a.parse_argument()) + print(':' + last_name + '", ', end='') + print(", ".join(parse_arguments) + "))") + print(" return NULL;") + for group in bundle: + if group is not None: + print(" " + group + "_group = 1;") + print(" break;") + print(" default:") + print(' PyErr_SetString(PyExc_TypeError, "' + self.name + ' requires', count_min, 'to', count_max, 'arguments");') + print(' return NULL;') + print(" }") + + impl_arguments = [] + for a in self.arguments: + impl_arguments.extend(a.impl_argument()) + print() + print(' return ' + impl + '(self, ' + ', '.join(impl_arguments) + ');') + + print("}") + print() + + # the header for the impl! + print(impl_prototype) + + # write output! + text = "".join(faux.output) + data = text.encode('ascii') + checksum = hashlib.sha1(data).hexdigest() + self.output.append("".join([ + text, + self.dsl_end_output_prefix, + checksum, + self.dsl_end_output_suffix, + ])) + + self.next(self.state_reset) + + def dead_code(self): + argument_declarations = argument_declarations + + + pyarg_format_string += ":" + name + + + if 0: # else: + # the first line of the docstring + + + parse_code = keywords + """ + if (!PyArg_ParseTupleAndKeywords(args, kwargs, "{pyarg_format_string}", + _keywords{pyarg_arguments})) + return NULL;""" + + template = """ +{docstring} +{impl_prototype}; + +#define {module_name_upper}_{name_upper}_METHODDEF \\ + {{"{name}", (PyCFunction){module_name}_{name}, METH_VARARGS | METH_KEYWORDS, {module_name}_{name}__doc__}} + +static PyObject * +{module_name}_{name}(PyObject *self, PyObject *args, PyObject *kwargs) +{{ + {argument_declarations} +""" + parse_code + """ + return {module_name}_{name}_impl(self{invocation_arguments}); +}} + +{impl_prototype} +""" + + text = template.format(**locals()) + data = text.encode('utf-8') + + +def main(argv): + import argparse + cmdline = argparse.ArgumentParser() + cmdline.add_argument("-f", "--force", action='store_true') + cmdline.add_argument("-o", "--output", type=str) + cmdline.add_argument("filename", type=str, nargs="+") + ns = cmdline.parse_args(argv) + if ns.output and (len(ns.filename) > 1): + sys.exit("Error: can't use -o with multiple filenames") + for filename in ns.filename: + clinic = Clinic(verify=not (ns.force or ns.output)) + clinic.read(filename) + clinic.write(ns.output or filename) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff -r 026b9f5dd97f clinic.txt --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/clinic.txt Fri Dec 07 12:25:36 2012 -0800 @@ -0,0 +1,248 @@ +The Argument Clinic DSL + +Larry Hastings +larry@hastings.org +November 2012 + + +Summary +======= + +Section | Example +---------------------+---------------------- +Clinic DSL start | /*[clinic] +Function declaration | module.function_name -> return_annotation +Function flags | flag flag2 flag3=value +Argument declaration | type name = default +Argument flags | flag flag2 flag3=value +Argument docstring | Lorem ipsum dolor sit amet, consectetur + | adipisicing elit, sed do eiusmod tempor +Function docstring | Lorem ipsum dolor sit amet, consectetur adipisicing + | elit, sed do eiusmod tempor incididunt ut labore et +Clinic DSL end | [clinic]*/ +Clinic output | ... +Clinic output end | /*[clinic end output:]*/ + + +In General +========== + + All lines support # as a line comment *except* docstrings. + Blank lines are always ignored. + + Like Python itself, whitespace is significant in the Argument Clinic DSL. + The first line of the "function" section is the declaration; all subsequent + lines at the same indent are function flags. Once you indent, the first + line is an argument declaration; subsequent lines at that indent are + argument flags. Indent one more time for the lines of the argument + docstring. Finally, outdent back to the same level as the function + declaration for the function docstring. + +Function Declaration +==================== + + The return annotation is optional. + +Argument Declaration +==================== + + The "type" is a C type. If it's a pointer type, you must specify + a single space between the type and the '*', and zero spaces between + the '*' and the name. (e.g. "PyObject *foo", not "PyObject* foo") + + The "name" must be a legal C identifier. + + The "default" is a Python value. Default values are optional; + if not specified you must omit the equals sign too. Arguments + which don't have a default are required. The default value is + dynamically assigned, "live" in the generated C code. + + It's explicitly permitted to end the argument declaration line + with a semicolon. + +Flags +===== + + "Flags" are like "make -D" arguments. They're unordered. Flags lines + are parsed much like the shell (specifically, shlex.split). You can + have as many flag lines as you like. Specifying a flag twice is + currently an error. + + Supported flags for functions: + + basename : The basename to use for the generated C functions. + By default this is the name of the function from + the DSL, only with periods replaced by underscores. + + positional-only : This function only supports positional arguments, + not keyword arguments. See "Functions With Positional- + Only Arguments" section below. + + Supported flags for arguments: + + bitwise : If the Python integer passed in is signed, copy the + bits directly even if it is negative. Only valid + for unsigned integer types. + + converter : Backwards-compatibility support for argument "converter" + functions. The value should be the name of the converter + function in C. Only valid when the type of the argument + is void *. + + default : The Python value to use in place of the parameter's actual + default in Python contexts. Specifically, when specified, + this value will be used for the parameter's default in the + docstring, and in the Signature. (TBD: If the string is a + valid Python expression, renderable into a Python value + using eval(), then the result of eval() on it will be used + as the default in the Signature.) Ignored if there is no + default. + + encoding : Encoding to use when encoding a Unicode string to a char *. + Only valid when the type of the argument is char *. + + group= : This argument is part of a group of options that must either + all be specified or none specified. Arguments in the same + "group" must be contiguous. The value of the group flag + is the name used for the group. Only valid for functions + marked "positional-only"; see "Functions With Positional- + Only Arguments" section below. + + immutable : Only accept immutable values. + + keyword-only : This parameter (and all following parameters) is keyword-only. + Keyword-only parameters must also be optional. Not valid for + positional-only functions. + + length : This is an iterable type, and we also want its length. The + DSL will generate a second Py_ssize_t variable; its name will + be this argument's name appended with "_length". + + nullable : None is a legal value for this parameter. If None is + supplied on the Python side, this C parameter will be NULL. + Only valid for pointer types. + + required : Normally any parameter that has a default value is automatically + optional. A parameter that has "required" set will be + considered required (non-optional) even if it has a default + value. The generated documentation will also not show any + default value. + + types : Space-separated list of acceptable Python types for this + object. There are also four special-case types which + represent Python protocols: + buffer mapping number sequence + + zeroes : This parameter is a string type, and its value should be + allowed to have embedded zeroes. Not valid for all + varieties of string arguments. + + +Output +====== + +Argument Clinic writes its output in-line in the file, immediately after +the section of Clinic code. For "python" sections, the output is +everything printed using builtins.print. For "clinic" sections, the +output is valid C code, including: + * a #define providing the correct methoddef structure for the function + * a prototype for the "impl" function--this is what you'll write to + implement this function + * a function that handles all argument processing, which calls your + "impl" function + * the definition line of the "impl" function + * and a comment indicating the end of output. + +The intention is that you will write the body of your impl function +immediately after the output--as in, you write a left-curly-brace +immediately after the end-of-output comment and write the implementation +of the builtin in the body there. (It's a bit strange, but oddly +convenient.) + +Argument Clinic will define the arguments of the impl function for you. +The function will take the "self" parameter passed in originally, all +the arguments you define, and possibly some extra arguments ("length" +arguments, and/or "group" arguments, see next section). + +Argument Clinic also writes a checksum for the output section. This +is a valuable safety feature: if you modify the output by hand, Clinic +will notice that the checksum doesn't match, and will refuse to +overwrite the file. (You can force Clinic to overwrite with the "-f" +command-line parameter; Clinic will also ignore the checksums when +using the "-o" command-line parameter.) + + +Functions With Positional-Only Arguments +======================================== + +A significant fraction of Python builtins implemented in C use the +older positional-only API for processing arguments (PyArg_ParseTuple). +In some instances, these builtins parse their arguments differently +based on how many arguments were passed in. This can provide some +bewildering flexibility: there may be groups of optional arguments, +which must either all be specified or none specified. And occasionally +these groups are on the *left*! (For example: curses.window.addch().) + +Argument Clinic supports these legacy use-cases with a slightly altered +mode of operation. First, set the flag "positional-only" on the entire +function. Then, for every group of arguments that is collectively +optional, add a "group=" flag with a unique string to all the arguments +in that group. Note that these groups can be to the left or right of +any required arguments! + +The impl function generated by Clinic will add an extra parameter for +every group, "int _group". This parameter will be nonzero if +the group was specified on this call, and zero if it was not. + +Note that when operating in this mode, you cannot specify defaults to +parameters. You can simulate defaults by putting default arguments +in individual groups--but generally speaking it's better to simply not +use "positional-only" where it isn't absolutely necessary. (TBD: It +might be possible to relax this restriction. But adding default +arguments into the mix of groups would seemingly make calculating which +groups are active a good deal harder.) + +Also, note that it's possible--even easy--to specify a set of groups +to a function such that there are several valid mappings from the number +of arguments to a valid set of groups. If this happens, Clinic will exit +with an error message. This should not be a problem, as positional-only +operation is only intended for legacy use cases, and all the legacy +functions using this quirky behavior should have unambiguous mappings. + + +Notes / TBD +=========== + +* Guido proposed having the "function docstring" be hand-written inline, + in the middle of the output, something like this: + + /*[clinic] + ... prototype and arguments (including docstrings for arguments) goes here + [clinic]*/ + ... some output ... + /*[clinic docstring start]*/ + ... hand-edited function docstring goes here <-- you edit this by hand! + /*[clinic docstring end]*/ + ... more output + /*[clinic output end]*/ + + I tried it this way and don't like it--I think it's clumsy. I prefer that + everything you write goes in one place, rather than having an island of + hand-edited stuff in the middle of the DSL output. + +* Do we need to support tuple unpacking? (The "(OOO)" style format string.) + Boy I sure hope not. + +* What about Python functions that take no arguments? This syntax doesn't + provide for that. Perhaps a lone indented "None" should mean "no arguments"? + +* This approach removes some dynamism / flexibility. With the existing + syntax one could theoretically pass in different encodings at runtime for + the "es"/"et" format unit. AFAICT CPython doesn't do this itself, however + it's possible external users might do this. (Trivia: there are no uses + of "es" exercised by regrtest, and all the uses of "et" exercised are in + socketmodule.c, except for one in _ssl.c. They're all static, using "idna".) + +* Right now the "basename" flag on a function changes the #define methoddef name + too. Should it, or should the #define'd methoddef name always be + {module_name}_{function_name} ?