diff -r c65fcedc511c Modules/_cursesmodule.c --- a/Modules/_cursesmodule.c Thu Feb 21 23:17:34 2013 +0200 +++ b/Modules/_cursesmodule.c Sat Feb 23 16:43:37 2013 -0800 @@ -553,68 +553,133 @@ /* Addch, Addstr, Addnstr */ +/*[clinic] +curses.window.addch +positional-only + + int x; + group=coordinates + X-coordinate. + + int y; + group=coordinates + Y-coordinate. + + PyObject *ch; + Character to add. + + long attr; + group=attr + Attributes for the character. + +Paint character ch at (y, x) with attributes attr, +overwriting any character previously painter at that location. +By default, the character position and attributes are the +current settings for the window object. +[clinic]*/ + +PyDoc_STRVAR(curses_window_addch__doc__, +"curses.window.addch([x, y,] ch[, attr])\n" +"\n" +" x\n" +" X-coordinate.\n" +"\n" +" y\n" +" Y-coordinate.\n" +"\n" +" ch\n" +" Character to add.\n" +"\n" +" attr\n" +" Attributes for the character.\n" +"\n" +"Paint character ch at (y, x) with attributes attr,\n" +"overwriting any character previously painter at that location.\n" +"By default, the character position and attributes are the\n" +"current settings for the window object.\n"); + +#define CURSES_WINDOW_ADDCH_METHODDEF \ + {"addch", (PyCFunction)curses_window_addch, METH_VARARGS, curses_window_addch__doc__} + static PyObject * -PyCursesWindow_AddCh(PyCursesWindowObject *self, PyObject *args) +curses_window_addch_impl(PyObject *self, int coordinates_group, int x, int y, PyObject *ch, int attr_group, long attr); + +static PyObject * +curses_window_addch(PyObject *self, PyObject *args) { - int rtn, x, y, use_xy = FALSE; - PyObject *chobj; + int coordinates_group = 0; + int x; + int y; + PyObject *ch; + int attr_group = 0; + long attr; + + switch (PyTuple_Size(args)) { + case 1: + if (!PyArg_ParseTuple(args, "O:addch", &ch)) + return NULL; + break; + case 2: + if (!PyArg_ParseTuple(args, "Ol:addch", &ch, &attr)) + return NULL; + attr_group = 1; + break; + case 3: + if (!PyArg_ParseTuple(args, "iiO:addch", &x, &y, &ch)) + return NULL; + coordinates_group = 1; + break; + case 4: + if (!PyArg_ParseTuple(args, "iiOl:addch", &x, &y, &ch, &attr)) + return NULL; + coordinates_group = 1; + attr_group = 1; + break; + default: + PyErr_SetString(PyExc_TypeError, "window.addch requires 1 to 4 arguments"); + return NULL; + } + + return curses_window_addch_impl(self, coordinates_group, x, y, ch, attr_group, attr); +} + +static PyObject * +curses_window_addch_impl(PyObject *self, int coordinates_group, int x, int y, PyObject *ch, int attr_group, long attr) +/*[clinic end:355e50ee04fc6675476cbe9baa1a06ff14450c03]*/ +{ + PyCursesWindowObject *cwself = (PyCursesWindowObject *)self; + int rtn; int type; - chtype ch; + chtype cch; #ifdef HAVE_NCURSESW cchar_t wch; #endif - attr_t attr = A_NORMAL; - long lattr; const char *funcname; - switch (PyTuple_Size(args)) { - case 1: - if (!PyArg_ParseTuple(args, "O;ch or int", &chobj)) - return NULL; - break; - case 2: - if (!PyArg_ParseTuple(args, "Ol;ch or int,attr", &chobj, &lattr)) - return NULL; - attr = lattr; - break; - case 3: - if (!PyArg_ParseTuple(args,"iiO;y,x,ch or int", &y, &x, &chobj)) - return NULL; - use_xy = TRUE; - break; - case 4: - if (!PyArg_ParseTuple(args,"iiOl;y,x,ch or int, attr", - &y, &x, &chobj, &lattr)) - return NULL; - attr = lattr; - use_xy = TRUE; - break; - default: - PyErr_SetString(PyExc_TypeError, "addch requires 1 to 4 arguments"); - return NULL; - } + if (!attr_group) + attr = A_NORMAL; #ifdef HAVE_NCURSESW - type = PyCurses_ConvertToCchar_t(self, chobj, &ch, &wch); + type = PyCurses_ConvertToCchar_t(cwself, ch, &cch, &wch); if (type == 2) { funcname = "add_wch"; wch.attr = attr; - if (use_xy == TRUE) - rtn = mvwadd_wch(self->win,y,x, &wch); + if (coordinates_group) + rtn = mvwadd_wch(cwself->win,y,x, &wch); else { - rtn = wadd_wch(self->win, &wch); + rtn = wadd_wch(cwself->win, &wch); } } else #else - type = PyCurses_ConvertToCchar_t(self, chobj, &ch); + type = PyCurses_ConvertToCchar_t(cwself, chobj, &cch); #endif if (type == 1) { funcname = "addch"; - if (use_xy == TRUE) - rtn = mvwaddch(self->win,y,x, ch | attr); + if (coordinates_group) + rtn = mvwaddch(cwself->win,y,x, cch | attr); else { - rtn = waddch(self->win, ch | attr); + rtn = waddch(cwself->win, cch | attr); } } else { @@ -1950,7 +2015,7 @@ static PyMethodDef PyCursesWindow_Methods[] = { - {"addch", (PyCFunction)PyCursesWindow_AddCh, METH_VARARGS}, + CURSES_WINDOW_ADDCH_METHODDEF, {"addnstr", (PyCFunction)PyCursesWindow_AddNStr, METH_VARARGS}, {"addstr", (PyCFunction)PyCursesWindow_AddStr, METH_VARARGS}, {"attroff", (PyCFunction)PyCursesWindow_AttrOff, METH_VARARGS}, diff -r c65fcedc511c Modules/_dbmmodule.c --- a/Modules/_dbmmodule.c Thu Feb 21 23:17:34 2013 +0200 +++ b/Modules/_dbmmodule.c Sat Feb 23 16:43:37 2013 -0800 @@ -44,7 +44,7 @@ static PyObject *DbmError; static PyObject * -newdbmobject(char *file, int flags, int mode) +newdbmobject(const char *file, int flags, int mode) { dbmobject *dp; @@ -361,16 +361,69 @@ /* ----------------------------------------------------------------- */ +/*[clinic] +dbm.open -> mapping +basename=dbmopen + + const char *filename; + The filename to open. + + const char *flags="r"; + How to open the file. "r" for reading, "w" for writing, etc. + + int mode=0666; + default=0o666 + If creating a new file, the mode bits for the new file + (e.g. os.O_RDWR). + +Returns a database object. + +[clinic]*/ + +PyDoc_STRVAR(dbmopen__doc__, +"dbm.open(filename[, flags=\'r\'[, mode=0o666]]) -> mapping\n" +"\n" +" filename\n" +" The filename to open.\n" +"\n" +" flags\n" +" How to open the file. \"r\" for reading, \"w\" for writing, etc.\n" +"\n" +" mode\n" +" If creating a new file, the mode bits for the new file\n" +" (e.g. os.O_RDWR).\n" +"\n" +"Returns a database object.\n" +"\n"); + +#define DBMOPEN_METHODDEF \ + {"open", (PyCFunction)dbmopen, METH_VARARGS | METH_KEYWORDS, dbmopen__doc__} + static PyObject * -dbmopen(PyObject *self, PyObject *args) +dbmopen_impl(PyObject *self, const char *filename, const char *flags, int mode); + +static PyObject * +dbmopen(PyObject *self, PyObject *args, PyObject *kwargs) { - char *name; - char *flags = "r"; + const char *filename; + const char *flags = "r"; + int mode = 0666; + static char *_keywords[] = {"filename", "flags", "mode", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "s|si:open", _keywords, + &filename, &flags, &mode)) + return NULL; + + return dbmopen_impl(self, filename, flags, mode); +} + +static PyObject * +dbmopen_impl(PyObject *self, const char *filename, const char *flags, int mode) +/*[clinic end:e9fbed95f33e75b6955f97a464ce47252def3624]*/ +{ int iflags; - int mode = 0666; - if ( !PyArg_ParseTuple(args, "s|si:open", &name, &flags, &mode) ) - return NULL; if ( strcmp(flags, "r") == 0 ) iflags = O_RDONLY; else if ( strcmp(flags, "w") == 0 ) @@ -386,13 +439,11 @@ "arg 2 to open should be 'r', 'w', 'c', or 'n'"); return NULL; } - return newdbmobject(name, iflags, mode); + return newdbmobject(filename, iflags, mode); } static PyMethodDef dbmmodule_methods[] = { - { "open", (PyCFunction)dbmopen, METH_VARARGS, - "open(path[, flag[, mode]]) -> mapping\n" - "Return a database object."}, + DBMOPEN_METHODDEF, { 0, 0 }, }; diff -r c65fcedc511c Modules/posixmodule.c --- a/Modules/posixmodule.c Thu Feb 21 23:17:34 2013 +0200 +++ b/Modules/posixmodule.c Sat Feb 23 16:43:37 2013 -0800 @@ -8,6 +8,8 @@ of the compiler used. Different compilers define their own feature test macro, e.g. '__BORLANDC__' or '_MSC_VER'. */ + + #ifdef __APPLE__ /* * Step 1 of support for weak-linking a number of symbols existing on @@ -595,7 +597,7 @@ * path.function_name * If non-NULL, path_converter will use that as the name * of the function in error messages. - * (If path.argument_name is NULL it omits the function name.) + * (If path.function_name is NULL it omits the function name.) * path.argument_name * If non-NULL, path_converter will use that as the name * of the parameter in error messages. @@ -659,6 +661,9 @@ PyObject *cleanup; } path_t; +#define PATH_T_INITIALIZE(function_name, nullable, allow_fd) \ + {function_name, NULL, nullable, allow_fd, NULL, NULL, 0, 0, NULL, NULL} + static void path_cleanup(path_t *path) { if (path->cleanup) { @@ -1196,6 +1201,7 @@ #endif } + /* POSIX generic methods */ static PyObject * @@ -2230,45 +2236,138 @@ return _pystat_fromstructstat(&st); } -PyDoc_STRVAR(posix_stat__doc__, -"stat(path, *, dir_fd=None, follow_symlinks=True) -> stat result\n\n\ -Perform a stat system call on the given path.\n\ -\n\ -path may be specified as either a string or as an open file descriptor.\n\ -\n\ -If dir_fd is not None, it should be a file descriptor open to a directory,\n\ - and path should be relative; path will then be relative to that directory.\n\ - dir_fd may not be supported on your platform; if it is unavailable, using\n\ - it will raise a NotImplementedError.\n\ -If follow_symlinks is False, and the last element of the path is a symbolic\n\ - link, stat will examine the symbolic link itself instead of the file the\n\ - link points to.\n\ -It is an error to use dir_fd or follow_symlinks when specifying path as\n\ - an open file descriptor."); - -static PyObject * -posix_stat(PyObject *self, PyObject *args, PyObject *kwargs) -{ - static char *keywords[] = {"path", "dir_fd", "follow_symlinks", NULL}; +#ifdef HAVE_FSTATAT + #define OS_STAT_DIR_FD_CONVERTER dir_fd_converter +#else + #define OS_STAT_DIR_FD_CONVERTER dir_fd_unavailable +#endif + + +/*[python] +@type_map_register +class CVariable_path_t(CVariable): + + type = "path_t" + + def __init__(self, name, default=unspecified): + super().__init__(name, default) + self.flags['converter'] = "path_converter" + + def default(self, s): + assert s is unspecified + + def get(s): + return str(int(bool(self.flags.get(s)))) + + return "".join([ + "PATH_T_INITIALIZE(", + '"', + clinic.name, + '", ', + get('nullable'), + ", ", + get('allow_fd'), + ")" + ]) + + def cleanup(self): + return ["path_cleanup(&" + self.name + ");",] + +[python]*/ + +/*[python end:adc83b19e793491b1c6ea0fd8b46cd9f32e592fc]*/ + +/*[clinic] +os.stat -> stat result + path_t path; + allow_fd + Path to be examined; can be string, bytes, or open-file-descriptor int. + + int dir_fd = DEFAULT_DIR_FD; + default=None + converter=OS_STAT_DIR_FD_CONVERTER + keyword-only + If not None, it should be a file descriptor open to a directory, + and path should be a relative string; path will then be relative to + that directory. + + int follow_symlinks = 1; + default=True + types=bool + If False, and the last element of the path is a symbolic link, + stat will examine the symbolic link itself instead of the file + the link points to. + +Perform a stat system call on the given path. + +{arguments} + +dir_fd and follow_symlinks may not be implemented + on your platform. If they are unavailable, using them will raise a + NotImplementedError. + +It's an error to use dir_fd or follow_symlinks when specifying path as + an open file descriptor. + +[clinic]*/ + +PyDoc_STRVAR(os_stat__doc__, +"os.stat(path, *[, dir_fd=None[, follow_symlinks=True]]) -> stat result\n" +"\n" +"Perform a stat system call on the given path.\n" +"\n" +"path\n" +" Path to be examined; can be string, bytes, or open-file-descriptor int.\n" +"\n" +"dir_fd\n" +" If not None, it should be a file descriptor open to a directory,\n" +" and path should be a relative string; path will then be relative to\n" +" that directory.\n" +"\n" +"follow_symlinks\n" +" If False, and the last element of the path is a symbolic link,\n" +" stat will examine the symbolic link itself instead of the file\n" +" the link points to.\n" +"\n" +"dir_fd and follow_symlinks may not be implemented\n" +" on your platform. If they are unavailable, using them will raise a\n" +" NotImplementedError.\n" +"\n" +"It's an error to use dir_fd or follow_symlinks when specifying path as\n" +" an open file descriptor.\n" +"\n"); + +#define OS_STAT_METHODDEF \ + {"stat", (PyCFunction)os_stat, METH_VARARGS | METH_KEYWORDS, os_stat__doc__} + +static PyObject * +os_stat_impl(PyObject *self, path_t path, int dir_fd, int follow_symlinks); + +static PyObject * +os_stat(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *_return_value; + path_t path = PATH_T_INITIALIZE("stat", 0, 1); int dir_fd = DEFAULT_DIR_FD; int follow_symlinks = 1; - PyObject *return_value; - - memset(&path, 0, sizeof(path)); - path.function_name = "stat"; - path.allow_fd = 1; - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&|$O&p:stat", keywords, - path_converter, &path, -#ifdef HAVE_FSTATAT - dir_fd_converter, &dir_fd, -#else - dir_fd_unavailable, &dir_fd, -#endif - &follow_symlinks)) - return NULL; - return_value = posix_do_stat("stat", &path, dir_fd, follow_symlinks); + static char *_keywords[] = {"path", "dir_fd", "follow_symlinks", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O&|$O&p:stat", _keywords, + path_converter, &path, OS_STAT_DIR_FD_CONVERTER, &dir_fd, &follow_symlinks)) + return NULL; + + _return_value = os_stat_impl(self, path, dir_fd, follow_symlinks); path_cleanup(&path); + return _return_value; +} + +static PyObject * +os_stat_impl(PyObject *self, path_t path, int dir_fd, int follow_symlinks) +/*[clinic end:80f49f64e690056b1c1103ff331ad3e59366239f]*/ +{ + PyObject *return_value = posix_do_stat("stat", &path, dir_fd, follow_symlinks); return return_value; } @@ -2297,44 +2396,131 @@ #endif )) return NULL; - return_value = posix_do_stat("stat", &path, dir_fd, follow_symlinks); + return_value = posix_do_stat("lstat", &path, dir_fd, follow_symlinks); path_cleanup(&path); return return_value; } -PyDoc_STRVAR(posix_access__doc__, -"access(path, mode, *, dir_fd=None, effective_ids=False,\ - follow_symlinks=True)\n\n\ -Use the real uid/gid to test for access to a path. Returns True if granted,\n\ -False otherwise.\n\ -\n\ -If dir_fd is not None, it should be a file descriptor open to a directory,\n\ - and path should be relative; path will then be relative to that directory.\n\ -If effective_ids is True, access will use the effective uid/gid instead of\n\ - the real uid/gid.\n\ -If follow_symlinks is False, and the last element of the path is a symbolic\n\ - link, access will examine the symbolic link itself instead of the file the\n\ - link points to.\n\ -dir_fd, effective_ids, and follow_symlinks may not be implemented\n\ - on your platform. If they are unavailable, using them will raise a\n\ - NotImplementedError.\n\ -\n\ -Note that most operations will use the effective uid/gid, therefore this\n\ - routine can be used in a suid/sgid environment to test if the invoking user\n\ - has the specified access to the path.\n\ -The mode argument can be F_OK to test existence, or the inclusive-OR\n\ - of R_OK, W_OK, and X_OK."); - -static PyObject * -posix_access(PyObject *self, PyObject *args, PyObject *kwargs) -{ - static char *keywords[] = {"path", "mode", "dir_fd", "effective_ids", - "follow_symlinks", NULL}; + +#ifdef HAVE_FACCESSAT + #define OS_ACCESS_DIR_FD_CONVERTER dir_fd_converter +#else + #define OS_ACCESS_DIR_FD_CONVERTER dir_fd_unavailable +#endif +/*[clinic] +os.access + path_t path; + allow_fd + Path to be tested; can be string, bytes, or open-file-descriptor int. + + int mode; + Operating-system mode bitfield. Can be F_OK to test existence, + or the inclusive-OR of R_OK, W_OK, and X_OK. + + int dir_fd = DEFAULT_DIR_FD; + default=None + converter=OS_ACCESS_DIR_FD_CONVERTER + keyword-only + If not None, it should be a file descriptor open to a directory, + and path should be relative; path will then be relative to that + directory. + + int effective_ids = 0; + default=False + types=bool + If True, access will use the effective uid/gid instead of + the real uid/gid. + + int follow_symlinks = 1; + default=True + types=bool + If False, and the last element of the path is a symbolic link, + access will examine the symbolic link itself instead of the file + the link points to. + +Use the real uid/gid to test for access to a path. +Returns True if granted, False otherwise. + +{arguments} + +dir_fd, effective_ids, and follow_symlinks may not be implemented + on your platform. If they are unavailable, using them will raise a + NotImplementedError. + +Note that most operations will use the effective uid/gid, therefore this + routine can be used in a suid/sgid environment to test if the invoking user + has the specified access to the path. + +[clinic]*/ + +PyDoc_STRVAR(os_access__doc__, +"os.access(path, mode, *[, dir_fd=None[, effective_ids=False[, follow_symlinks=True]]])\n" +"\n" +"Use the real uid/gid to test for access to a path.\n" +"Returns True if granted, False otherwise.\n" +"\n" +"path\n" +" Path to be tested; can be string, bytes, or open-file-descriptor int.\n" +"\n" +"mode\n" +" Operating-system mode bitfield. Can be F_OK to test existence,\n" +" or the inclusive-OR of R_OK, W_OK, and X_OK.\n" +"\n" +"dir_fd\n" +" If not None, it should be a file descriptor open to a directory,\n" +" and path should be relative; path will then be relative to that\n" +" directory.\n" +"\n" +"effective_ids\n" +" If True, access will use the effective uid/gid instead of\n" +" the real uid/gid.\n" +"\n" +"follow_symlinks\n" +" If False, and the last element of the path is a symbolic link,\n" +" access will examine the symbolic link itself instead of the file\n" +" the link points to.\n" +"\n" +"dir_fd, effective_ids, and follow_symlinks may not be implemented\n" +" on your platform. If they are unavailable, using them will raise a\n" +" NotImplementedError.\n" +"\n" +"Note that most operations will use the effective uid/gid, therefore this\n" +" routine can be used in a suid/sgid environment to test if the invoking user\n" +" has the specified access to the path.\n" +"\n"); + +#define OS_ACCESS_METHODDEF \ + {"access", (PyCFunction)os_access, METH_VARARGS | METH_KEYWORDS, os_access__doc__} + +static PyObject * +os_access_impl(PyObject *self, path_t path, int mode, int dir_fd, int effective_ids, int follow_symlinks); + +static PyObject * +os_access(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *_return_value; + path_t path = PATH_T_INITIALIZE("access", 0, 1); int mode; int dir_fd = DEFAULT_DIR_FD; int effective_ids = 0; int follow_symlinks = 1; + static char *_keywords[] = {"path", "mode", "dir_fd", "effective_ids", "follow_symlinks", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "O&i|$O&pp:access", _keywords, + path_converter, &path, &mode, OS_ACCESS_DIR_FD_CONVERTER, &dir_fd, &effective_ids, &follow_symlinks)) + return NULL; + + _return_value = os_access_impl(self, path, mode, dir_fd, effective_ids, follow_symlinks); + path_cleanup(&path); + return _return_value; +} + +static PyObject * +os_access_impl(PyObject *self, path_t path, int mode, int dir_fd, int effective_ids, int follow_symlinks) +/*[clinic end:69aac9fd37cfaa7dc4910c8e47acfceec8e06eaf]*/ +{ PyObject *return_value = NULL; #ifdef MS_WINDOWS @@ -2343,17 +2529,6 @@ int result; #endif - memset(&path, 0, sizeof(path)); - if (!PyArg_ParseTupleAndKeywords(args, kwargs, "O&i|$O&pp:access", keywords, - path_converter, &path, &mode, -#ifdef HAVE_FACCESSAT - dir_fd_converter, &dir_fd, -#else - dir_fd_unavailable, &dir_fd, -#endif - &effective_ids, &follow_symlinks)) - return NULL; - #ifndef HAVE_FACCESSAT if (follow_symlinks_specified("access", follow_symlinks)) goto exit; @@ -2409,7 +2584,6 @@ #ifndef HAVE_FACCESSAT exit: #endif - path_cleanup(&path); return return_value; } @@ -10282,10 +10456,11 @@ #endif /* defined(TERMSIZE_USE_CONIO) || defined(TERMSIZE_USE_IOCTL) */ + static PyMethodDef posix_methods[] = { - {"access", (PyCFunction)posix_access, - METH_VARARGS | METH_KEYWORDS, - posix_access__doc__}, + + OS_ACCESS_METHODDEF, + #ifdef HAVE_TTYNAME {"ttyname", posix_ttyname, METH_VARARGS, posix_ttyname__doc__}, #endif @@ -10374,9 +10549,7 @@ {"rmdir", (PyCFunction)posix_rmdir, METH_VARARGS | METH_KEYWORDS, posix_rmdir__doc__}, - {"stat", (PyCFunction)posix_stat, - METH_VARARGS | METH_KEYWORDS, - posix_stat__doc__}, + OS_STAT_METHODDEF, {"stat_float_times", stat_float_times, METH_VARARGS, stat_float_times__doc__}, #if defined(HAVE_SYMLINK) {"symlink", (PyCFunction)posix_symlink, diff -r c65fcedc511c Modules/zlibmodule.c --- a/Modules/zlibmodule.c Thu Feb 21 23:17:34 2013 +0200 +++ b/Modules/zlibmodule.c Sat Feb 23 16:43:37 2013 -0800 @@ -8,6 +8,14 @@ #include "structmember.h" #include "zlib.h" +/*[python] + +clinic.write_clinic_file() + +[python]*/ + +/*[python end:adc83b19e793491b1c6ea0fd8b46cd9f32e592fc]*/ + #ifdef WITH_THREAD #include "pythread.h" #define ENTER_ZLIB(obj) \ @@ -604,38 +612,42 @@ return 0; } -PyDoc_STRVAR(decomp_decompress__doc__, -"decompress(data, max_length) -- Return a string containing the decompressed\n" -"version of the data.\n" -"\n" -"After calling this function, some of the input data may still be stored in\n" -"internal buffers for later processing.\n" -"Call the flush() method to clear these buffers.\n" -"If the max_length parameter is specified then the return value will be\n" -"no longer than max_length. Unconsumed input data will be stored in\n" -"the unconsumed_tail attribute."); +/*[clinic] +zlib.decompress + Py_buffer data + types="bytes bytearray buffer" + The binary data to decompress. + int max_length = 0 + The maximum allowable length of the decompressed data. + Unconsumed input data will be stored in + the unconsumed_tail attribute. + +Return a string containing the decompressed version of the data. + +After calling this function, some of the input data may still be stored in +internal buffers for later processing. +Call the flush() method to clear these buffers. +[clinic]*/ static PyObject * -PyZlib_objdecompress(compobject *self, PyObject *args) +zlib_decompress_impl(PyObject *self, Py_buffer data, int max_length) +/*[clinic end:6108b2a7cb884537c6169d04456df6dbc527ec8b]*/ { - int err, max_length = 0; + compobject *zself = (compobject *)self; + int err; unsigned int inplen; Py_ssize_t old_length, length = DEFAULTALLOC; PyObject *RetVal = NULL; - Py_buffer pinput; Byte *input; unsigned long start_total_out; - if (!PyArg_ParseTuple(args, "y*|i:decompress", &pinput, - &max_length)) - return NULL; - if (pinput.len > UINT_MAX) { + if (data.len > UINT_MAX) { PyErr_SetString(PyExc_OverflowError, "Size does not fit in an unsigned int"); goto error_outer; } - input = pinput.buf; - inplen = pinput.len; + input = data.buf; + inplen = data.len; if (max_length < 0) { PyErr_SetString(PyExc_ValueError, "max_length must be greater than zero"); @@ -648,43 +660,43 @@ if (!(RetVal = PyBytes_FromStringAndSize(NULL, length))) goto error_outer; - ENTER_ZLIB(self); + ENTER_ZLIB(zself); - start_total_out = self->zst.total_out; - self->zst.avail_in = inplen; - self->zst.next_in = input; - self->zst.avail_out = length; - self->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal); + start_total_out = zself->zst.total_out; + zself->zst.avail_in = inplen; + zself->zst.next_in = input; + zself->zst.avail_out = length; + zself->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal); Py_BEGIN_ALLOW_THREADS - err = inflate(&(self->zst), Z_SYNC_FLUSH); + err = inflate(&(zself->zst), Z_SYNC_FLUSH); Py_END_ALLOW_THREADS - if (err == Z_NEED_DICT && self->zdict != NULL) { + if (err == Z_NEED_DICT && zself->zdict != NULL) { Py_buffer zdict_buf; - if (PyObject_GetBuffer(self->zdict, &zdict_buf, PyBUF_SIMPLE) == -1) { + if (PyObject_GetBuffer(zself->zdict, &zdict_buf, PyBUF_SIMPLE) == -1) { Py_DECREF(RetVal); RetVal = NULL; goto error; } - err = inflateSetDictionary(&(self->zst), zdict_buf.buf, zdict_buf.len); + err = inflateSetDictionary(&(zself->zst), zdict_buf.buf, zdict_buf.len); PyBuffer_Release(&zdict_buf); if (err != Z_OK) { - zlib_error(self->zst, err, "while decompressing data"); + zlib_error(zself->zst, err, "while decompressing data"); Py_DECREF(RetVal); RetVal = NULL; goto error; } /* Repeat the call to inflate. */ Py_BEGIN_ALLOW_THREADS - err = inflate(&(self->zst), Z_SYNC_FLUSH); + err = inflate(&(zself->zst), Z_SYNC_FLUSH); Py_END_ALLOW_THREADS } /* While Z_OK and the output buffer is full, there might be more output. So extend the output buffer and try again. */ - while (err == Z_OK && self->zst.avail_out == 0) { + while (err == Z_OK && zself->zst.avail_out == 0) { /* If max_length set, don't continue decompressing if we've already reached the limit. */ @@ -702,16 +714,16 @@ RetVal = NULL; goto error; } - self->zst.next_out = + zself->zst.next_out = (unsigned char *)PyBytes_AS_STRING(RetVal) + old_length; - self->zst.avail_out = length - old_length; + zself->zst.avail_out = length - old_length; Py_BEGIN_ALLOW_THREADS - err = inflate(&(self->zst), Z_SYNC_FLUSH); + err = inflate(&(zself->zst), Z_SYNC_FLUSH); Py_END_ALLOW_THREADS } - if (save_unconsumed_input(self, err) < 0) { + if (save_unconsumed_input(zself, err) < 0) { Py_DECREF(RetVal); RetVal = NULL; goto error; @@ -720,27 +732,27 @@ if (err == Z_STREAM_END) { /* This is the logical place to call inflateEnd, but the old behaviour of only calling it on flush() is preserved. */ - self->eof = 1; + zself->eof = 1; } else if (err != Z_OK && err != Z_BUF_ERROR) { /* We will only get Z_BUF_ERROR if the output buffer was full but there wasn't more output when we tried again, so it is not an error condition. */ - zlib_error(self->zst, err, "while decompressing data"); + zlib_error(zself->zst, err, "while decompressing data"); Py_DECREF(RetVal); RetVal = NULL; goto error; } - if (_PyBytes_Resize(&RetVal, self->zst.total_out - start_total_out) < 0) { + if (_PyBytes_Resize(&RetVal, zself->zst.total_out - start_total_out) < 0) { Py_DECREF(RetVal); RetVal = NULL; } error: - LEAVE_ZLIB(self); + LEAVE_ZLIB(zself); error_outer: - PyBuffer_Release(&pinput); + PyBuffer_Release(&data); return RetVal; } @@ -1047,8 +1059,7 @@ static PyMethodDef Decomp_methods[] = { - {"decompress", (binaryfunc)PyZlib_objdecompress, METH_VARARGS, - decomp_decompress__doc__}, + ZLIB_DECOMPRESS_METHODDEF, {"flush", (binaryfunc)PyZlib_unflush, METH_VARARGS, decomp_flush__doc__}, #ifdef HAVE_ZLIB_COPY @@ -1141,6 +1152,14 @@ return PyLong_FromUnsignedLong(signed_val & 0xffffffffU); } +/*[python] + +clinic.include_clinic_file() + +[python]*/ +#include "zlibmodule_clinic.c" +/*[python end:1ddca76f444ebefd62e7a5445655681098e02362]*/ + static PyMethodDef zlib_methods[] = { @@ -1154,8 +1173,7 @@ crc32__doc__}, {"decompress", (PyCFunction)PyZlib_decompress, METH_VARARGS, decompress__doc__}, - {"decompressobj", (PyCFunction)PyZlib_decompressobj, METH_VARARGS|METH_KEYWORDS, - decompressobj__doc__}, + ZLIB_DECOMPRESS_METHODDEF, {NULL, NULL} }; diff -r c65fcedc511c Modules/zlibmodule_clinic.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Modules/zlibmodule_clinic.c Sat Feb 23 16:43:37 2013 -0800 @@ -0,0 +1,41 @@ + +PyDoc_STRVAR(zlib_decompress__doc__, +"zlib.decompress(data[, max_length=0])\n" +"\n" +" data\n" +" The binary data to decompress.\n" +" max_length\n" +" The maximum allowable length of the decompressed data.\n" +" Unconsumed input data will be stored in\n" +" the unconsumed_tail attribute.\n" +"\n" +"Return a string containing the decompressed version of the data.\n" +"\n" +"After calling this function, some of the input data may still be stored in\n" +"internal buffers for later processing.\n" +"Call the flush() method to clear these buffers.\n"); + +#define ZLIB_DECOMPRESS_METHODDEF \ + {"decompress", (PyCFunction)zlib_decompress, METH_VARARGS | METH_KEYWORDS, zlib_decompress__doc__} + +static PyObject * +zlib_decompress_impl(PyObject *self, Py_buffer data, int max_length); + +static PyObject * +zlib_decompress(PyObject *self, PyObject *args, PyObject *kwargs) +{ + PyObject *_return_value; + Py_buffer data; + int max_length = 0; + static char *_keywords[] = {"data", "max_length", NULL}; + + if (!PyArg_ParseTupleAndKeywords(args, kwargs, + "y*|i:decompress", _keywords, + &data, &max_length)) + return NULL; + + _return_value = zlib_decompress_impl(self, data, max_length); + return _return_value; +} + + diff -r c65fcedc511c clinic-pep.rst --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/clinic-pep.rst Sat Feb 23 16:43:37 2013 -0800 @@ -0,0 +1,481 @@ +PEP: --- +Title: The Argument Clinic DSL +Version: $Revision$ +Last-Modified: $Date$ +Author: Larry Hastings +Discussions-To: Python-Dev +Status: Draft +Type: Standards Track +Content-Type: text/x-rst +Created: 22-Feb-2013 + + +Abstract +======== + +This document proposes "Argument Clinic", a DSL designed +to facilitate argument processing for built-in functions +in the implementation of CPython. + +Rationale and Goals +=================== + +The primary implementation of Python, "CPython", is written in +a mixture of Python and C. One of the implementation details +of CPython is what are called "built-in" functions--functions +available to Python programs but written in C. When a +Python program calls a built-in function and passes in +arguments, those arguments must be translated from Python +values into C values. This process is called "parsing arguments". + +As of CPython 3.3, arguments to functions are primarily +parsed with one of two functions: the original +``PyArg_ParseTuple()``, [1]_ and the more modern +``PyArg_ParseTupleAndKeywords()``. [2]_ +The former function only handles positional parameters; the +latter also accomodates keyword and keyword-only parameters, +and is preferred for new code. + +``PyArg_ParseTuple()`` was a reasonable approach when it was +first concieved. The programmer specified the translation for +the arguments in a "format string": [3]_ each parameter matched to +a "format unit", a one-or-two character sequence telling +``PyArg_ParseTuple()`` what Python types to accept and how +to translate them into the appropriate C value for that +parameter. There were only a dozen or so of these "format +units", and each one was distinct and easy to understand. + +Over the years the ``PyArg_Parse`` interface has been extended in +numerous ways. The modern API is quite complex, to the point +that it is somewhat painful to use. Consider: + + * There are now forty different "format units"; a few are + even three characters long. + This overload of symbology makes it difficult to understand + what the format string says without constantly cross-indexing + it with the documentation. + * There are also six meta-format units that may be buried + in the format string. (They are: ``"()|$:;"``.) + * The more format units are added, the less likely it is the + implementor can pick an easy-to-use mnemonic for the format + unit, because the character of choice is probably already in + use. In other words, the more format units we have, the more + obtuse the format units become. + * Several format units are nearly identical to others, having + only subtle differences. This makes understanding the exact + semantics of the format string even harder. + * The docstring is specified as a static C string, + which is mildly bothersome to read and edit. + * When adding a new parameter to a function using + ``PyArg_ParseTupleAndKeywords()``, it's necessary to + touch six different places in the code: [4]_ + + * Declaring the variable to store the argument. + * Passing in a pointer to that variable in the correct + spot in ``PyArg_ParseTupleAndKeywords()``, also passing + in any "length" or "converter" arguments in the correct + order. + * Adding the name of the argument in the correct spot + of the "keywords" array passed in to + ``PyArg_ParseTupleAndKeywords()``. + * Adding the format unit to the correct spot in the + format string. + * Adding the parameter to the prototype in the + docstring. + * Documenting the parameter in the docstring. + + * There is currently no mechanism for builtin functions + to provide their "signature" information (see + ``inspect.getfullargspec`` and ``inspect.Signature``). + Adding this information using a mechanism similar to + the existing ``PyArg_Parse`` functions would require + repeating ourselves yet again. + +The goal of Argument Clinic is to replace this API with a +mechanism inheriting none of these downsides: + + * You need specify each parameter only once. + * All information about a parameter is kept together in one place. + * For each parameter, you specify its type in C; + Argument Clinic handles the translation from + Python value into C value for you. + * Argument Clinic also allows for fine-tuning + of argument processing behavior with + highly-readable "flags", both per-parameter + and applying across the whole function. + * Docstrings are written in plain text. + * From this, Argument Clinic generates for you all + the mundane, repetitious code and data structures + CPython needs internally. Once you've specified + the interface, the next step is simply to write your + implementation using native C types. Every detail + of argument parsing is handled for you. + +Future goals of Argument Clinic include: + + * providing signature information for builtins, and + * speed improvements to the generated code. + +DSL Syntax Summary +================== + +The Argument Clinic DSL is specified as a comment +embedded in a C file, as follows. The "Example" column on the +right shows you sample input to the Argument Clinic DSL, +and the "Section" column on the left specifies what each line +represents in turn. + +:: + + +-----------------------+-----------------------------------------------------+ + | Section | Example | + +-----------------------+-----------------------------------------------------+ + | Clinic DSL start | /*[clinic] | + | Function declaration | module.function_name -> return_annotation | + | Function flags | flag flag2 flag3=value | + | Parameter declaration | type name = default | + | Parameter flags | flag flag2 flag3=value | + | Parameter docstring | Lorem ipsum dolor sit amet, consectetur | + | | adipisicing elit, sed do eiusmod tempor | + | Function docstring | Lorem ipsum dolor sit amet, consectetur adipisicing | + | | elit, sed do eiusmod tempor incididunt ut labore et | + | Clinic DSL end | [clinic]*/ | + | Clinic output | ... | + | Clinic output end | /*[clinic end output:]*/ | + +-----------------------+-----------------------------------------------------+ + + +General Behavior Of the Argument Clinic DSL +------------------------------------------- + +All lines support ``#`` as a line comment delimiter *except* docstrings. +Blank lines are always ignored. + +Like Python itself, leading whitespace is significant in the Argument Clinic +DSL. The first line of the "function" section is the declaration; +all subsequent lines at the same indent are function flags. Once you indent, +the first line is a parameter declaration; subsequent lines at that indent +are parameter flags. Indent one more time for the lines of the parameter +docstring. Finally, outdent back to the same level as the function +declaration for the function docstring. + +Function Declaration +-------------------- + +The return annotation is optional. If skipped, the arrow ("``->``") must also be omitted. + +Parameter Declaration +--------------------- + +The "type" is a C type. If it's a pointer type, you must specify +a single space between the type and the "``*``", and zero spaces between +the "``*``" and the name. (e.g. "``PyObject *foo``", not "``PyObject* foo``") + +The "name" must be a legal C identifier. + +The "default" is a Python value. Default values are optional; +if not specified you must omit the equals sign too. Parameters +which don't have a default are implicitly required. The default +value is dynamically assigned, "live" in the generated C code, +and although it's specified as a Python value, it's translated +into a native C value in the generated C code. + +It's explicitly permitted to end the parameter declaration line +with a semicolon, though the semicolon is optional. This is +intended to allow directly cutting and pasting in declarations +from C code. However, the preferred style is without the semicolon. + + +Flags +----- + +"Flags" are like "``make -D``" arguments. They're unordered. Flags lines +are parsed much like the shell (specifically, using ``shlex.split()`` [5]_ ). +You can have as many flag lines as you like. Specifying a flag twice +is currently an error. + +Supported flags for functions: + +``basename`` + The basename to use for the generated C functions. + By default this is the name of the function from + the DSL, only with periods replaced by underscores. + +``positional-only`` + This function only supports positional parameters, + not keyword parameters. See `Functions With + Positional-Only Parameters`_ below. + +Supported flags for parameters: + +``bitwise`` + If the Python integer passed in is signed, copy the + bits directly even if it is negative. Only valid + for unsigned integer types. + +``converter`` + Backwards-compatibility support for parameter "converter" + functions. [6]_ The value should be the name of the converter + function in C. Only valid when the type of the parameter + is ``void *``. + +``default`` + The Python value to use in place of the parameter's actual + default in Python contexts. Specifically, when specified, + this value will be used for the parameter's default in the + docstring, and in the ``Signature``. (TBD: If the string is a + valid Python expression, renderable into a Python value + using ``eval()``, then the result of ``eval()`` on it will be used + as the default in the ``Signature``.) Ignored if there is no + default. + +``encoding`` + Encoding to use when encoding a Unicode string to a ``char *``. + Only valid when the type of the parameter is ``char *``. + +``group=`` + This parameter is part of a group of options that must either + all be specified or none specified. Parameters in the same + "group" must be contiguous. The value of the group flag + is the name used for the group variable, and therefore must + be legal as a C identifier. Only valid for functions + marked "``positional-only``"; see `Functions With + Positional-Only Parameters`_ below. + +``immutable`` + Only accept immutable values. + +``keyword-only`` + This parameter (and all subsequent parameters) is + keyword-only. Keyword-only parameters must also be + optional parameters. Not valid for positional-only functions. + +``length`` + This is an iterable type, and we also want its length. The + DSL will generate a second ``Py_ssize_t`` variable; + its name will be this parameter's name appended with + "``_length``". + +``nullable`` + ``None`` is a legal argument for this parameter. If ``None`` is + supplied on the Python side, the equivalent C argument will be + ``NULL``. Only valid for pointer types. + +``required`` + Normally any parameter that has a default value is + automatically optional. A parameter that has "required" + set will be considered required (non-optional) even if + it has a default value. The generated documentation + will also not show any default value. + +``types`` + Space-separated list of acceptable Python types for this + object. There are also four special-case types which + represent Python protocols: + + * buffer + * mapping + * number + * sequence + +``zeroes`` + This parameter is a string type, and its value should be + allowed to have embedded zeroes. Not valid for all + varieties of string parameters. + + +Python Code +----------- + +Argument Clinic also permits embedding Python code inside C files, +which is executed in-place when Argument Clinic processes the file. +Embedded code looks like this: + +:: + + /*[python] + + # this is python code! + print("/" + "* Hello world! *" + "/") + + [python]*/ + +Any Python code is valid. Python code sections in Argument Clinic +can also be used to modify Clinic's behavior at runtime; for example, +see `Extending Argument Clinic`_. + + +Output +====== + +Argument Clinic writes its output in-line in the C file, immediately after +the section of Clinic code. For "python" sections, the output is +everything printed using ``builtins.print``. For "clinic" sections, the +output is valid C code, including: + + * a ``#define`` providing the correct ``methoddef`` structure for the + function + * a prototype for the "impl" function--this is what you'll write to + implement this function + * a function that handles all argument processing, which calls your + "impl" function + * the definition line of the "impl" function + * and a comment indicating the end of output. + +The intention is that you will write the body of your impl function +immediately after the output--as in, you write a left-curly-brace +immediately after the end-of-output comment and write the implementation +of the builtin in the body there. (It's a bit strange at first--but oddly +convenient.) + +Argument Clinic will define the parameters of the impl function for you. +The function will take the "self" parameter passed in originally, all +the parameters you define, and possibly some extra generated parameters +("length" parameters; also "group" parameters, see next section). + +Argument Clinic also writes a checksum for the output section. This +is a valuable safety feature: if you modify the output by hand, Clinic +will notice that the checksum doesn't match, and will refuse to +overwrite the file. (You can force Clinic to overwrite with the "``-f``" +command-line argument; Clinic will also ignore the checksums when +using the "``-o``" command-line argument.) + + +Functions With Positional-Only Parameters +========================================= + +A significant fraction of Python builtins implemented in C use the +older positional-only API for processing arguments (``PyArg_ParseTuple()``). +In some instances, these builtins parse their arguments differently +based on how many arguments were passed in. This can provide some +bewildering flexibility: there may be groups of optional parameters, +which must either all be specified or none specified. And occasionally +these groups are on the *left!* (For example: ``curses.window.addch()``.) + +Argument Clinic supports these legacy use-cases with a special set +of flags. First, set the flag "``positional-only``" on the entire +function. Then, for every group of parameters that is collectively +optional, add a "``group=``" flag with a unique string to all the +parameters in that group. Note that these groups are permitted on +the right *or left* of any required parameters! However, all groups +(including the group of required parameters) must be contiguous. + +The impl function generated by Clinic will add an extra parameter for +every group, "``int _group``". This argument will be nonzero if +the group was specified on this call, and zero if it was not. + +Note that when operating in this mode, you cannot specify default +arguments. You can simulate defaults by putting parameters in +individual groups and detecting whether or not they were +specified--but generally speaking it's better to simply not +use "positional-only" where it isn't absolutely necessary. (TBD: It +might be possible to relax this restriction. But adding default +arguments into the mix of groups would seemingly make calculating which +groups are active a good deal harder.) + +Also, note that it's possible--even easy--to specify a set of groups +to a function such that there are several valid mappings from the number +of arguments to a valid set of groups. If this happens, Clinic will exit +with an error message. This should not be a problem, as positional-only +operation is only intended for legacy use cases, and all the legacy +functions using this quirky behavior should have unambiguous mappings. + + +Current Status +============== + +As of this writing, there is a working prototype implementation of +Argument Clinic available online. [7]_ The prototype implements +the syntax above, and generates code using the existing ``PyArg_Parse`` +APIs. It supports translating to all current format units except ``"w*"``. +Sample functions using Argument Clinic exercise all major features, +including positional-only argument parsing. + +Extending Argument Clinic +------------------------- + +The prototype also currently provides an experimental extension mechanism, +allowing adding support for new types on-the-fly. See ``Modules/posixmodule.c`` +in the prototype for an example of its use. + + +Notes / TBD +=========== + +* Guido proposed having the "function docstring" be hand-written inline, + in the middle of the output, something like this: + + :: + + /*[clinic] + ... prototype and parameters (including parameter docstrings) go here + [clinic]*/ + ... some output ... + /*[clinic docstring start]*/ + ... hand-edited function docstring goes here <-- you edit this by hand! + /*[clinic docstring end]*/ + ... more output + /*[clinic output end]*/ + + I tried it this way and don't like it--I think it's clumsy. I prefer that + everything you write goes in one place, rather than having an island of + hand-edited stuff in the middle of the DSL output. + +* Do we need to support tuple unpacking? (The "``(OOO)``" style format string.) + Boy I sure hope not. + +* What about Python functions that take no arguments? This syntax doesn't + provide for that. Perhaps a lone indented "None" should mean "no arguments"? + +* This approach removes some dynamism / flexibility. With the existing + syntax one could theoretically pass in different encodings at runtime for + the "``es``"/"``et``" format units. AFAICT CPython doesn't do this itself, + however it's possible external users might do this. (Trivia: there are no + uses of "``es``" exercised by regrtest, and all the uses of "``et``" + exercised are in socketmodule.c, except for one in _ssl.c. They're all + static, specifying the encoding ``"idna"``.) + +* Right now the "basename" flag on a function changes the ``#define methoddef`` name + too. Should it, or should the #define'd methoddef name always be + ``{module_name}_{function_name}`` ? + + +References +========== + +.. [1] ``PyArg_ParseTuple()``: + http://docs.python.org/3/c-api/arg.html#PyArg_ParseTuple + +.. [2] ``PyArg_ParseTupleAndKeywords()``: + http://docs.python.org/3/c-api/arg.html#PyArg_ParseTupleAndKeywords + +.. [3] ``PyArg_`` format units: + http://docs.python.org/3/c-api/arg.html#strings-and-buffers + +.. [4] Keyword parameters for extension functions: + http://docs.python.org/3/extending/extending.html#keyword-parameters-for-extension-functions + +.. [5] ``shlex.split()``: + http://docs.python.org/3/library/shlex.html#shlex.split + +.. [6] ``PyArg_`` "converter" functions, see ``"O&"`` in this section: + http://docs.python.org/3/c-api/arg.html#other-objects + +.. [7] Argument Clinic prototype: + https://bitbucket.org/larry/python-clinic/ + +Copyright +========= + +This document has been placed in the public domain. + + + +.. + Local Variables: + mode: indented-text + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 70 + coding: utf-8 + End: diff -r c65fcedc511c clinic.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/clinic.py Sat Feb 23 16:43:37 2013 -0800 @@ -0,0 +1,1463 @@ +#!/usr/bin/env python3 +# +# Argument Clinic (clinic.py) +# +# DSL for argument preprocessing +# +# Copyright 2012 by Larry Hastings. +# Licensed to the PSF under a Contributor Agreement. +# + + +import atexit +import builtins +import collections +import hashlib +import os +import re +import shlex +import sys +import tempfile +import textwrap + + +tempfiles = [] +def remove_tempfiles(): + for filename in tempfiles: + try: + os.unlink(filename) + except BaseException: + pass + +atexit.register(remove_tempfiles) + + + +NULL = object() + +class Unspecified: + def __repr__(self): + return '' + +unspecified = Unspecified() + + +type_map = {} + +def type_map_register(cls): + type_map[cls.type] = cls + return cls + +def type_map_alias(type): + def decorator(cls): + type_map[type] = cls + return cls + return decorator + + +class CVariable: + """ + An individual C variable required by an Argument. + (Some Arguments require more than one.) + + When subclassing, the most relevant things to override are: + type + format_unit_map + format_unit_flags + default + cleanup + """ + + type = None + + def __init__(self, name, default=unspecified): + self.name = name + self.str_default = default + self.flags = {} + + def impl_argument(self): + """ + The C code to pass this argument in to the impl function. + Returns an iterable. + """ + assert self.name and self.type + return (self.name,) + + def keyword(self): + """ + The value to add to the 'keywords' array for PyArg_ParseTupleAndKeywords. + Returns an iterable. + """ + assert self.name and self.type + return ('"' + self.name + '"',) + + def parse_argument(self): + """ + The C code to pass this argument in to PyArg_ParseTuple &c. + Returns an iterable. + """ + assert self.name and self.type + return ("&" + self.name,) + + def prototype(self): + """ + The C code to define this variable as an argument to a function. + Returns an iterable. + """ + assert self.name and self.type + prototype = [self.type] + if not self.type.endswith('*'): + prototype.append(" ") + prototype.append(self.name) + return ("".join(prototype),) + + value_to_cvalue_map = { + NULL: "NULL", + None: "Py_None", + } + + def default(self, s): + """ + Returns a string to be used as the initializer for this variable, or None if no initializer is needed. + s is a Python value. + If s is the special value 'unspecified', no default was specified. + """ + if s is unspecified: + return None + return self.value_to_cvalue_map.get(s, s) + + def declaration(self): + """ + The C statement to declare this variable. + Returns an iterable. + """ + declaration = list(self.prototype()) + default = self.default(self.str_default) + if default: + declaration.append(" = ") + declaration.append(default) + declaration.append(";") + return ("".join(declaration),) + + def cleanup(self): + """ + The C statements required to clean up after this variable. + Returns an iterable. + (If no cleanup is necessary, returns an empty iterable.) + """ + return () + + # should be a map of frozenset(flags) -> string + # the string represent "true" flags + # if there are also associated "types", then the key becomes a tuple + # first element is frozenset(flags) + # second element is frozenset(types) + format_unit_map = {} + + # the set of flags get_format_unit() will pay attention + # to when computing the format unit. + format_unit_flags = frozenset("bitwise encoding immutable length nullable zeroes".split()) + + @property + def format_unit(self): + if 'converter' in self.flags: + return "O&" + + flags = dict(self.flags) + if 'types' in flags: + types = frozenset(flags['types'].split()) + del flags['types'] + else: + types = None + + key = set() + for flag in self.format_unit_flags: + if flags.get(flag): + key.add(flag) + key = frozenset(key) + if types: + key = (key, types) + format_unit = self.format_unit_map.get(key) + if not format_unit: + sys.exit("No valid format unit for " + self.__class__.__name__ + " " + self.name + " (no match for flags " + repr(key) + ")") + return format_unit + + +@type_map_register +class CVariable_PyObject_star(CVariable): + """ + PyObject * = O + """ + + type = "PyObject *" + + format_unit_map = { + frozenset() : "O", + } + + +@type_map_register +class CVariable_Py_UNICODE_star(CVariable): + """ + Py_UNICODE * = u + Py_UNICODE * length = u# + Py_UNICODE * nullable = Z + Py_UNICODE * length nullable = Z# + """ + + type = "Py_UNICODE *" + + format_unit_map = { + frozenset() : "u", + frozenset(('length',)) : "u#", + frozenset(('nullable',)) : "Z", + frozenset(('length', 'nullable')) : "Z#", + } + +@type_map_register +class CVariable_char_star(CVariable): + """ + char * encoding = es + char * encoding (str bytes bytearray) = et + char * encoding length = es# + char * encoding length (str bytes bytearray) = et# + """ + + type = "char *" + + str_types = frozenset(('str', 'bytes', 'bytearray')) + + format_unit_map = { + frozenset(('encoding',)) : "es", + frozenset(('encoding', 'length')) : "es#", + (frozenset(('encoding',)), str_types) : "et", + (frozenset(('encoding', 'length')), str_types) : "et#", + } + +@type_map_register +class CVariable_const_char_star(CVariable): + """ + const char * = s + const char * nullable = z + const char * (bytes) = y + const char * length (bytes buffer) = y# + const char * length zeroes (str bytes buffer) = s# + const char * length nullable zeroes (str bytes buffer) = z# + """ + + type = "const char *" + + types_bytes = frozenset(('bytes,')) + types_bytes_buffer = frozenset(('buffer,')) | types_bytes + types_str_bytes_buffer = frozenset(('str,')) | types_bytes_buffer + + format_unit_map = { + frozenset() : "s", + frozenset(('nullable',)) : "z", + (frozenset(), types_bytes) : "y", + (frozenset(('length',)), types_bytes_buffer) : "y#", + + (frozenset(('length', 'zeroes')), types_str_bytes_buffer) : "s#", + (frozenset(('length', 'nullable', 'zeroes')), types_str_bytes_buffer) : "z#", + } + +@type_map_register +class CVariable_Py_buffer(CVariable): + """ + Py_buffer (bytes bytearray buffer) = y* + Py_buffer zeroes (str bytes bytearray buffer) = s* + Py_buffer nullable zeroes (str bytes bytearray buffer) = z* + """ + + type = 'Py_buffer' + + bytes_types = frozenset(('bytes', 'bytearray', 'buffer')) + bytes_types_and_str = frozenset(('str',)) | bytes_types + + format_unit_map = { + (frozenset(), bytes_types) : "y*", + (frozenset(('zeroes',)), bytes_types_and_str) : "s*", + (frozenset(('nullable', 'zeroes',)), bytes_types_and_str) : "z*", + } + + +@type_map_register +class CVariable_PyUnicodeObject_star(CVariable): + """ + PyUnicodeObject * = U + """ + + type = 'PyUnicodeObject *' + + format_unit_map = { + frozenset() : "U", + } + + +@type_map_register +class CVariable_PyBytesObject_star(CVariable): + """ + PyBytesObject * = S + """ + + type = 'PyBytesObject *' + + format_unit_map = { + frozenset() : "S", + } + + +@type_map_register +class CVariable_PyBytesArrayObject_star(CVariable): + """ + PyBytesArrayObject * = Y + """ + + type = 'PyBytesArrayObject *' + + format_unit_map = { + frozenset() : "Y", + } + +@type_map_register +class CVariable_unsigned_char(CVariable): + """ + unsigned char = b + unsigned char bitwise = B + """ + + type = 'unsigned char' + + format_unit_map = { + frozenset() : "b", + frozenset(('bitwise',)) : "B", + } + +@type_map_register +@type_map_alias('short int') +@type_map_alias('signed short') +@type_map_alias('signed short int') +class CVariable_short(CVariable): + """ + short = h + """ + + type = 'short' + + format_unit_map = { + frozenset() : "h", + } + + + +@type_map_register +@type_map_alias('unsigned short int') +class CVariable_unsigned_short(CVariable): + """ + unsigned short bitwise = H + """ + + type = 'unsigned short' + + format_unit_map = { + frozenset(('bitwise',)) : "H", + } + + +@type_map_register +@type_map_alias('signed int') +class CVariable_int(CVariable): + """ + int = i + int (bool) = p + """ + + type = 'int' + + format_unit_map = { + frozenset() : "i", + (frozenset(), frozenset(('bool',))) : "p", + } + +@type_map_register +@type_map_alias('unsigned') +class CVariable_unsigned_int(CVariable): + """ + unsigned int bitwise = I + """ + + type = 'unsigned int' + + format_unit_map = { + frozenset(('bitwise',)) : "I", + } + +@type_map_register +@type_map_alias('signed long') +@type_map_alias('long int') +@type_map_alias('signed long int') +class CVariable_long(CVariable): + """ + long = l + """ + + type = 'long' + + format_unit_map = { + frozenset() : "l", + } + +@type_map_register +@type_map_alias('unsigned long int') +class CVariable_unsigned_long(CVariable): + """ + unsigned long bitwise = k + """ + + type = 'unsigned long' + + format_unit_map = { + frozenset(('bitwise',)) : "k", + } + + +@type_map_register +class CVariable_PY_LONG_LONG(CVariable): + """ + PY_LONG_LONG = L + """ + + type = 'PY_LONG_LONG' + + format_unit_map = { + frozenset() : "L", + } + +@type_map_register +class CVariable_unsigned_PY_LONG_LONG(CVariable): + """ + unsigned PY_LONG_LONG bitwise = K + """ + + type = 'unsigned PY_LONG_LONG' + + format_unit_map = { + frozenset(('bitwise',)) : "K", + } + + + + +@type_map_register +class CVariable_Py_ssize_t(CVariable): + """ + Py_ssize_t = n + """ + + type = 'Py_ssize_t' + + format_unit_map = { + frozenset() : "n", + } + +@type_map_register +class CVariable_char(CVariable): + """ + char = C + char (bytes bytearray) = c + """ + + type = 'char' + + format_unit_map = { + frozenset() : "C", + (frozenset(), frozenset(('bytes', 'bytearray'))) : "c", + } + +@type_map_register +class CVariable_float(CVariable): + """ + float = f + """ + + type = 'float' + + format_unit_map = { + frozenset() : "f", + } + + + +@type_map_register +class CVariable_double(CVariable): + """ + double = d + """ + + type = 'double' + + format_unit_map = { + frozenset() : "d", + } + +@type_map_register +class CVariable_Py_complex_star(CVariable): + """ + Py_complex * = D + """ + + type = 'Py_complex *' + + format_unit_map = { + frozenset() : "D", + } + + +class CHardCodedParserArgument(CVariable): + def __init__(self, argument): + self._parse_argument = (argument,) + + def parse_argument(self): + return self._parse_argument + + def impl_argument(self): + return () + + prototype = declaration = keyword = impl_argument + + +class CGroupBoolean(CVariable): + def __init__(self, name): + super().__init__("int", name, 0) + + def parse_argument(self): + return () + + keyword = parse_argument + + +class ArgumentBase: + + def __init__(self): + self.variables = [] + self.flags = {} + + def flag(self, key, default=False): + return self.flags.get(key, default) + + def required(): + return (self.default == unspecified) or self.flag('required') + + def _finalize(self): + pass + + def _from_variables(self, name): + self._finalize() + l = [] + for v in self.variables: + l.extend(getattr(v, name)()) + return l + + def is_optional(self): + return False + + def impl_argument(self): + return self._from_variables("impl_argument") + + def keyword(self): + return self._from_variables("keyword") + + def parse_argument(self): + return self._from_variables("parse_argument") + + def prototype(self): + return self._from_variables("prototype") + + def declaration(self): + return self._from_variables("declaration") + + def cleanup(self): + return self._from_variables("cleanup") + + def format_unit(self): + return '' + + + +class Optional(ArgumentBase): + def __init__(self): + super().__init__() + self.name = '* optional *' + + def format_unit(self): + return "|" + + def docstring(self): + return () + + +class KeywordOnly(ArgumentBase): + def __init__(self): + super().__init__() + self.name = '* keyword-only *' + + def format_unit(self): + return "$" + + def docstring(self): + return ('*',) + + +class Argument(ArgumentBase): + def __init__(self, c_type, name, flags, line, default=unspecified): + self.variables = [] + + type = type_map.get(c_type) + if not type: + sys.exit("Could not create variable of type " + repr(c_type)) + self.variable = type(name, default) + self.variable.flags.update(flags) + + if 'converter' in flags: + self.variable.type = c_type + + self.name = name + self.line = line + self.default = default + self.docstrings = [] + + @property + def flags(self): + return self.variable.flags + + def is_optional(self): + return (self.default != unspecified) and not self.flag('required') + + def _finalize(self): + if self.variables: + return + + if self.flag('group-first'): + group = self.flag('group', None) + if group is not None: + group_boolean = CGroupBoolean(group + "_group") + self.variables.append(group_boolean) + + converter = self.flag('converter', '') + encoding = self.flag('encoding', '') + assert not (converter and encoding) + if encoding: + v = CHardCodedParserArgument('"' + encoding + '"') + self.variables.append(v) + elif converter: + v = CHardCodedParserArgument(converter) + self.variables.append(v) + + self.variables.append(self.variable) + + if self.flag('length'): + length_name = self.name + "_length" + length = CVariable('Py_ssize_t', length_name, -1) + self.variables.append(length) + + def docstring(self): + l = [self.name] + if self.is_optional(): + l.append('=') + value = self.flag('default', unspecified) + if value is unspecified: + try: + value = repr(eval(self.default)) + except (ValueError, NameError): + value = self.default + l.append(value) + return l + + def format_unit(self): + """ + Render argument "format unit" in PyArg_ParseTupleAndKeywords format string. + Returns string. + """ + return self.variable.format_unit + + +re_argument = re.compile(r"^(?P[A-Za-z0-9_\s]+)((?P\s*\*\s*)|(\s+))(?P\w+)(\s*=\s*(?P\S.*))?$") + +class FauxPrint: + def __init__(self): + self.output = [] + + def print(self, *args, sep=' ', end='\n', file=sys.stdout, flush=False): + line = sep.join(str(a) for a in args) + end + invalid_lines = ( + Clinic.python_begin_string, + Clinic.dsl_begin_string, + ) + if line.startswith(invalid_lines): + error = ["Invalid line in output! You can't use any of the following"] + error.append("in the output of a Python section:") + error.extend([" " + invalid for invalid in invalid_lines]) + sys.exit("\n".join(error)) + if file == sys.stdout: + self.output.append(line) + else: + file.write(line) + if flush: + file.flush() + + +class CountedDeque: + def __init__(self, iterable=()): + self.line = -1 + self.deque = collections.deque(iterable) + self.left = [] + + def __bool__(self): + return bool(self.deque) or bool(self.left) + + def popleft(self): + self.line += 1 + if self.left: + return self.left.pop() + return self.deque.popleft() + + def extendleft(self, i): + for line in i: + self.line -= 1 + self.left.append(line) + +class Clinic: + def __init__(self, *, verify=True): + self.state = self.state_reset + self.verify = verify + self.impl_file = None + + def write_clinic_file(self, filename=None): + filename = filename or (self.filename.partition('.c')[0] + "_clinic.c") + self.clinic_filename = filename + if os.path.isfile(filename): + os.unlink(filename) + self.impl_file = open(filename, "wt") + + def include_clinic_file(self, filename=None): + print('#include "' + os.path.basename(self.clinic_filename) + '"') + + def rewrite(self, filename): + self.read(filename) + self.write(filename) + + def read(self, filename): + self.filename = filename + + self.output = [] + with open(self.filename, "rt") as f: + self.input = CountedDeque(f.readlines()) + + def write(self, filename): + if not self.output: + self._rewrite() + + self.filename = filename + + body = "\n".join(self.output) + # always end .c files with a blank line + if not body.endswith("\n"): + body += "\n" + body = body.encode("utf-8") + directory = os.path.dirname(filename) + f, name = tempfile.mkstemp(text=True, dir=directory) + tempfiles.append(name) + os.write(f, body) + os.close(f) + os.rename(name, self.filename) + + if self.impl_file: + i = self.impl_file + self.impl_file = None + i.close() + + tempfiles.remove(name) + + python_begin_string = "/*[python]" + python_end_string = "[python]*/" + python_end_output_prefix = "/*[python end:" + python_end_output_suffix = "]*/" + + dsl_begin_string = "/*[clinic]" + dsl_end_string = "[clinic]*/" + dsl_end_output_prefix = "/*[clinic end:" + dsl_end_output_suffix = "]*/" + + def _rewrite(self): + while self.input: + line = self.input.popleft().rstrip() + # Always append the line to the output here. + # When we throw away lines from the input, + # we do so from a separate input-reading spot. + self.output.append(line) + self.state(line) + assert self.state == self.state_idle + self.next(self.state_reset) + + @staticmethod + def ignore_line(line): + # ignore comment-only lines + if line.lstrip().startswith('#'): + return True + + # Ignore empty lines too + # (but not in docstring sections!) + if not line.strip(): + return True + + return False + + @staticmethod + def tab_nanny(line): + if '\t' in line: + sys.exit('Error: you used a tab in the DSL.\n\t' + repr(line)) + + @staticmethod + def calculate_indent(line): + return len(line) - len(line.strip()) + + def flag(self, name, default=False): + return self.flags.get(name, default) + + def found_dsl_end(self, line): + if line == self.dsl_end_string: + self.next(self.state_dsl_end, line) + return True + return False + + @staticmethod + def parse_flags_line(d, line): + line = line.strip() + if not line: + return + + if '#' in line: + # we want to support # line comments on flag lines. + # but there could be a # inside a quoted string! what to do? + # + # shlex.split is smart about complaining about unterminated quotes. + # so: try clipping the line at every instance of a '#' + # and use the first one where shlex.split doesn't complain. + subline = "" + for segment in line.split('#'): + subline += segment + try: + attempt = shlex.split(subline) + line = subline + break + except ValueError: + subline += '#' + continue + for field in shlex.split(line): + if '=' in field: + name, _, value = field.partition('=') + name = name.strip() + value = value.strip() + else: + name = field + value = True + if name in d: + sys.exit("Error: Can't specify " + repr(name) + " twice.\n\t" + line) + d[name] = value + + def next(self, state, line=None): + # print(self.state.__name__, "->", state.__name__, ", line=", line) + self.state = state + if line is not None: + self.state(line) + + def find_terminator(self, terminator_prefix, begin_string): + # read ahead until we find the end marker + saved = [] + + while self.input: + line = self.input.popleft().rstrip() + saved.append(line) + if line == begin_string: + break + if line.startswith(terminator_prefix): + if self.verify: + saved.pop() # remove terminator_prefix line from the end + # verify checksum + checksum_and_comment_end = line[len(terminator_prefix):] + stored_checksum, comment, eol = checksum_and_comment_end.partition(']*/') + if eol or not comment: + sys.exit('Invalid marker line found:\n ' + line + '\nGiving up.') + data = "\n".join(saved).encode('utf-8') + b'\n' + checksum = hashlib.sha1(data).hexdigest() + if checksum != stored_checksum: + sys.exit('Checksum doesn\'t match for marker line:\n ' + line + '\nGiving up.') + return + + # if we got here, we didn't find a valid end marker for this block. + # restore the entire remainder of the file. + # (we'll automatically insert the marker with the template) + self.input.extendleft(reversed(saved)) + + def state_reset(self, line=None): + # dsl + self.module_name = self.name = None + self.return_annotation = None + self.flags = {} + self.indent = self.argument_indent = None + self.arguments = [] + self.is_optional = self.is_keyword_only = False + self.docstring = [] + self.groups = set() + + # python + self.code = [] + + self.next(self.state_idle, line) + + def state_idle(self, line): + if line == self.dsl_begin_string: + self.next(self.state_dsl_start) + if line == self.python_begin_string: + self.next(self.state_python_start) + + def state_python_start(self, line): + self.next(self.state_python_body, line) + + def state_python_body(self, line): + if line == self.python_end_string: + self.next(self.state_python_end, line) + return + self.code.append(line) + + def state_python_end(self, line): + self.find_terminator(self.python_end_output_prefix, self.python_begin_string) + code = "\n".join(self.code) + + faux = FauxPrint() + + old_print = builtins.print + builtins.print = faux.print + exec(code) + builtins.print = old_print + + output = "".join(faux.output) + if output and output[-1] == '\n': + output = output[:-1] + self.output.append(output) + data = output.encode('utf-8') + b'\n' + line = "".join([ + self.python_end_output_prefix, + hashlib.sha1(data).hexdigest(), + self.python_end_output_suffix, + ]) + self.output.append(line) + + self.next(self.state_reset) + + def state_dsl_start(self, line): + self.next(self.state_modulename_name, line) + + def state_modulename_name(self, line): + # looking for declaration at left column + # line should be + # modulename.fnname -> return annotation + # everything after here ^ + # is optional. + + if self.ignore_line(line) or self.found_dsl_end(line): + return + + assert not line[0].isspace(), "first line of clinic dsl should not start with whitespace:" + repr(line) + self.tab_nanny(line) + line, arrow, return_annotation = line.partition('->') + self.return_annotation = return_annotation.strip() + self.module_name, period, self.name = line.strip().partition('.') + self.next(self.state_function_flags) + + def state_function_flags(self, line): + if self.ignore_line(line) or self.found_dsl_end(line): + return + + if line[0].isspace(): + return self.next(self.state_argument, line) + + self.tab_nanny(line) + self.parse_flags_line(self.flags, line) + + + # in the arguments section. + # the rule is: + # * the first line must be an argument declaration. + # * this first line establishes the indent for arguments. + # * thenceforth: + # * one or more lines immediately after an argument declaration + # at the same indent level are flags lines. + # * lines indented further are docstrings for the previous arg. + # * the section ends if we hit the function docstring. + # * the first line of the function docstring must be at column 0. + # * it's illegal to have something outdented but not at column 0 + # (until you get to the function docstring). + + def state_argument(self, line): + if self.ignore_line(line) or self.found_dsl_end(line): + return + + self.tab_nanny(line) + + if self.argument_indent == None: + self.argument_indent = self.calculate_indent(line) + + # be nice: allow ";" but clip it here + original_line = line + if line.endswith(';'): + line = line[:-1].strip() + + match = re_argument.match(line) + d = match.groupdict() + ctype = d['ctype'].strip() + if d.get('splat'): + ctype += ' *' + aname = d['name'] + # can't use d.get for the default argument, + # it's always set to None if it wasn't found. + # (thanks, re module!) + default = d["default"] + if default == None: + default = unspecified + elif self.flag('positional-only'): + sys.exit("Error: Can't use default values for arguments with positional-only functions (see " + self.module_name + "." + self.name + ")") + + self.variable_ctype = ctype + self.variable_name = aname + self.variable_line = line + self.variable_original_line = original_line + self.variable_default = default + self.variable_flags = {} + + # self.arguments.append(a) + + self.next(self.state_argument_flags) + + def state_argument_flags(self, line): + if self.ignore_line(line) or self.found_dsl_end(line): + return + + self.tab_nanny(line) + + indent = self.calculate_indent(line) + if indent == self.argument_indent: + self.parse_flags_line(self.variable_flags, line) + return + + # transition to next state, + # this is a good time for a whole bunch o' processing + a = Argument(self.variable_ctype, self.variable_name, self.variable_flags, self.variable_line, self.variable_default) + a.original_line = self.variable_original_line + self.arguments.append(a) + + keyword_only = a.flag('keyword-only') + optional = (a.default != unspecified) and (not a.flag('required')) + + if keyword_only and not optional: + sys.exit("Error: You can't have a parameter which is keyword-only and required:\n\t" + a.original_line) + + if optional: + if not self.is_optional: + self.is_optional = True + self.arguments[-1] = Optional() + self.arguments.append(a) + elif self.is_optional: + sys.exit("Error: required parameter:\n\t" + a.original_line + "\nspecified after optional parameter.") + + if keyword_only and not self.is_keyword_only: + self.is_keyword_only = True + self.arguments[-1] = KeywordOnly() + self.arguments.append(a) + + group = a.flag('group') + if group and group not in self.groups: + self.groups.add(group) + a.flags['group-first'] = True + + return self.next(self.state_arguments_docstring, line) + + + def state_arguments_docstring(self, line): + if line.lstrip().startswith('#'): + return + + self.tab_nanny(line) + + assert self.arguments + a = self.arguments[-1] + + if not line.strip(): + a.docstrings.append('') + return + + indent = self.calculate_indent(line) + + if indent == self.argument_indent: + return self.next(self.state_argument, line) + elif indent < self.argument_indent: + assert not indent, "The first line of the function docstring must be at column 0." + return self.next(self.state_docstring, line) + + a.docstrings.append(line[self.argument_indent:]) + + def state_docstring(self, line): + if line.lstrip().startswith('#'): + return + + if self.found_dsl_end(line): + return + + self.tab_nanny(line) + + self.docstring.append(line) + + + def argument_list(self, name): + l = [] + for a in self.arguments: + l.extend(getattr(a, name)()) + return l + + def state_dsl_end(self, line): + self.find_terminator(self.dsl_end_output_prefix, self.dsl_begin_string) + + # used by .format(**locals()) + module_name = self.module_name + name = self.name + # for names which are themselves dotted + last_name = name.rpartition('.')[2] + + is_optional = False + is_keyword_only = False + using_groups = False + + # ensure that optional and keyword-only stuff is sane + for a in self.arguments: + + if isinstance(a, Optional): + assert not is_optional + is_optional = True + continue + + if isinstance(a, KeywordOnly): + assert is_optional + assert not is_keyword_only + is_keyword_only = True + continue + + using_groups = using_groups or a.flag('group') + + if using_groups and not self.flag('positional-only'): + sys.exit("Error: can't use groups without positional-only (see " + repr(module_name + '.' + name) + ")") + + module_name_upper = module_name.upper() + name_upper = name.upper() + prototype_name = "{module_name}_{name}".format(**locals()).replace('.', '_') + prototype_name = self.flag('basename') or prototype_name + prototype_name_upper = prototype_name.upper() + + debug = builtins.print + + faux = FauxPrint() + + if self.impl_file: + impl_printer = FauxPrint() + print = impl_printer.print + else: + print = faux.print + + ## + ## compute stuff that's the same whether or not we're doing positional-only + ## + + # compute square brackets for docstring first line + if self.flag('positional-only'): + # count how many arguments there are in each group + groups = [] + counter = collections.Counter() + group_to_arguments = {} + left_groups = [] + right_groups = [] + append_to = left_groups + last = unspecified + for a in self.arguments: + group = a.flag('group', None) + if group != last: + last = group + if group in groups: + sys.exit("Non-contiguous use of group " + repr(group)) + groups.append(group) + group_to_arguments[group] = [] + if group is None: + append_to = right_groups + append_to.append(group) + counter[group] += 1 + group_to_arguments[group].append(a) + left_groups = list(reversed(left_groups)) + + right_bracket_counts = {None:0} + # skip None group (which is first in right_groups) + for group_list in (left_groups, right_groups[1:]): + right_bracket_counts.update({group: i+1 for (i, group) in enumerate(group_list)}) + last = unspecified + for a in self.arguments: + group = a.flag('group', None) + a.right_bracket_count = right_bracket_counts[group] + else: + # not positional-only + # all optional arguments are ungrouped and separately right-bracketed + # but: all keyword-only arguments don't get brackets! + i = 0 + is_keyword_only = False + for a in self.arguments: + if 0 and isinstance(a, KeywordOnly): + is_keyword_only = True + i = 0 + elif not is_keyword_only and a.is_optional(): + i += 1 + a.right_bracket_count = i + + # the docstring! + docstring_first_line = 'PyDoc_STRVAR({prototype_name}__doc__,\n"{module_name}.{name}('.format(**locals()) + + first_line = [] + docstring_arguments = [] + right_bracket_count = 0 + def fix_right_bracket_count(desired): + nonlocal right_bracket_count + s = '' + while right_bracket_count < desired: + s += '[' + right_bracket_count += 1 + while right_bracket_count > desired: + s += ']' + right_bracket_count -= 1 + return s + + for a in self.arguments: + s = "".join(a.docstring()) + if s: + s = fix_right_bracket_count(a.right_bracket_count) + s + for old, new in ( + ('"', '\\"'), + ("'", "\\'"), + ): + s = s.replace(old, new) + first_line.append(s) + if isinstance(a, Argument): + docstring_arguments.append(a.name) + for line in a.docstrings: + docstring_arguments.append(line.rstrip()) + + docstring_first_line += ', '.join(first_line) + fix_right_bracket_count(0) + ')' + + # now fix up the places where the brackets look wrong + # and quote the strings + for old, new in ( + (', ]', ',] '), + (', [', '[, '), + ): + docstring_first_line = docstring_first_line.replace(old, new) + + if self.return_annotation: + annotation = self.return_annotation + docstring_first_line += ' -> ' + annotation + docstring_first_line += '\\n"\n"\\n"\n' + + # the contents of the {arguments} string for the docstring + arguments_docstring = "\n".join(docstring_arguments) + + docstring_end = '\n'.join(self.docstring) + if docstring_end: + if '{arguments}' not in docstring_end: + docstring_end = ' {arguments}\n\n' + docstring_end + arguments_docstring = arguments_docstring.rstrip() + final = [] + for line in docstring_end.split('\n'): + indent, is_arguments_line, _ = line.partition('{arguments}') + if is_arguments_line: + final.append(textwrap.indent(arguments_docstring, indent)) + else: + final.append(line) + docstring_end = "\n".join(final) + docstring_end = '"' + docstring_end.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n"\n"') + '\\n"' + + docstring = docstring_first_line + docstring_end + ');' + + print() + print(docstring) + print() + + # the methoddef define! + meth_keywords = " | METH_KEYWORDS" if not self.flag('positional-only') else '' + print('#define {prototype_name_upper}_METHODDEF \\'.format(**locals())) + print(' {{"{last_name}", (PyCFunction){prototype_name}, METH_VARARGS{meth_keywords}, {prototype_name}__doc__}}'.format(**locals())) + print() + + # the prototype! + prototype_arguments = ", ".join(self.argument_list('prototype')) + impl = prototype_name + "_impl" + impl_prototype = "static PyObject *\n{impl}(PyObject *self, {prototype_arguments})".format(**locals()) + print(impl_prototype + ';') + print() + + print("static PyObject *") + kwargs = ", PyObject *kwargs" if not self.flag('positional-only') else '' + print("{prototype_name}(PyObject *self, PyObject *args{kwargs})".format(**locals())) + print("{") + print(" PyObject *_return_value;") + + # declarations, with defaults + for a in self.arguments: + for line in a.declaration(): + print(" ", line) + + if not self.flag('positional-only'): + # not positional only, aka uses PyArg_ParseTupleAndKeywords + + format_units = '' + parse_arguments = [] + for a in self.arguments: + format_units += a.format_unit() + parse_arguments.extend(a.parse_argument()) + format_units += ':' + last_name + + # "keywords" array for PyArg_ParseTupleAndKeywords + quoted_names = self.argument_list('keyword') + quoted_names.append("NULL") + keywords = " static char *_keywords[] = {" + ", ".join(quoted_names) + "};" + print(keywords) + + print() + + print(" if (!PyArg_ParseTupleAndKeywords(args, kwargs,") + print(' "' + format_units + '", _keywords,') + print(' ' + ', '.join(parse_arguments) + '))') + print(' return NULL;') + + else: + # positional only, grouped, optional arguments! + # can be optional on the left or right. + # here's an example: + # + # [ [ [ A1 A2 ] B1 B2 B3 ] C1 C2 ] D1 D2 D3 [ E1 E2 E3 [ F1 F2 F3 ] ] + # + # Here group D are required, and all other groups are optional. + # (Group D's "group" is actually None.) + # We can figure out which sets of arguments we have based on + # how many arguments are in the tuple. + # + # Note that you need to count up on both sides. For example, + # you could have groups C and D, or C & D & E, or C & D & E & F. + # + # What if the number of arguments leads us to an ambiguous result? + # Why, then, you're a *loser* my dear! Clinic will exit with + # an error message. + + left_bundles = [()] + l = [] + for group in left_groups: + l.append(group) + left_bundles.append(tuple(l)) + + count_min = 10000 + count_max = -1 + print() + print(" switch (PyTuple_Size(args)) {") + bundle_counts = {} + for b in left_bundles: + l = list(b) + for group in right_groups: + l.append(group) + bundle = tuple(l) + count = sum(counter[g] for g in bundle) + count_min = min(count_min, count) + count_max = max(count_max, count) + if count in bundle_counts: + sys.exit("Can't process arguments for " + self.module_name + "." + self.name + ": two bundles with same argument counts, " + repr(bundle) + " and " + repr(bundle_counts[count])) + bundle_counts[bundle] = count + + print(" case " + str(count) + ":") + print(' if (!PyArg_ParseTuple(args, "', end='') + parse_arguments = [] + group_booleans = [] + for group in bundle: + if group is not None: + group_booleans.append(group + "_group") + for a in group_to_arguments[group]: + print(a.format_unit(), end='') + parse_arguments.extend(a.parse_argument()) + print(':' + last_name + '", ', end='') + print(", ".join(parse_arguments) + "))") + print(" return NULL;") + for group in bundle: + if group is not None: + print(" " + group + "_group = 1;") + print(" break;") + print(" default:") + print(' PyErr_SetString(PyExc_TypeError, "' + self.name + ' requires', count_min, 'to', count_max, 'arguments");') + print(' return NULL;') + print(" }") + + impl_arguments = [] + for a in self.arguments: + impl_arguments.extend(a.impl_argument()) + print() + print(' _return_value = ' + impl + '(self, ' + ', '.join(impl_arguments) + ');') + + for a in self.arguments: + for line in a.cleanup(): + print(" ", line) + + print(" return _return_value;") + print("}") + print() + + # the header for the impl! + print = faux.print + print(impl_prototype) + + # write output! + text = "".join(faux.output) + data = text.encode('ascii') + checksum = hashlib.sha1(data).hexdigest() + self.output.append("".join([ + text, + self.dsl_end_output_prefix, + checksum, + self.dsl_end_output_suffix, + ])) + + if self.impl_file: + impl_printer.print() + self.impl_file.write("".join(impl_printer.output)) + + self.next(self.state_reset) + + +clinic = None + +def main(argv): + global clinic + import argparse + cmdline = argparse.ArgumentParser() + cmdline.add_argument("-f", "--force", action='store_true') + cmdline.add_argument("-o", "--output", type=str) + cmdline.add_argument("filename", type=str, nargs="+") + ns = cmdline.parse_args(argv) + if ns.output and (len(ns.filename) > 1): + sys.exit("Error: can't use -o with multiple filenames") + for filename in ns.filename: + clinic = Clinic(verify=not (ns.force or ns.output)) + clinic.read(filename) + clinic.write(ns.output or filename) + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:]))