diff -r 57280a01aecf -r ebc7b147efd5 Doc/library/os.rst --- a/Doc/library/os.rst Thu Jan 27 21:38:46 2011 +0100 +++ b/Doc/library/os.rst Sat Jan 29 06:55:42 2011 +0200 @@ -775,6 +775,47 @@ :meth:`~file.readline` methods. +.. function:: sendfile(out, in, offset, nbytes) + sendfile(out, in, offset, nbytes, headers=None, trailers=None, flags=0) + + Copy *nbytes* bytes from file descriptor *in* to file descriptor *out*. + + The first case is supported by all platforms that define :func:`sendfile`. The + return value is a tuple, (byteswritten, offset), where offset is a value + pointing to the byte following the last byte read. + + On Linux, if *offset* is given as ``None``, the bytes are read from the + current position of *in* and the position of *in* is updated. It returns + the same as above with offset being ``None``. + + The second case may be used on Mac OS X and FreeBSD where *headers* and + *trailers* are arbitrary sequences of buffers that are written before and + after the data from *in* is written. It returns the same as the first case. + + On Mac OS X and FreeBSD, a value of 0 for *nbytes* specifies to send until + the end of *in* is reached. + + On Solaris, *out* may be the file descriptor of a regular file or the file + descriptor of a socket. On all other platforms, *out* must be the file + descriptor of an open socket. + + Availability: Unix. + + .. versionadded:: 3.3 + + +.. data:: SF_NODISKIO + SF_MNOWAIT + SF_SYNC + + Parameters to the :func:`sendfile` function, if the implementation supports + them. + + Availability: Unix. + + .. versionadded:: 3.3 + + .. function:: tcgetpgrp(fd) Return the process group associated with the terminal given by *fd* (an open diff -r 57280a01aecf -r ebc7b147efd5 Lib/test/test_posix.py --- a/Lib/test/test_posix.py Thu Jan 27 21:38:46 2011 +0100 +++ b/Lib/test/test_posix.py Sat Jan 29 06:55:42 2011 +0200 @@ -121,6 +121,35 @@ finally: fp.close() + @unittest.skipUnless(hasattr(posix, 'sendfile'), "test needs os.sendfile()") + def test_sendfile(self): + with open(support.TESTFN, "wb") as fp: + fp.write(b"testdata") + + import socket + serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + serv.bind(('localhost', 0)) + serv.listen(5) + cli = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + cli.connect(('localhost', serv.getsockname()[1])) + with open(support.TESTFN, "rb") as fp: + try: + self.assertEqual((5, 6), os.sendfile(cli.fileno(), fp.fileno(), 1, 5)) + recv = serv.accept()[0] + self.assertEqual(b"estda", recv.recv(1024)) + + if sys.platform.startswith('freebsd') or \ + sys.platform == 'darwin' or \ + sys.platform.startswith('dragonfly'): + self.assertEqual((13, 14), os.sendfile(cli.fileno(), fp.fileno(), + 1, 5, headers=(b'aa', b'bb'), trailers=(b'yy', b'zz'))) + self.assertEqual(b"aabbestdayyzz", recv.recv(1024)) + + finally: + cli.close() + serv.close() + recv.close() + def test_ftruncate(self): if hasattr(posix, 'ftruncate'): fp = open(support.TESTFN, 'w+') diff -r 57280a01aecf -r ebc7b147efd5 Modules/posixmodule.c --- a/Modules/posixmodule.c Thu Jan 27 21:38:46 2011 +0100 +++ b/Modules/posixmodule.c Sat Jan 29 06:55:42 2011 +0200 @@ -95,6 +95,20 @@ #include #endif +#ifdef HAVE_SYS_SENDFILE_H +#include +#endif + +#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) +#ifdef HAVE_SYS_SOCKET_H +#include +#endif + +#ifdef HAVE_SYS_UIO_H +#include +#endif +#endif + /* Various compilers have only certain posix functions */ /* XXX Gosh I wish these were all moved into pyconfig.h */ #if defined(PYCC_VACPP) && defined(PYOS_OS2) @@ -349,6 +363,20 @@ #endif #endif +int +PyParse_off_t(PyObject* arg, void* addr) +{ +#if !defined(HAVE_LARGEFILE_SUPPORT) + *((off_t*)addr) = PyLong_AsLong(arg); +#else + *((off_t*)addr) = PyLong_Check(arg) ? PyLong_AsLongLong(arg) + : PyLong_AsLong(arg); +#endif + if (PyErr_Occurred()) + return 0; + return 1; +} + #if defined _MSC_VER && _MSC_VER >= 1400 /* Microsoft CRT in VS2005 and higher will verify that a filehandle is * valid and throw an assertion if it isn't. @@ -5723,6 +5751,175 @@ return PyLong_FromSsize_t(size); } +#ifdef HAVE_SENDFILE +#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) +static int +iov_setup(struct iovec **iov, Py_buffer **buf, PyObject *seq, int cnt, int type) +{ + int i, j; + *iov = PyMem_New(struct iovec, cnt); + if (*iov == NULL) { + PyErr_NoMemory(); + return 0; + } + *buf = PyMem_New(Py_buffer, cnt); + if (*buf == NULL) { + PyMem_Del(*iov); + PyErr_NoMemory(); + return 0; + } + + for (i = 0; i < cnt; i++) { + if (PyObject_GetBuffer(PySequence_GetItem(seq, i), &(*buf)[i], + type) == -1) { + PyMem_Del(*iov); + for (j = 0; j < i; j++) { + PyBuffer_Release(&(*buf)[j]); + } + PyMem_Del(*buf); + return 0; + } + (*iov)[i].iov_base = (*buf)[i].buf; + (*iov)[i].iov_len = (*buf)[i].len; + } + return 1; +} + +static void +iov_cleanup(struct iovec *iov, Py_buffer *buf, int cnt) +{ + int i; + PyMem_Del(iov); + for (i = 0; i < cnt; i++) { + PyBuffer_Release(&buf[i]); + } + PyMem_Del(buf); +} +#endif + +PyDoc_STRVAR(posix_sendfile__doc__, +"sendfile(out, in, offset, nbytes) -> (byteswritten, offset)\n\ +sendfile(out, in, offset, nbytes, headers=None, trailers=None, flags=0)\n\ + -> (byteswritten, offset)\n\ +Copy nbytes bytes from file descriptor in to file descriptor out."); + +static PyObject * +posix_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) +{ + int in, out; + Py_ssize_t ret; + off_t offset; + +#if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) +#ifndef __APPLE__ + Py_ssize_t len; +#endif + PyObject *headers = NULL, *trailers = NULL; + Py_buffer *hbuf, *tbuf; + off_t sbytes; + struct sf_hdtr sf; + int flags = 0; + sf.headers = NULL; + sf.trailers = NULL; + static char *keywords[] = {"out", "in", + "offset", "count", + "headers", "trailers", "flags", NULL}; + +#ifdef __APPLE__ + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiO&O&|OOi:sendfile", + keywords, &out, &in, PyParse_off_t, &offset, PyParse_off_t, &sbytes, +#else + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiO&n|OOi:sendfile", + keywords, &out, &in, PyParse_off_t, &offset, &len, +#endif + &headers, &trailers, &flags)) + return NULL; + if (headers != NULL) { + if (!PySequence_Check(headers)) { + PyErr_SetString(PyExc_TypeError, + "sendfile() headers must be a sequence or None"); + return NULL; + } else { + sf.hdr_cnt = PySequence_Size(headers); + if (sf.hdr_cnt > 0 && !iov_setup(&(sf.headers), &hbuf, + headers, sf.hdr_cnt, PyBUF_SIMPLE)) + return NULL; + } + } + if (trailers != NULL) { + if (!PySequence_Check(trailers)) { + PyErr_SetString(PyExc_TypeError, + "sendfile() trailers must be a sequence or None"); + return NULL; + } else { + sf.trl_cnt = PySequence_Size(trailers); + if (sf.trl_cnt > 0 && !iov_setup(&(sf.trailers), &tbuf, + trailers, sf.trl_cnt, PyBUF_SIMPLE)) + return NULL; + } + } + + Py_BEGIN_ALLOW_THREADS +#ifdef __APPLE__ + ret = sendfile(in, out, offset, &sbytes, &sf, flags); +#else + ret = sendfile(in, out, offset, len, &sf, &sbytes, flags); +#endif + Py_END_ALLOW_THREADS + + if (sf.headers != NULL) + iov_cleanup(sf.headers, hbuf, sf.hdr_cnt); + if (sf.trailers != NULL) + iov_cleanup(sf.trailers, tbuf, sf.trl_cnt); + + if (ret < 0) { + if ((errno == EAGAIN) || (errno == EBUSY)) { +#if !defined(HAVE_LARGEFILE_SUPPORT) + return Py_BuildValue("ll", sbytes, offset + sbytes); +#else + return Py_BuildValue("LL", sbytes, offset + sbytes); +#endif + } + return posix_error(); + } +#if !defined(HAVE_LARGEFILE_SUPPORT) + return Py_BuildValue("ll", sbytes, offset + sbytes); +#else + return Py_BuildValue("LL", sbytes, offset + sbytes); +#endif +#else + Py_ssize_t count; + PyObject *offobj; + static char *keywords[] = {"out", "in", + "offset", "count", NULL}; + if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiOn:sendfile", + keywords, &out, &in, &offobj, &count)) + return NULL; +#ifdef linux + if (offobj == Py_None) { + Py_BEGIN_ALLOW_THREADS + ret = sendfile(out, in, NULL, count); + Py_END_ALLOW_THREADS + if (ret < 0) + return posix_error(); + Py_INCREF(Py_None); + return Py_BuildValue("nO", ret, Py_None); + } +#endif + PyParse_off_t(offobj, &offset); + Py_BEGIN_ALLOW_THREADS + ret = sendfile(out, in, &offset, count); + Py_END_ALLOW_THREADS + if (ret < 0) + return posix_error(); +#if !defined(HAVE_LARGEFILE_SUPPORT) + return Py_BuildValue("nl", ret, offset); +#else + return Py_BuildValue("nL", ret, offset); +#endif +#endif +} +#endif PyDoc_STRVAR(posix_fstat__doc__, "fstat(fd) -> stat result\n\n\ @@ -7971,6 +8168,10 @@ {"lseek", posix_lseek, METH_VARARGS, posix_lseek__doc__}, {"read", posix_read, METH_VARARGS, posix_read__doc__}, {"write", posix_write, METH_VARARGS, posix_write__doc__}, +#ifdef HAVE_SENDFILE + {"sendfile", (PyCFunction)posix_sendfile, METH_VARARGS | METH_KEYWORDS, + posix_sendfile__doc__}, +#endif {"fstat", posix_fstat, METH_VARARGS, posix_fstat__doc__}, {"isatty", posix_isatty, METH_VARARGS, posix_isatty__doc__}, #ifdef HAVE_PIPE @@ -8362,6 +8563,17 @@ if (ins(d, "ST_NOSUID", (long)ST_NOSUID)) return -1; #endif /* ST_NOSUID */ + /* FreeBSD sendfile() constants */ +#ifdef SF_NODISKIO + if (ins(d, "SF_NODISKIO", (long)SF_NODISKIO)) return -1; +#endif +#ifdef SF_MNOWAIT + if (ins(d, "SF_MNOWAIT", (long)SF_MNOWAIT)) return -1; +#endif +#ifdef SF_SYNC + if (ins(d, "SF_SYNC", (long)SF_SYNC)) return -1; +#endif + #ifdef HAVE_SPAWNV #if defined(PYOS_OS2) && defined(PYCC_GCC) if (ins(d, "P_WAIT", (long)P_WAIT)) return -1; diff -r 57280a01aecf -r ebc7b147efd5 configure.in --- a/configure.in Thu Jan 27 21:38:46 2011 +0100 +++ b/configure.in Sat Jan 29 06:55:42 2011 +0200 @@ -1291,10 +1291,10 @@ unistd.h utime.h \ sys/audioio.h sys/bsdtty.h sys/epoll.h sys/event.h sys/file.h sys/loadavg.h \ sys/lock.h sys/mkdev.h sys/modem.h \ -sys/param.h sys/poll.h sys/select.h sys/socket.h sys/statvfs.h sys/stat.h \ -sys/termio.h sys/time.h \ -sys/times.h sys/types.h sys/un.h sys/utsname.h sys/wait.h pty.h libutil.h \ -sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ +sys/param.h sys/poll.h sys/select.h sys/sendfile.h sys/socket.h sys/statvfs.h \ +sys/stat.h sys/termio.h sys/time.h \ +sys/times.h sys/types.h sys/uio.h sys/un.h sys/utsname.h sys/wait.h pty.h \ +libutil.h sys/resource.h netpacket/packet.h sysexits.h bluetooth.h \ bluetooth/bluetooth.h linux/tipc.h spawn.h util.h) AC_HEADER_DIRENT AC_HEADER_MAJOR @@ -1893,6 +1893,7 @@ # checks for libraries +AC_CHECK_LIB(sendfile, sendfile) AC_CHECK_LIB(dl, dlopen) # Dynamic linking for SunOS/Solaris and SYSV AC_CHECK_LIB(dld, shl_load) # Dynamic linking for HP-UX @@ -2541,7 +2542,7 @@ initgroups kill killpg lchmod lchown lstat mbrtowc mkfifo mknod mktime \ mremap nice pathconf pause plock poll pthread_init \ putenv readlink realpath \ - select sem_open sem_timedwait sem_getvalue sem_unlink setegid seteuid \ + select sem_open sem_timedwait sem_getvalue sem_unlink sendfile setegid seteuid \ setgid \ setlocale setregid setreuid setresuid setresgid setsid setpgid setpgrp setuid setvbuf \ sigaction siginterrupt sigrelse snprintf strftime strlcpy \