diff -Naur old/Doc/lib/lib.tex new/Doc/lib/lib.tex --- old/Doc/lib/lib.tex 2006-08-21 10:42:42.000000000 +0200 +++ new/Doc/lib/lib.tex 2006-08-23 11:22:06.000000000 +0200 @@ -263,6 +263,7 @@ \input{libspwd} \input{libgrp} \input{libcrypt} +\input{libdirectio} \input{libdl} \input{libtermios} \input{libtty} diff -Naur old/Doc/lib/libdirectio.tex new/Doc/lib/libdirectio.tex --- old/Doc/lib/libdirectio.tex 1970-01-01 01:00:00.000000000 +0100 +++ new/Doc/lib/libdirectio.tex 2006-08-23 11:21:08.000000000 +0200 @@ -0,0 +1,77 @@ +\section{\module{directio} --- + Interface to open(), read(), write() on a direct I/O context} + +\declaremodule{extension}{directio} + \platform{Linux} +\moduleauthor{Omar AitMous}{Omar.AitMous@gmail.com} +\sectionauthor{Omar AitMous}{Omar.AitMous@gmail.com} + +\modulesynopsis{An interface to file read and write operations with the +O\_DIRECT flag.} + + +The \module{directio} is an interface to the open(),read(),write() and close() +system calls on a Direct I/O context (O_DIRECT flag). + +The O_DIRECT flag allows to open a file in "direct" mode -- +effectively bypassing the buffer-cache. This behaviour is generally +not a good thing since it will strongly degrade performances, but it +can be desirable for some purposes (benchmark, or optimized read of +many concurrent video streams, for instance). + +A special interface is needed, since the read() and write() system calls +requires aligned buffers when used on O_DIRECT file descriptors. One does not +have to care about buffers beeing aligned since this module align buffers +before calling to read() and write(). Still, transfer sizes must be multiples +of the logical block size of the file system. + +This module should work on SGI IRIX and FreeBSD 4.x too, but it hasn't been +tested. Support for O_DIRECT was added under Linux in kernel version 2.4.10. +Older Linux kernels simply ignore this flag. + + +The \module{directio} module defines the following functions: + +\begin{funcdesc}{open}{pathname, flags\optional{, mode}} +Given a \var{pathname}, \function{open} returns a file descriptor, a small, non +negative integer for use in subsequent calls to \function{read} and +\function{write} and set various flags according to \var{flags} and possibly +its mode according to \var{mode}. The O_DIRECT flag is automatically added +to the \var{flags} argument so as to try to minimize cache effects of the I/O +to and from this file. In general, this will degrade performance, but it is +useful in special. +\end{funcdesc} + +\begin{funcdesc}{read}{fd, n} +\function{read} attempts to read up to \var{n} bytes from the file descriptor +\var{fd} into a string. +Upon success, \function{read} returns a string containing the bytes read, or an +empty one if the end of file has been reached. +\end{funcdesc} + +\begin{funcdesc}{write}{fd, str} +\function{write} writes the buffer \var{str} to the file descriptor \var{fd} +and returns the number of bytes actually written. +\end{funcdesc} + +\begin{funcdesc}{close}{fd} +\function{close} closees a file descriptor \var{fd}. +\end{funcdesc} + +The following data items are available for use in constructing the \var{flags} +parameter to the \function{open} function. +One of the following three 'modes' must be included : +\begin{datadesc}{O_RDONLY} +\dataline{O_WRONLY} +\dataline{O_RDWR} +\end{datadesc} +Other flags available are : +\begin{datadesc}{O_CREAT} +\dataline{O_APPEND} +\dataline{O_TRUNC} +\dataline{O_EXCL} +\end{datadesc} +\begin{note} +The flag O_DIRECT is not available in the flags since the module adds it +automatticly to the \var{flags} given to \function{read}. +\end{note} diff -Naur old/Lib/test/regrtest.py new/Lib/test/regrtest.py --- old/Lib/test/regrtest.py 2006-08-21 10:44:01.000000000 +0200 +++ new/Lib/test/regrtest.py 2006-08-23 11:22:39.000000000 +0200 @@ -817,6 +817,7 @@ test_crypt test_curses test_dbm + test_directio test_dl test_fcntl test_fork1 @@ -877,6 +878,7 @@ test_crypt test_curses test_dbm + test_directio test_dl test_fcntl test_fork1 @@ -915,6 +917,7 @@ test_bsddb185 test_cd test_cl + test_directio test_dl test_gl test_imgfile @@ -939,6 +942,7 @@ test_bsddb185 test_cd test_cl + test_directio test_dl test_gl test_imgfile @@ -964,6 +968,7 @@ test_bsddb185 test_cd test_cl + test_directio test_dl test_fork1 test_gettext @@ -1002,6 +1007,7 @@ test_commands test_crypt test_dbm + test_directio test_dl test_fcntl test_fork1 @@ -1040,6 +1046,7 @@ test_cd test_cl test_curses + test_directio test_gdbm test_gl test_imgfile @@ -1065,6 +1072,7 @@ test_cl test_curses test_dbm + test_directio test_gdbm test_gl test_gzip @@ -1085,6 +1093,7 @@ test_cd test_cl test_curses + test_directio test_dl test_gdbm test_gl @@ -1113,6 +1122,7 @@ test_cd test_cl test_curses + test_directio test_dl test_gdbm test_gl @@ -1140,6 +1150,7 @@ test_cl test_curses test_dbm + test_directio test_gl test_imgfile test_ioctl @@ -1163,6 +1174,7 @@ test_cl test_commands test_curses + test_directio test_dl test_gl test_imgfile @@ -1225,6 +1237,7 @@ test_bz2 test_cd test_cl + test_directio test_dl test_gdbm test_gl @@ -1254,6 +1267,7 @@ test_cd test_cl test_ctypes + test_directio test_dl test_gdbm test_gl @@ -1288,6 +1302,7 @@ test_cl test_ctypes test_curses + test_directio test_dl test_gdbm test_gl diff -Naur old/Lib/test/test_directio.py new/Lib/test/test_directio.py --- old/Lib/test/test_directio.py 1970-01-01 01:00:00.000000000 +0100 +++ new/Lib/test/test_directio.py 2006-08-23 11:21:08.000000000 +0200 @@ -0,0 +1,36 @@ +import unittest +import directio +import os +import resource +from test import test_support + +class DirectioModuleTest(unittest.TestCase): + def read_what_was_written(self, fd, buf): + buf_tmp = "" + while len(buf_tmp) < len(buf): + buf_tmp += directio.read(fd, len(buf) - len(buf_tmp)) + if buf_tmp != buf: + self.fail("failed in reading what has been written"%(buf_tmp,buf)) + + def test_read_write(self): + flags = directio.O_RDWR | directio.O_CREAT + fd = directio.open("tmp_directio_file", flags, 0644) + buf = "a"*resource.getpagesize() + sent = 0 + while sent < len(buf): + count = directio.write(fd, buf[sent:]) + sent += count + os.lseek(fd, 0, 0) + self.read_what_was_written(fd, buf) + directio.close(fd) + os.unlink("tmp_directio_file") + + + +def test_main(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(DirectioModuleTest)) + test_support.run_suite(suite) + +if __name__ == "__main__": + test_main() diff -Naur old/Modules/directiomodule.c new/Modules/directiomodule.c --- old/Modules/directiomodule.c 1970-01-01 01:00:00.000000000 +0100 +++ new/Modules/directiomodule.c 2006-08-23 11:21:08.000000000 +0200 @@ -0,0 +1,293 @@ +/* py-directio file : directiomodule.c + A Python module interface to 'open', 'read' and 'write' on a + direct I/O context. + + This is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with the GNU C Library; if not, write to the Free + Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA + 02111-1307 USA. */ + +#include + +#include +#include +#include +#include + +#include +#include + +static unsigned int alignment; + +static inline void get_alignment_size(void) { + struct utsname tmp; + char twodotfour[] = "2.4", twodotsix[] = "2.6"; + int ret = 0; + + Py_BEGIN_ALLOW_THREADS; + ret == uname(&tmp); + Py_END_ALLOW_THREADS; + + if(ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + goto end; + } + if(!memcmp(twodotfour,tmp.release,3)) { + if(atoi((tmp.release)+4) < 10) + goto ignored; + alignment = 4096; + } + else if(!memcmp(twodotsix,tmp.release,3)) + alignment = 512; + else + goto ignored; + goto end; + +ignored: + alignment = 0; + PyErr_Warn(NULL,"the O_DIRECT flag is being ignored (Linux kernel release \ + should be 2.4.10 or higher).\nThis module's methods will work exactly the \ + same as the os module ones."); +end: + ; +} + +static PyObject * +method_open(PyObject *self, PyObject *args) +{ + int fd; + char *pathname; + int flags; + mode_t mode; + + if(!PyArg_ParseTuple(args, "si|i", &pathname, &flags, &mode)) + return NULL; + + Py_BEGIN_ALLOW_THREADS; + fd = open(pathname, flags | O_DIRECT, mode); + Py_END_ALLOW_THREADS; + + if(fd == -1) { + PyErr_SetFromErrnoWithFilename(PyExc_OSError, pathname); + return NULL; + } else { + return Py_BuildValue("i", fd); + } +} + +static PyObject * +method_close(PyObject *self, PyObject *args) +{ + int fd; + int ret; + + if(!PyArg_ParseTuple(args, "i", &fd)) + return NULL; + + Py_BEGIN_ALLOW_THREADS; + ret = close(fd); + Py_END_ALLOW_THREADS; + + if(ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } else { + return Py_BuildValue(""); + } +} + +static PyObject * +method_read(PyObject *self, PyObject *args) +{ + int fd; + char *buff = NULL, *realbuff = NULL, *alignedbuff = NULL; + size_t count; + + ssize_t ret; + + if(!PyArg_ParseTuple(args, "iI", &fd, &count)) + return NULL; + + if(count < 0) { + PyErr_SetString(PyExc_ValueError, "transfer size must be positive."); + return NULL; + } + + if(alignment && count%alignment) { + if(alignment == 512) + PyErr_SetString(PyExc_ValueError, "transfer size must be a multiple of \ + a 512."); + else + PyErr_SetString(PyExc_ValueError, "transfer size must be a multiple of \ + 4096."); + return NULL; + } + + /* alloc and align */ + if(!(realbuff = calloc(alignment + count, 1))) { + return PyErr_NoMemory(); + } + if(alignment) + alignedbuff = (char*)((((unsigned int)realbuff + alignment - 1)/alignment)\ + *alignment); + else + alignedbuff = realbuff; + + Py_BEGIN_ALLOW_THREADS; + ret = read(fd, alignedbuff, count); + Py_END_ALLOW_THREADS; + + + if(ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + free(realbuff); + return NULL; + } else if(ret){ + if(!(buff = calloc(ret+1, 1))) { + free(realbuff); + return PyErr_NoMemory(); + } + memcpy(buff, alignedbuff, ret); + free(realbuff); + return Py_BuildValue("s", buff); + } else { + if(!(buff = calloc(1, 1))) { + free(realbuff); + return PyErr_NoMemory(); + } + buff[0] = '\0'; + return Py_BuildValue("s", buff); + } +} + +static PyObject * +method_write(PyObject *self, PyObject *args) +{ + int fd; + char *buff = NULL, *realbuff = NULL, *alignedbuff = NULL; + int count = 0; + ssize_t ret = 0; + + if(!PyArg_ParseTuple(args, "is", &fd, &buff)) + return NULL; + count = strlen(buff); + + if(alignment && count%alignment) { + PyErr_SetString(PyExc_ValueError, "transfer size must be a multiple of \ + the logical block size of the file system."); + return NULL; + } + + /* alloc and align */ + if(!(realbuff = calloc(alignment + count, 1))) { + return PyErr_NoMemory(); + } + alignedbuff = (char*)((((unsigned int)realbuff + alignment - 1)/alignment)\ + *alignment); + memcpy(alignedbuff, buff, count); + + Py_BEGIN_ALLOW_THREADS; + ret = write(fd, alignedbuff, count); + Py_END_ALLOW_THREADS; + + free(realbuff); + + if(ret == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + return Py_BuildValue("i", ret); +} + +static PyMethodDef SpliceTeeMethods[] = { + {"open", method_open, METH_VARARGS, +"open(pathname, flags[, mode]) = fd\n" +"\n" +"Given a pathname for a file, open() returns a file descriptor, a small, \ +non-negative integer for use in subsequent system calls (read(2), write(2), \ +lseek(2), fcntl(2), etc.). The file descriptor returned by a successful call \ +will be the lowest-numbered file descriptor not currently open for the \ +process.\n" +"The O_DIRECT flag is automatically added to the 'flags' in argument so as to \ +try to minimize cache effects of the I/O to and from this file. In general \ +this will degrade performance, but it is useful in special situations, such \ +as when applications do their own caching. File I/O is done directly to/from \ +user space buffers. The I/O is synchronous, i.e., at the completion of a \ +read(2) or write(2), data is guaranteed to have been transferred.\n" +"Under Linux 2.4 transfer sizes, and the alignment of user buffer and file \ +offset must all be multiples of the logical block size of the file system.\n" +"Under Linux 2.6 alignment to 512-byte boundaries suffices.\n" +"Note that on a NFS file system, O_DIRECT flag is ignored.\n" +"\n" +"Upon success, 'open' returns the new file descriptor.\n" + }, + {"read", method_read, METH_VARARGS, +"read(fd, cout) = string\n" +"\n" +"read() attempts to read up to count bytes from file descriptor fd into a \ +buffer.\n" +"\n" +"Upon success, 'read' returns a buffer containing the bytes read.\n" + }, + {"write", method_write, METH_VARARGS, +"write(fd,buf) = sent\n" +"\n" +" write() writes up to count bytes to the file referenced by the file \ +descriptor fd from the buffer 'buf'.\n" +"On a direct I/O context, this will result in data stored directly in hd, \ +ignoring the buffer cache.\n" + }, + {"close", method_close, METH_VARARGS, +"close(fd)\n" +"\n" +"close() closes a file descriptor, so that it no longer refers to any file \ +and may be reused." + }, + {NULL, NULL, 0, NULL} /* Sentinel */ +}; + +static void +insint (PyObject *d, char *name, int value) +{ + PyObject *v = PyInt_FromLong((long) value); + if (!v || PyDict_SetItemString(d, name, v)) + PyErr_Clear(); + + Py_XDECREF(v); +} + +PyMODINIT_FUNC +initdirectio(void) +{ + PyObject *m, *d; + + get_alignment_size(); + + m = Py_InitModule("directio", SpliceTeeMethods); + if(!m) + return; + + d = PyModule_GetDict (m); + + insint (d, "O_RDONLY", O_RDONLY); + insint (d, "O_WRONLY", O_WRONLY); + insint (d, "O_RDWR", O_RDWR); + insint (d, "O_APPEND", O_APPEND); + insint (d, "O_CREAT", O_CREAT); + insint (d, "O_EXCL", O_EXCL); + insint (d, "O_TRUNC", O_TRUNC); + + PyModule_AddStringConstant(m, "__doc__", "Direct interface to 'open', 'read'\ + , 'write' and 'close' system calls on a direct I/O context."); + PyModule_AddStringConstant(m, "__version__", "1.0"); +} diff -Naur old/setup.py new/setup.py --- old/setup.py 2006-08-21 10:45:00.000000000 +0200 +++ new/setup.py 2006-08-23 11:24:20.000000000 +0200 @@ -887,6 +887,9 @@ if platform not in ['mac', 'win32']: # Steen Lumholt's termios module exts.append( Extension('termios', ['termios.c']) ) + # Omar AitMous's read/write module on a Direct I/O context + if platform == 'linux2': + exts.append( Extension('directio', ['directiomodule.c']) ) # Jeremy Hylton's rlimit interface if platform not in ['atheos']: exts.append( Extension('resource', ['resource.c']) )