diff --git a/Lib/select.py b/Lib/select.py new file mode 100644 --- /dev/null +++ b/Lib/select.py @@ -0,0 +1,262 @@ +"""Select module. + +This module supports asynchronous I/O on multiple file descriptors. +""" + + +from _select import * + + +# genereric events, that must be mapped to implementation-specific ones +SELECT_IN = (1 << 0) +SELECT_OUT = (1 << 1) +SELECT_ERR = (1 << 2) + + +class _BaseSelector: + """Base selector class. + + A selector supports registering and unregistering file descriptors, and then + polling them for I/O events. A selector can use various implementations + (select(), poll(), epoll()...) depending on the platform. The default + `Selector` class uses the most performant implementation on the current + platform. + """ + + def __init__(self): + # this maps file-descriptor to file object + self._readers = {} + self._writers = {} + + def register(self, fileobj, eventmask): + """Register a file object. + + Parameters: + fileobj -- file descriptor, or any object with a `fileno()` method + eventmask -- bitwise mask of events to monitor + """ + # find the corresponding FD + fd = self._file_to_fd(fileobj) + if eventmask & SELECT_IN: + self._readers[fd] = fileobj + if eventmask & SELECT_OUT: + self._writers[fd] = fileobj + return fd + + def unregister(self, fileobj): + """Unregister a file object. + + Parameters: + fileobj -- file descriptor, or any object with a `fileno()` method + """ + fd = self._file_to_fd(fileobj) + if fd not in self._readers and fd not in self._writers: + raise ValueError("Cannot unregister non registered file: " + "{!r}".format(fileobj)) + for store in self._readers, self._writers: + try: + del store[fd] + except KeyError: + pass + return fd + + def modify(self, fileobj, eventmask): + """Change the events to monitor for a given file object. + + Parameters: + fileobj -- file descriptor, or any object with a `fileno()` method + """ + self.unregister(fileobj) + self.register(fileobj, eventmask) + + def select(self, timeout=None): + """Perform the actual selection, until some monitored file objects are + ready or a timeout expires. + + Parameters: + timeout -- if timeout > 0, this specifies the maximum wait time, in + seconds + if timeout == 0, the select() call won't block, and will + report the currently ready files + if timeout is None, select() will block until a monitored + file descriptor becomes ready + + Returns: + list of (fileobj, event mask) for ready file object + """ + raise NotImplementedError() + + def close(self): + """Close the selector. + + This must be called to make sure that any underlying resource is freed. + """ + self._readers.clear() + self._writers.clear() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + @staticmethod + def _file_to_fd(fileobj): + """Helper method that returns the file descriptor for a given file + object. + + Parameters: + fileobj -- file descriptor, or any object with a `fileno()` method + + Returns: + corresponding FD + """ + if isinstance(fileobj, int): + fd = fileobj + else: + try: + fd = int(fileobj.fileno()) + except (ValueError, TypeError): + raise ValueError("Invalid file object: {!r}".format(fileobj)) + return fd + + def _fd_to_file(self, fd): + """Helper method that returns the file object for a given file + descriptor. + + Parameters: + fd -- file descriptor + + Returns: + corresponding file object, or fd + """ + for store in self._readers, self._writers: + try: + return store[fd] + except KeyError: + pass + return fd + + +class SelectSelector(_BaseSelector): + """Select-based selector.""" + + def __init__(self): + super().__init__() + + def select(self, timeout=None): + r, w, e = select(self._readers, self._writers, + self._readers.keys()|self._writers.keys(), timeout) + r = set(r) + w = set(w) + e = set(e) + ready = [] + for fd in r | w | e: + mask = 0 + if fd in r: + mask |= SELECT_IN + if fd in w: + mask |= SELECT_OUT + if fd in e: + # exceptional condition, report as read-ready + mask |= SELECT_IN + ready.append((self._fd_to_file(fd), mask)) + return ready + + +class PollSelector(_BaseSelector): + """Poll-based selector.""" + + def __init__(self): + super().__init__() + self._poll = poll() + + def _update(self, fd): + mask = 0 + if fd in self._readers: + mask |= POLLIN|POLLPRI + if fd in self._writers: + mask |= POLLOUT + if mask: + self._poll.register(fd, mask) + else: + self._poll.unregister(fd) + + def register(self, fileobj, eventmask): + fd = super().register(fileobj, eventmask) + self._update(fd) + + def unregister(self, fileobj): + fd = super().unregister(fileobj) + self._update(fd) + + def select(self, timeout=None): + timeout = None if timeout is None else int(1000 * timeout) + ready = [] + for fd, event in self._poll.poll(timeout): + mask = 0 + if event & (POLLIN|POLLPRI|POLLHUP): + mask |= SELECT_IN + if event & POLLOUT: + mask |= SELECT_OUT + if event & (POLLNVAL|POLLERR): + mask |= SELECT_ERR + + ready.append((self._fd_to_file(fd), mask)) + return ready + + +class EpollSelector(_BaseSelector): + """Epoll-based selector.""" + + def __init__(self): + super().__init__() + self._epoll = epoll() + + def _update(self, fd): + mask = 0 + if fd in self._readers: + mask |= EPOLLIN|EPOLLPRI + if fd in self._writers: + mask |= EPOLLOUT + if mask: + self._epoll.register(fd, mask) + else: + self._epoll.unregister(fd) + + def register(self, fileobj, eventmask): + fd = super().register(fileobj, eventmask) + self._update(fd) + + def unregister(self, fileobj): + fd = super().unregister(fileobj) + self._update(fd) + + def select(self, timeout=None): + timeout = -1 if timeout is None else timeout + ready = [] + for fd, event in self._epoll.poll(timeout): + mask = 0 + if event & (EPOLLIN|EPOLLPRI|EPOLLHUP): + mask |= SELECT_IN + if event & EPOLLOUT: + mask |= SELECT_OUT + if event & EPOLLERR: + mask |= SELECT_ERR + + ready.append((self._fd_to_file(fd), mask)) + return ready + + def close(self): + super().close() + self._epoll.close() + + +# Choose the best implementation: roughly, epoll|kqueue > poll > select. +# select() also can't accept a FD > FD_SETSIZE (usually around 1024) +if 'epoll' in globals(): + Selector = EpollSelector +elif 'poll' in globals(): + Selector = PollSelector +else: + Selector = SelectSelector diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -1,10 +1,20 @@ import errno import os +import random import select import sys import unittest from test import support + +def find_ready_matching(ready, flag): + match = [] + for fd, mode in ready: + if mode & flag: + match.append(fd) + return match + + @unittest.skipIf((sys.platform[:3]=='win'), "can't easily test on this system") class SelectTestCase(unittest.TestCase): @@ -75,9 +85,148 @@ a[:] = [F()] * 10 self.assertEqual(select.select([], a, []), ([], a[:5], [])) + +class BasicSelectorTestCase(unittest.TestCase): + + def test_constants(self): + select.SELECT_IN + select.SELECT_OUT + select.SELECT_ERR + + +class BaseSelectorTestCase(unittest.TestCase): + + def test_error_conditions(self): + s = self.SELECTOR() + self.assertRaises(TypeError, s.register) + self.assertRaises(TypeError, s.register, 0) + self.assertRaises(TypeError, s.register, 0, 1, 2) + self.assertRaises(TypeError, s.unregister, 0, 1) + self.assertRaises(TypeError, s.modify, 0) + self.assertRaises(TypeError, s.select, 0, 1) + + def test_basic(self): + with self.SELECTOR() as s: + rd, wr = os.pipe() + self.addCleanup(os.close, rd) + self.addCleanup(os.close, wr) + + s.register(rd, select.SELECT_IN) + self.assertFalse(s.select(0.1)) + + s.register(wr, select.SELECT_OUT) + self.assertEqual(set(((wr, select.SELECT_OUT),)), set(s.select())) + + s.unregister(rd) + s.unregister(wr) + + def test_selector(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + NUM_PIPES = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_PIPES): + rd, wr = os.pipe() + s.register(rd, select.SELECT_IN) + s.register(wr, select.SELECT_OUT) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = s.select() + ready_writers = find_ready_matching(ready, select.SELECT_OUT) + if not ready_writers: + self.fail("no pipes ready for writing") + wr = random.choice(ready_writers) + os.write(wr, MSG) + + ready = s.select() + ready_readers = find_ready_matching(ready, select.SELECT_IN) + if not ready_readers: + self.fail("no pipes ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = os.read(rd, MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + os.close(r2w[rd]) ; os.close(rd) + s.unregister(r2w[rd]) + s.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_PIPES) + + def test_timeout(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + s.register(p, select.SELECT_IN) + + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print('timeout =', tout) + + ready = s.select(tout) + if not ready: + continue + if set(ready) == set(((p, select.SELECT_IN),)): + line = p.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select(): %r' % ready) + p.close() + + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectorTestCase(BaseSelectorTestCase): + + SELECTOR = select.Selector + + +class SelectSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = select.SelectSelector + + +@unittest.skipUnless(hasattr(select, 'poll'), "Test needs select.poll()") +class PollSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = select.PollSelector + + +@unittest.skipUnless(hasattr(select, 'epoll'), "Test needs select.epoll()") +class EpollSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = select.EpollSelector + + def test_main(): - support.run_unittest(SelectTestCase) + tests = [SelectTestCase] + tests.extend([BasicSelectorTestCase, SelectorTestCase, + SelectSelectorTestCase, PollSelectorTestCase, + EpollSelectorTestCase]) + support.run_unittest(*tests) support.reap_children() + if __name__ == "__main__": test_main() diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2129,7 +2129,7 @@ static struct PyModuleDef selectmodule = { PyModuleDef_HEAD_INIT, - "select", + "_select", module_doc, -1, select_methods, @@ -2143,7 +2143,7 @@ PyMODINIT_FUNC -PyInit_select(void) +PyInit__select(void) { PyObject *m; m = PyModule_Create(&selectmodule); diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -623,7 +623,7 @@ missing.append('spwd') # select(2); not on ancient System V - exts.append( Extension('select', ['selectmodule.c']) ) + exts.append( Extension('_select', ['selectmodule.c']) ) # Fred Drake's interface to the Python parser exts.append( Extension('parser', ['parsermodule.c']) )