diff --git a/Lib/select.py b/Lib/select.py new file mode 100644 --- /dev/null +++ b/Lib/select.py @@ -0,0 +1,249 @@ +"""Select module. + +This module supports asynchronous I/O on multiple file descriptors. +""" + + +from _select import * + + +# generic events, that must be mapped to implementation-specific ones +# read event +SELECT_IN = (1 << 0) +# write event +SELECT_OUT = (1 << 1) + + +class _BaseSelector: + """Base selector class. + + A selector supports registering file descriptors to be monitored for + specific I/O events. + + A selector can use various implementations (select(), poll(), epoll()...) + depending on the platform. The default `Selector` class uses the most + performant implementation on the current platform. + + An arbitrary object can be attached to the file descriptor, which can be + used for example to store context information, a callback, etc. + """ + + def __init__(self): + # this maps file-descriptor to attached data + self._readers = {} + self._writers = {} + + def register(self, fd, events, data=None): + """Register a file descriptor for specific I/O events. + + Parameters: + fd -- file descriptor + events -- events to monitor (bitwise mask of SELECT_IN|SELECT_OUT) + data -- attached data + """ + if (not events) or (events & ~(SELECT_IN|SELECT_OUT)): + raise ValueError("fd {}: invalid events: {}".format(fd, events)) + if events & SELECT_IN: + self._readers[fd] = data + if events & SELECT_OUT: + self._writers[fd] = data + + def unregister(self, fd): + """Unregister a file descriptor. + + Parameters: + fd -- file descriptor + """ + if fd not in self._readers and fd not in self._writers: + raise ValueError("Cannot unregister non registered " + "file descriptor: {}".format(fd)) + for store in self._readers, self._writers: + try: + del store[fd] + except KeyError: + pass + + def modify(self, fd, events, data=None): + """Change a registered file descriptor monitored I/O events or attached + data. + + Parameters: + fd -- file descriptor + events -- events to monitor (bitwise mask of SELECT_IN|SELECT_OUT) + data -- attached data + """ + self.unregister(fd) + self.register(fd, events, data) + + def select(self, timeout=None): + """Perform the actual selection, until some monitored file objects are + ready or a timeout expires. + + Parameters: + timeout -- if timeout > 0, this specifies the maximum wait time, in + seconds + if timeout == 0, the select() call won't block, and will + report the currently ready file descriptors + if timeout is None, select() will block until a monitored + file descriptor becomes ready + + Returns: + list of (fd, events, attached data) for ready file descriptors + `events` will be a bitwise mask of SELECT_IN|SELECT_OUT + """ + raise NotImplementedError() + + def close(self): + """Close the selector. + + This must be called to make sure that any underlying resource is freed. + """ + self._readers.clear() + self._writers.clear() + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def _fd_to_data(self, fd): + """Helper method that returns the data associated to a given file + descriptor. + + Parameters: + fd -- file descriptor + + Returns: + corresponding data, or None + """ + for store in self._readers, self._writers: + try: + return store[fd] + except KeyError: + pass + return None + + +class SelectSelector(_BaseSelector): + """Select-based selector.""" + + def select(self, timeout=None): + r, w, _ = select(self._readers, self._writers, [], timeout) + r = set(r) + w = set(w) + ready = [] + for fd in r | w: + events = 0 + if fd in r: + events |= SELECT_IN + if fd in w: + events |= SELECT_OUT + ready.append((fd, events, self._fd_to_data(fd))) + return ready + + +class PollSelector(_BaseSelector): + """Poll-based selector.""" + + def __init__(self): + super().__init__() + self._poll = poll() + + def _update(self, fd): + events = 0 + if fd in self._readers: + events |= POLLIN|POLLPRI + if fd in self._writers: + events |= POLLOUT + if events: + self._poll.register(fd, events) + else: + self._poll.unregister(fd) + + def register(self, fd, events, data=None): + super().register(fd, events, data) + self._update(fd) + + def unregister(self, fd): + super().unregister(fd) + self._update(fd) + + def select(self, timeout=None): + timeout = None if timeout is None else int(1000 * timeout) + ready = [] + for fd, event in self._poll.poll(timeout): + events = 0 + if event & (POLLHUP|POLLNVAL|POLLERR): + if fd in self._readers: + events |= SELECT_IN + if fd in self._writers: + events |= SELECT_OUT + else: + if event & (POLLIN|POLLPRI): + events |= SELECT_IN + if event & POLLOUT: + events |= SELECT_OUT + + ready.append((fd, events, self._fd_to_data(fd))) + return ready + + +class EpollSelector(_BaseSelector): + """Epoll-based selector.""" + + def __init__(self): + super().__init__() + self._epoll = epoll() + + def _update(self, fd): + events = 0 + if fd in self._readers: + events |= EPOLLIN|EPOLLPRI + if fd in self._writers: + events |= EPOLLOUT + if events: + self._epoll.register(fd, events) + else: + self._epoll.unregister(fd) + + def register(self, fd, events, data=None): + super().register(fd, events, data) + self._update(fd) + + def unregister(self, fd): + super().unregister(fd) + self._update(fd) + + def select(self, timeout=None): + timeout = -1 if timeout is None else timeout + ready = [] + for fd, event in self._epoll.poll(timeout): + events = 0 + if event & (EPOLLHUP|EPOLLERR): + if fd in self._readers: + events |= SELECT_IN + if fd in self._writers: + events |= SELECT_OUT + else: + if event & (EPOLLIN|EPOLLPRI): + events |= SELECT_IN + if event & EPOLLOUT: + events |= SELECT_OUT + + ready.append((fd, events, self._fd_to_data(fd))) + return ready + + def close(self): + super().close() + self._epoll.close() + + +# Choose the best implementation: roughly, epoll|kqueue > poll > select. +# select() also can't accept a FD > FD_SETSIZE (usually around 1024) +if 'epoll' in globals(): + Selector = EpollSelector +elif 'poll' in globals(): + Selector = PollSelector +else: + Selector = SelectSelector diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -1,10 +1,20 @@ import errno import os +import random import select import sys import unittest from test import support + +def find_ready_matching(ready, flag): + match = [] + for fd, mode, data in ready: + if mode & flag: + match.append(fd) + return match + + @unittest.skipIf((sys.platform[:3]=='win'), "can't easily test on this system") class SelectTestCase(unittest.TestCase): @@ -75,9 +85,166 @@ a[:] = [F()] * 10 self.assertEqual(select.select([], a, []), ([], a[:5], [])) + +class BasicSelectorTestCase(unittest.TestCase): + + def test_constants(self): + select.SELECT_IN + select.SELECT_OUT + + +class BaseSelectorTestCase(unittest.TestCase): + + def test_error_conditions(self): + s = self.SELECTOR() + self.assertRaises(TypeError, s.register) + self.assertRaises(TypeError, s.register, 0) + self.assertRaises(ValueError, s.register, 0, 18) + self.assertRaises(TypeError, s.unregister, 0, 1) + self.assertRaises(TypeError, s.modify, 0) + self.assertRaises(TypeError, s.select, 0, 1) + + def test_basic(self): + with self.SELECTOR() as s: + rd, wr = os.pipe() + self.addCleanup(os.close, rd) + self.addCleanup(os.close, wr) + + # test without attached data + s.register(wr, select.SELECT_OUT) + self.assertEqual(set(((wr, select.SELECT_OUT, None),)), set(s.select())) + + # unregister + s.unregister(wr) + + # test with attached data + s.register(wr, select.SELECT_OUT, sys.stdin) + self.assertEqual(set(((wr, select.SELECT_OUT, sys.stdin),)), set(s.select())) + + # modify + s.modify(wr, select.SELECT_OUT, sys.stdout) + self.assertEqual(set(((wr, select.SELECT_OUT, sys.stdout),)), set(s.select())) + + # test timeout + s.unregister(wr) + s.register(rd, select.SELECT_IN) + self.assertFalse(s.select(0.1)) + s.register(wr, select.SELECT_OUT) + self.assertEqual(set(((wr, select.SELECT_OUT, None),)), + set(s.select(0.1))) + + s.unregister(rd) + s.unregister(wr) + # unregistering twice should raise an error + self.assertRaises(ValueError, s.unregister, wr) + + def test_selector(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + NUM_PIPES = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_PIPES): + rd, wr = os.pipe() + s.register(rd, select.SELECT_IN) + s.register(wr, select.SELECT_OUT) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = s.select() + ready_writers = find_ready_matching(ready, select.SELECT_OUT) + if not ready_writers: + self.fail("no pipes ready for writing") + wr = random.choice(ready_writers) + os.write(wr, MSG) + + ready = s.select() + ready_readers = find_ready_matching(ready, select.SELECT_IN) + if not ready_readers: + self.fail("no pipes ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = os.read(rd, MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + os.close(r2w[rd]) ; os.close(rd) + s.unregister(r2w[rd]) + s.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_PIPES) + + def test_timeout(self): + s = self.SELECTOR() + self.addCleanup(s.close) + + cmd = 'for i in 0 1 2 3 4 5 6 7 8 9; do echo testing...; sleep 1; done' + p = os.popen(cmd, 'r') + s.register(p.fileno(), select.SELECT_IN, p) + + for tout in (0, 1, 2, 4, 8, 16) + (None,)*10: + if support.verbose: + print('timeout =', tout) + + ready = s.select(tout) + if not ready: + continue + if set(ready) == set(((p.fileno(), select.SELECT_IN, p),)): + line = p.readline() + if support.verbose: + print(repr(line)) + if not line: + if support.verbose: + print('EOF') + break + continue + self.fail('Unexpected return values from select(): %r' % ready) + p.close() + + +@unittest.skipIf((sys.platform[:3]=='win'), + "can't easily test on this system") +class SelectorTestCase(BaseSelectorTestCase): + + SELECTOR = select.Selector + + +class SelectSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = select.SelectSelector + + +@unittest.skipUnless(hasattr(select, 'poll'), "Test needs select.poll()") +class PollSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = select.PollSelector + + +@unittest.skipUnless(hasattr(select, 'epoll'), "Test needs select.epoll()") +class EpollSelectorTestCase(BaseSelectorTestCase): + + SELECTOR = select.EpollSelector + + def test_main(): - support.run_unittest(SelectTestCase) + tests = [SelectTestCase] + tests.extend([BasicSelectorTestCase, SelectorTestCase, + SelectSelectorTestCase, PollSelectorTestCase, + EpollSelectorTestCase]) + support.run_unittest(*tests) support.reap_children() + if __name__ == "__main__": test_main() diff --git a/Modules/selectmodule.c b/Modules/selectmodule.c --- a/Modules/selectmodule.c +++ b/Modules/selectmodule.c @@ -2129,7 +2129,7 @@ static struct PyModuleDef selectmodule = { PyModuleDef_HEAD_INIT, - "select", + "_select", module_doc, -1, select_methods, @@ -2143,7 +2143,7 @@ PyMODINIT_FUNC -PyInit_select(void) +PyInit__select(void) { PyObject *m; m = PyModule_Create(&selectmodule); diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -623,7 +623,7 @@ missing.append('spwd') # select(2); not on ancient System V - exts.append( Extension('select', ['selectmodule.c']) ) + exts.append( Extension('_select', ['selectmodule.c']) ) # Fred Drake's interface to the Python parser exts.append( Extension('parser', ['parsermodule.c']) )