diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/argparse.rst --- a/Doc/library/argparse.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/library/argparse.rst Tue Dec 03 16:02:48 2013 +0100 @@ -1371,14 +1371,12 @@ >>> parser.parse_args(['--', '-f']) Namespace(foo='-f', one=None) -.. _prefix-matching: -Argument abbreviations (prefix matching) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Argument abbreviations +^^^^^^^^^^^^^^^^^^^^^^ The :meth:`~ArgumentParser.parse_args` method allows long options to be -abbreviated to a prefix, if the abbreviation is unambiguous (the prefix matches -a unique option):: +abbreviated if the abbreviation is unambiguous:: >>> parser = argparse.ArgumentParser(prog='PROG') >>> parser.add_argument('-bacon') @@ -1856,12 +1854,6 @@ >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) -.. warning:: - :ref:`Prefix matching ` rules apply to - :meth:`parse_known_args`. The parser may consume an option even if it's just - a prefix of one of its known options, instead of leaving it in the remaining - arguments list. - Customizing file parsing ^^^^^^^^^^^^^^^^^^^^^^^^ diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/asyncio-eventloop.rst --- a/Doc/library/asyncio-eventloop.rst Tue Dec 03 19:17:25 2013 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,357 +0,0 @@ -.. module:: asyncio - -.. _event-loop: - -Event loops -=========== - -The event loop is the central execution device provided by :mod:`asyncio`. -It provides multiple facilities, amongst which: - -* Registering, executing and cancelling delayed calls (timeouts) - -* Creating client and server :ref:`transports ` for various - kinds of communication - -* Launching subprocesses and the associated :ref:`transports ` - for communication with an external program - -* Delegating costly function calls to a pool of threads - -Event loop functions --------------------- - -The easiest way to get an event loop is to call the :func:`get_event_loop` -function. - -.. function:: get_event_loop() - - Get the event loop for current context. Returns an event loop object - implementing :class:`BaseEventLoop` interface, or raises an exception in case no - event loop has been set for the current context and the current policy does - not specify to create one. It should never return ``None``. - -.. function:: set_event_loop(loop) - - XXX - -.. function:: new_event_loop() - - XXX - - -Event loop policy ------------------ - -.. function:: get_event_loop_policy() - - XXX - -.. function:: set_event_loop_policy(policy) - - XXX - - -Run an event loop ------------------ - -.. method:: BaseEventLoop.run_forever() - - Run until :meth:`stop` is called. - -.. method:: BaseEventLoop.run_until_complete(future) - - Run until the :class:`Future` is done. - - If the argument is a coroutine, it is wrapped in a :class:`Task`. - - Return the Future's result, or raise its exception. - -.. method:: BaseEventLoop.is_running() - - Returns running status of event loop. - -.. method:: BaseEventLoop.stop() - - Stop running the event loop. - - Every callback scheduled before :meth:`stop` is called will run. - Callback scheduled after :meth:`stop` is called won't. However, those - callbacks will run if :meth:`run_forever` is called again later. - -.. method:: BaseEventLoop.close() - - Close the event loop. The loop should not be running. - - This clears the queues and shuts down the executor, but does not wait for - the executor to finish. - - This is idempotent and irreversible. No other methods should be called after - this one. - - -Calls ------ - -.. method:: BaseEventLoop.call_soon(callback, \*args) - - Arrange for a callback to be called as soon as possible. - - This operates as a FIFO queue, callbacks are called in the order in - which they are registered. Each callback will be called exactly once. - - Any positional arguments after the callback will be passed to the - callback when it is called. - -.. method:: BaseEventLoop.call_soon_threadsafe(callback, \*args) - - Like :meth:`call_soon`, but thread safe. - - -Delayed calls -------------- - -The event loop has its own internal clock for computing timeouts. -Which clock is used depends on the (platform-specific) event loop -implementation; ideally it is a monotonic clock. This will generally be -a different clock than :func:`time.time`. - -.. method:: BaseEventLoop.call_later(delay, callback, *args) - - Arrange for the *callback* to be called after the given *delay* - seconds (either an int or float). - - A "handle" is returned: an opaque object with a :meth:`cancel` method - that can be used to cancel the call. - - *callback* will be called exactly once per call to :meth:`call_later`. - If two callbacks are scheduled for exactly the same time, it is - undefined which will be called first. - - The optional positional *args* will be passed to the callback when it - is called. If you want the callback to be called with some named - arguments, use a closure or :func:`functools.partial`. - -.. method:: BaseEventLoop.call_at(when, callback, *args) - - Arrange for the *callback* to be called at the given absolute timestamp - *when* (an int or float), using the same time reference as :meth:`time`. - - This method's behavior is the same as :meth:`call_later`. - -.. method:: BaseEventLoop.time() - - Return the current time, as a :class:`float` value, according to the - event loop's internal clock. - -.. seealso:: - - The :func:`asyncio.sleep` function. - - -Creating connections --------------------- - -.. method:: BaseEventLoop.create_connection(protocol_factory, host=None, port=None, \*, ssl=None, family=0, proto=0, flags=0, sock=None, local_addr=None, server_hostname=None) - - Create a streaming transport connection to a given Internet *host* and - *port*. *protocol_factory* must be a callable returning a - :ref:`protocol ` instance. - - This method returns a :ref:`coroutine ` which will try to - establish the connection in the background. When successful, the - coroutine returns a ``(transport, protocol)`` pair. - - The chronological synopsis of the underlying operation is as follows: - - #. The connection is established, and a :ref:`transport ` - is created to represent it. - - #. *protocol_factory* is called without arguments and must return a - :ref:`protocol ` instance. - - #. The protocol instance is tied to the transport, and its - :meth:`connection_made` method is called. - - #. The coroutine returns successfully with the ``(transport, protocol)`` - pair. - - The created transport is an implementation-dependent bidirectional stream. - - .. note:: - *protocol_factory* can be any kind of callable, not necessarily - a class. For example, if you want to use a pre-created - protocol instance, you can pass ``lambda: my_protocol``. - - Options allowing to change how the connection is created: - - * *ssl*: if given and not false, a SSL/TLS transport is created - (by default a plain TCP transport is created). If *ssl* is - a :class:`ssl.SSLContext` object, this context is used to create - the transport; if *ssl* is :const:`True`, a context with some - unspecified default settings is used. - - * *server_hostname*, is only for use together with *ssl*, - and sets or overrides the hostname that the target server's certificate - will be matched against. By default the value of the *host* argument - is used. If *host* is empty, there is no default and you must pass a - value for *server_hostname*. If *server_hostname* is an empty - string, hostname matching is disabled (which is a serious security - risk, allowing for man-in-the-middle-attacks). - - * *family*, *proto*, *flags* are the optional address family, protocol - and flags to be passed through to getaddrinfo() for *host* resolution. - If given, these should all be integers from the corresponding - :mod:`socket` module constants. - - * *sock*, if given, should be an existing, already connected - :class:`socket.socket` object to be used by the transport. - If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags* - and *local_addr* should be specified. - - * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket to locally. The *local_host* and *local_port* - are looked up using getaddrinfo(), similarly to *host* and *port*. - - -Creating listening connections ------------------------------- - -.. method:: BaseEventLoop.create_server(protocol_factory, host=None, port=None, \*, family=socket.AF_UNSPEC, flags=socket.AI_PASSIVE, sock=None, backlog=100, ssl=None, reuse_address=None) - - A :ref:`coroutine ` which creates a TCP server bound to host and - port. - - The return value is a :class:`AbstractServer` object which can be used to stop - the service. - - If *host* is an empty string or None all interfaces are assumed - and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). - - *family* can be set to either :data:`~socket.AF_INET` or - :data:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. If not set - it will be determined from host (defaults to :data:`~socket.AF_UNSPEC`). - - *flags* is a bitmask for :meth:`getaddrinfo`. - - *sock* can optionally be specified in order to use a preexisting - socket object. - - *backlog* is the maximum number of queued connections passed to - :meth:`~socket.socket.listen` (defaults to 100). - - ssl can be set to an :class:`~ssl.SSLContext` to enable SSL over the - accepted connections. - - *reuse_address* tells the kernel to reuse a local socket in - TIME_WAIT state, without waiting for its natural timeout to - expire. If not specified will automatically be set to True on - UNIX. - - This method returns a :ref:`coroutine `. - -.. method:: BaseEventLoop.create_datagram_endpoint(protocol_factory, local_addr=None, remote_addr=None, \*, family=0, proto=0, flags=0) - - Create datagram connection. - - This method returns a :ref:`coroutine `. - - - -Resolve name ------------- - -.. method:: BaseEventLoop.getaddrinfo(host, port, \*, family=0, type=0, proto=0, flags=0) - - XXX - -.. method:: BaseEventLoop.getnameinfo(sockaddr, flags=0) - - XXX - - -Running subprocesses --------------------- - -Run subprocesses asynchronously using the :mod:`subprocess` module. - -.. method:: BaseEventLoop.subprocess_exec(protocol_factory, \*args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False, shell=False, bufsize=0, \*\*kwargs) - - XXX - - This method returns a :ref:`coroutine `. - - See the constructor of the :class:`subprocess.Popen` class for parameters. - -.. method:: BaseEventLoop.subprocess_shell(protocol_factory, cmd, \*, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=False, shell=True, bufsize=0, \*\*kwargs) - - XXX - - This method returns a :ref:`coroutine `. - - See the constructor of the :class:`subprocess.Popen` class for parameters. - -.. method:: BaseEventLoop.connect_read_pipe(protocol_factory, pipe) - - Register read pipe in eventloop. - - *protocol_factory* should instantiate object with :class:`Protocol` - interface. pipe is file-like object already switched to nonblocking. - Return pair (transport, protocol), where transport support - :class:`ReadTransport` interface. - - This method returns a :ref:`coroutine `. - -.. method:: BaseEventLoop.connect_write_pipe(protocol_factory, pipe) - - Register write pipe in eventloop. - - *protocol_factory* should instantiate object with :class:`BaseProtocol` - interface. Pipe is file-like object already switched to nonblocking. - Return pair (transport, protocol), where transport support - :class:`WriteTransport` interface. - - This method returns a :ref:`coroutine `. - - -Executor --------- - -Call a function in an :class:`~concurrent.futures.Executor` (pool of threads or -pool of processes). By default, an event loop uses a thread pool executor -(:class:`~concurrent.futures.ThreadPoolExecutor`). - -.. method:: BaseEventLoop.run_in_executor(executor, callback, \*args) - - Arrange for a callback to be called in the specified executor. - - *executor* is a :class:`~concurrent.futures.Executor` instance, - the default executor is used if *executor* is ``None``. - -.. method:: BaseEventLoop.set_default_executor(executor) - - Set the default executor used by :meth:`run_in_executor`. - - -.. _asyncio-hello-world-callback: - -Example: Hello World (callback) -------------------------------- - -Print ``Hello World`` every two seconds, using a callback:: - - import asyncio - - def print_and_repeat(loop): - print('Hello World') - loop.call_later(2, print_and_repeat, loop) - - loop = asyncio.get_event_loop() - print_and_repeat(loop) - loop.run_forever() - -.. seealso:: - - :ref:`Hello World example using a coroutine `. - diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/asyncio-protocol.rst --- a/Doc/library/asyncio-protocol.rst Tue Dec 03 19:17:25 2013 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,642 +0,0 @@ -.. module:: asyncio - -++++++++++++++++++++++++ -Transports and protocols -++++++++++++++++++++++++ - -.. _transport: - -Transports -========== - -Transports are classed provided by :mod:`asyncio` in order to abstract -various kinds of communication channels. You generally won't instantiate -a transport yourself; instead, you will call a :class:`BaseEventLoop` method -which will create the transport and try to initiate the underlying -communication channel, calling you back when it succeeds. - -Once the communication channel is established, a transport is always -paired with a :ref:`protocol ` instance. The protocol can -then call the transport's methods for various purposes. - -:mod:`asyncio` currently implements transports for TCP, UDP, SSL, and -subprocess pipes. The methods available on a transport depend on -the transport's kind. - - -BaseTransport -------------- - -.. class:: BaseTransport - - Base class for transports. - - .. method:: close(self) - - Close the transport. If the transport has a buffer for outgoing - data, buffered data will be flushed asynchronously. No more data - will be received. After all buffered data is flushed, the - protocol's :meth:`connection_lost` method will be called with - :const:`None` as its argument. - - - .. method:: get_extra_info(name, default=None) - - Return optional transport information. *name* is a string representing - the piece of transport-specific information to get, *default* is the - value to return if the information doesn't exist. - - This method allows transport implementations to easily expose - channel-specific information. - - * socket: - - - ``'peername'``: the remote address to which the socket is connected, - result of :meth:`socket.socket.getpeername` (``None`` on error) - - ``'socket'``: :class:`socket.socket` instance - - ``'sockname'``: the socket's own address, - result of :meth:`socket.socket.getsockname` - - * SSL socket: - - - ``'compression'``: the compression algorithm being used as a string, - or ``None`` if the connection isn't compressed; result of - :meth:`ssl.SSLSocket.compression` - - ``'cipher'``: a three-value tuple containing the name of the cipher - being used, the version of the SSL protocol that defines its use, and - the number of secret bits being used; result of - :meth:`ssl.SSLSocket.cipher` - - ``'peercert'``: peer certificate; result of - :meth:`ssl.SSLSocket.getpeercert` - - ``'sslcontext'``: :class:`ssl.SSLContext` instance - - * pipe: - - - ``'pipe'``: pipe object - - * subprocess: - - - ``'subprocess'``: :class:`subprocess.Popen` instance - - -ReadTransport -------------- - -.. class:: ReadTransport - - Interface for read-only transports. - - .. method:: pause_reading() - - Pause the receiving end of the transport. No data will be passed to - the protocol's :meth:`data_received` method until meth:`resume_reading` - is called. - - .. method:: resume_reading() - - Resume the receiving end. The protocol's :meth:`data_received` method - will be called once again if some data is available for reading. - - -WriteTransport --------------- - -.. class:: WriteTransport - - Interface for write-only transports. - - .. method:: abort() - - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`connection_lost` method will eventually be - called with :const:`None` as its argument. - - .. method:: can_write_eof() - - Return :const:`True` if the transport supports :meth:`write_eof`, - :const:`False` if not. - - .. method:: get_write_buffer_size() - - Return the current size of the output buffer used by the transport. - - .. method:: set_write_buffer_limits(high=None, low=None) - - Set the *high*- and *low*-water limits for write flow control. - - These two values control when call the protocol's - :meth:`pause_writing` and :meth:`resume_writing` methods are called. - If specified, the low-water limit must be less than or equal to the - high-water limit. Neither *high* nor *low* can be negative. - - The defaults are implementation-specific. If only the - high-water limit is given, the low-water limit defaults to a - implementation-specific value less than or equal to the - high-water limit. Setting *high* to zero forces *low* to zero as - well, and causes :meth:`pause_writing` to be called whenever the - buffer becomes non-empty. Setting *low* to zero causes - :meth:`resume_writing` to be called only once the buffer is empty. - Use of zero for either limit is generally sub-optimal as it - reduces opportunities for doing I/O and computation - concurrently. - - .. method:: write(data) - - Write some *data* bytes to the transport. - - This method does not block; it buffers the data and arranges for it - to be sent out asynchronously. - - .. method:: writelines(list_of_data) - - Write a list (or any iterable) of data bytes to the transport. - This is functionally equivalent to calling :meth:`write` on each - element yielded by the iterable, but may be implemented more efficiently. - - .. method:: write_eof() - - Close the write end of the transport after flushing buffered data. - Data may still be received. - - This method can raise :exc:`NotImplementedError` if the transport - (e.g. SSL) doesn't support half-closes. - - -DatagramTransport ------------------ - -.. method:: DatagramTransport.sendto(data, addr=None) - - Send the *data* bytes to the remote peer given by *addr* (a - transport-dependent target address). If *addr* is :const:`None`, the - data is sent to the target address given on transport creation. - - This method does not block; it buffers the data and arranges for it - to be sent out asynchronously. - -.. method:: DatagramTransport.abort() - - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`connection_lost` method will eventually be - called with :const:`None` as its argument. - - -BaseSubprocessTransport ------------------------ - -.. class:: BaseSubprocessTransport - - .. method:: get_pid() - - Return the subprocess process id as an integer. - - .. method:: get_pipe_transport(fd) - - Return the transport for the communication pipe correspondong to the - integer file descriptor *fd*. The return value can be a readable or - writable streaming transport, depending on the *fd*. If *fd* doesn't - correspond to a pipe belonging to this transport, :const:`None` is - returned. - - .. method:: get_returncode() - - Return the subprocess returncode as an integer or :const:`None` - if it hasn't returned, similarly to the - :attr:`subprocess.Popen.returncode` attribute. - - .. method:: kill(self) - - Kill the subprocess, as in :meth:`subprocess.Popen.kill` - - On POSIX systems, the function sends SIGKILL to the subprocess. - On Windows, this method is an alias for :meth:`terminate`. - - .. method:: send_signal(signal) - - Send the *signal* number to the subprocess, as in - :meth:`subprocess.Popen.send_signal`. - - .. method:: terminate() - - Ask the subprocess to stop, as in :meth:`subprocess.Popen.terminate`. - This method is an alias for the :meth:`close` method. - - On POSIX systems, this method sends SIGTERM to the subprocess. - On Windows, the Windows API function TerminateProcess() is called to - stop the subprocess. - - -StreamWriter ------------- - -.. class:: StreamWriter(transport, protocol, reader, loop) - - Wraps a Transport. - - This exposes :meth:`write`, :meth:`writelines`, :meth:`can_write_eof()`, - :meth:`write_eof`, :meth:`get_extra_info` and :meth:`close`. It adds - :meth:`drain` which returns an optional :class:`Future` on which you can - wait for flow control. It also adds a transport attribute which references - the :class:`Transport` directly. - - .. attribute:: transport - - Transport. - - .. method:: close() - - Close the transport: see :meth:`BaseTransport.close`. - - .. method:: drain() - - This method has an unusual return value. - - The intended use is to write:: - - w.write(data) - yield from w.drain() - - When there's nothing to wait for, :meth:`drain()` returns ``()``, and the - yield-from continues immediately. When the transport buffer is full (the - protocol is paused), :meth:`drain` creates and returns a - :class:`Future` and the yield-from will block until - that Future is completed, which will happen when the buffer is - (partially) drained and the protocol is resumed. - - .. method:: get_extra_info(name, default=None) - - Return optional transport information: see - :meth:`BaseTransport.get_extra_info`. - - .. method:: write(data) - - Write some *data* bytes to the transport: see - :meth:`WriteTransport.write`. - - .. method:: writelines(data) - - Write a list (or any iterable) of data bytes to the transport: - see :meth:`WriteTransport.writelines`. - - .. method:: can_write_eof() - - Return :const:`True` if the transport supports :meth:`write_eof`, - :const:`False` if not. See :meth:`WriteTransport.can_write_eof`. - - .. method:: write_eof() - - Close the write end of the transport after flushing buffered data: - see :meth:`WriteTransport.write_eof`. - - -StreamReader ------------- - -.. class:: StreamReader(limit=_DEFAULT_LIMIT, loop=None) - - .. method:: exception() - - Get the exception. - - .. method:: feed_eof() - - XXX - - .. method:: feed_data(data) - - XXX - - .. method:: set_exception(exc) - - Set the exception. - - .. method:: set_transport(transport) - - Set the transport. - - .. method:: read(n=-1) - - XXX - - This method returns a :ref:`coroutine `. - - .. method:: readline() - - XXX - - This method returns a :ref:`coroutine `. - - .. method:: readexactly(n) - - XXX - - This method returns a :ref:`coroutine `. - - - -.. _protocol: - -Protocols -========= - -:mod:`asyncio` provides base classes that you can subclass to implement -your network protocols. Those classes are used in conjunction with -:ref:`transports ` (see below): the protocol parses incoming -data and asks for the writing of outgoing data, while the transport is -responsible for the actual I/O and buffering. - -When subclassing a protocol class, it is recommended you override certain -methods. Those methods are callbacks: they will be called by the transport -on certain events (for example when some data is received); you shouldn't -call them yourself, unless you are implementing a transport. - -.. note:: - All callbacks have default implementations, which are empty. Therefore, - you only need to implement the callbacks for the events in which you - are interested. - - -Protocol classes ----------------- - -.. class:: Protocol - - The base class for implementing streaming protocols (for use with - e.g. TCP and SSL transports). - -.. class:: DatagramProtocol - - The base class for implementing datagram protocols (for use with - e.g. UDP transports). - -.. class:: SubprocessProtocol - - The base class for implementing protocols communicating with child - processes (through a set of unidirectional pipes). - - -Connection callbacks --------------------- - -These callbacks may be called on :class:`Protocol` and -:class:`SubprocessProtocol` instances: - -.. method:: BaseProtocol.connection_made(transport) - - Called when a connection is made. - - The *transport* argument is the transport representing the - connection. You are responsible for storing it somewhere - (e.g. as an attribute) if you need to. - -.. method:: BaseProtocol.connection_lost(exc) - - Called when the connection is lost or closed. - - The argument is either an exception object or :const:`None`. - The latter means a regular EOF is received, or the connection was - aborted or closed by this side of the connection. - -:meth:`connection_made` and :meth:`connection_lost` are called exactly once -per successful connection. All other callbacks will be called between those -two methods, which allows for easier resource management in your protocol -implementation. - -The following callbacks may be called only on :class:`SubprocessProtocol` -instances: - -.. method:: SubprocessProtocol.pipe_data_received(fd, data) - - Called when the child process writes data into its stdout or stderr pipe. - *fd* is the integer file descriptor of the pipe. *data* is a non-empty - bytes object containing the data. - -.. method:: SubprocessProtocol.pipe_connection_lost(fd, exc) - - Called when one of the pipes communicating with the child process - is closed. *fd* is the integer file descriptor that was closed. - -.. method:: SubprocessProtocol.process_exited() - - Called when the child process has exited. - - -Streaming protocols -------------------- - -The following callbacks are called on :class:`Protocol` instances: - -.. method:: Protocol.data_received(data) - - Called when some data is received. *data* is a non-empty bytes object - containing the incoming data. - - .. note:: - Whether the data is buffered, chunked or reassembled depends on - the transport. In general, you shouldn't rely on specific semantics - and instead make your parsing generic and flexible enough. However, - data is always received in the correct order. - -.. method:: Protocol.eof_received() - - Calls when the other end signals it won't send any more data - (for example by calling :meth:`write_eof`, if the other end also uses - asyncio). - - This method may return a false value (including None), in which case - the transport will close itself. Conversely, if this method returns a - true value, closing the transport is up to the protocol. Since the - default implementation returns None, it implicitly closes the connection. - - .. note:: - Some transports such as SSL don't support half-closed connections, - in which case returning true from this method will not prevent closing - the connection. - -:meth:`data_received` can be called an arbitrary number of times during -a connection. However, :meth:`eof_received` is called at most once -and, if called, :meth:`data_received` won't be called after it. - -Datagram protocols ------------------- - -The following callbacks are called on :class:`DatagramProtocol` instances. - -.. method:: DatagramProtocol.datagram_received(data, addr) - - Called when a datagram is received. *data* is a bytes object containing - the incoming data. *addr* is the address of the peer sending the data; - the exact format depends on the transport. - -.. method:: DatagramProtocol.error_received(exc) - - Called when a previous send or receive operation raises an - :class:`OSError`. *exc* is the :class:`OSError` instance. - - This method is called in rare conditions, when the transport (e.g. UDP) - detects that a datagram couldn't be delivered to its recipient. - In many conditions though, undeliverable datagrams will be silently - dropped. - - -Flow control callbacks ----------------------- - -These callbacks may be called on :class:`Protocol` and -:class:`SubprocessProtocol` instances: - -.. method:: BaseProtocol.pause_writing() - - Called when the transport's buffer goes over the high-water mark. - -.. method:: BaseProtocol.resume_writing() - - Called when the transport's buffer drains below the low-water mark. - - -:meth:`pause_writing` and :meth:`resume_writing` calls are paired -- -:meth:`pause_writing` is called once when the buffer goes strictly over -the high-water mark (even if subsequent writes increases the buffer size -even more), and eventually :meth:`resume_writing` is called once when the -buffer size reaches the low-water mark. - -.. note:: - If the buffer size equals the high-water mark, - :meth:`pause_writing` is not called -- it must go strictly over. - Conversely, :meth:`resume_writing` is called when the buffer size is - equal or lower than the low-water mark. These end conditions - are important to ensure that things go as expected when either - mark is zero. - - -Server ------- - -.. class:: AbstractServer - - Abstract server returned by :func:`BaseEventLoop.create_server`. - - .. method:: close() - - Stop serving. This leaves existing connections open. - - .. method:: wait_closed() - - Coroutine to wait until service is closed. - - -Network functions -================= - -.. function:: open_connection(host=None, port=None, *, loop=None, limit=_DEFAULT_LIMIT, **kwds) - - A wrapper for create_connection() returning a (reader, writer) pair. - - The reader returned is a StreamReader instance; the writer is a - :class:`Transport`. - - The arguments are all the usual arguments to - :meth:`BaseEventLoop.create_connection` except *protocol_factory*; most - common are positional host and port, with various optional keyword arguments - following. - - Additional optional keyword arguments are *loop* (to set the event loop - instance to use) and *limit* (to set the buffer limit passed to the - StreamReader). - - (If you want to customize the :class:`StreamReader` and/or - :class:`StreamReaderProtocol` classes, just copy the code -- there's really - nothing special here except some convenience.) - - This function returns a :ref:`coroutine `. - -.. function:: start_server(client_connected_cb, host=None, port=None, *, loop=None, limit=_DEFAULT_LIMIT, **kwds) - - Start a socket server, call back for each client connected. - - The first parameter, *client_connected_cb*, takes two parameters: - *client_reader*, *client_writer*. *client_reader* is a - :class:`StreamReader` object, while *client_writer* is a - :class:`StreamWriter` object. This parameter can either be a plain callback - function or a :ref:`coroutine `; if it is a coroutine, it will be - automatically converted into a :class:`Task`. - - The rest of the arguments are all the usual arguments to - :meth:`~BaseEventLoop.create_server()` except *protocol_factory*; most - common are positional host and port, with various optional keyword arguments - following. The return value is the same as - :meth:`~BaseEventLoop.create_server()`. - - Additional optional keyword arguments are *loop* (to set the event loop - instance to use) and *limit* (to set the buffer limit passed to the - :class:`StreamReader`). - - The return value is the same as :meth:`~BaseEventLoop.create_server()`, i.e. - a :class:`AbstractServer` object which can be used to stop the service. - - This function returns a :ref:`coroutine `. - - -Protocol example: TCP echo client and server -============================================ - -Echo server ------------ - -TCP echo server example:: - - import asyncio - - class EchoServer(asyncio.Protocol): - def connection_made(self, transport): - print('connection made') - self.transport = transport - - - def data_received(self, data): - print('data received:', data.decode()) - self.transport.write(data) - - # close the socket - self.transport.close() - - def connection_lost(self, exc): - print('connection lost') - - loop = asyncio.get_event_loop() - f = loop.create_server(EchoServer, '127.0.0.1', 8888) - s = loop.run_until_complete(f) - print('serving on', s.sockets[0].getsockname()) - loop.run_forever() - - -Echo client ------------ - -TCP echo client example:: - - import asyncio - - class EchoClient(asyncio.Protocol): - message = 'This is the message. It will be echoed.' - - def connection_made(self, transport): - self.transport = transport - self.transport.write(self.message.encode()) - print('data sent:', self.message) - - def data_received(self, data): - print('data received:', data.decode()) - - def connection_lost(self, exc): - print('connection lost') - asyncio.get_event_loop().stop() - - loop = asyncio.get_event_loop() - task = loop.create_connection(EchoClient, '127.0.0.1', 8888) - loop.run_until_complete(task) - loop.run_forever() - loop.close() - diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/asyncio-sync.rst --- a/Doc/library/asyncio-sync.rst Tue Dec 03 19:17:25 2013 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,341 +0,0 @@ -.. _sync: - -Synchronization primitives -========================== - -Locks ------ - -.. class:: Lock(\*, loop=None) - - Primitive lock objects. - - A primitive lock is a synchronization primitive that is not owned by a - particular coroutine when locked. A primitive lock is in one of two states, - 'locked' or 'unlocked'. - - It is created in the unlocked state. It has two basic methods, :meth:`acquire` - and :meth:`release`. When the state is unlocked, acquire() changes the state to - locked and returns immediately. When the state is locked, acquire() blocks - until a call to release() in another coroutine changes it to unlocked, then - the acquire() call resets it to locked and returns. The release() method - should only be called in the locked state; it changes the state to unlocked - and returns immediately. If an attempt is made to release an unlocked lock, - a :exc:`RuntimeError` will be raised. - - When more than one coroutine is blocked in acquire() waiting for the state - to turn to unlocked, only one coroutine proceeds when a release() call - resets the state to unlocked; first coroutine which is blocked in acquire() - is being processed. - - :meth:`acquire` is a coroutine and should be called with ``yield from``. - - Locks also support the context manager protocol. ``(yield from lock)`` - should be used as context manager expression. - - Usage:: - - lock = Lock() - ... - yield from lock - try: - ... - finally: - lock.release() - - Context manager usage:: - - lock = Lock() - ... - with (yield from lock): - ... - - Lock objects can be tested for locking state:: - - if not lock.locked(): - yield from lock - else: - # lock is acquired - ... - - .. method:: locked() - - Return ``True`` if lock is acquired. - - .. method:: acquire() - - Acquire a lock. - - This method blocks until the lock is unlocked, then sets it to locked and - returns ``True``. - - This method returns a :ref:`coroutine `. - - .. method:: release() - - Release a lock. - - When the lock is locked, reset it to unlocked, and return. If any other - coroutines are blocked waiting for the lock to become unlocked, allow - exactly one of them to proceed. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - - There is no return value. - - -.. class:: Event(\*, loop=None) - - An Event implementation, asynchronous equivalent to :class:`threading.Event`. - - Class implementing event objects. An event manages a flag that can be set to - true with the :meth:`set` method and reset to false with the :meth:`clear` - method. The :meth:`wait` method blocks until the flag is true. The flag is - initially false. - - .. method:: clear() - - Reset the internal flag to false. Subsequently, coroutines calling - :meth:`wait` will block until :meth:`set` is called to set the internal - flag to true again. - - .. method:: is_set() - - Return ``True`` if and only if the internal flag is true. - - .. method:: set() - - Set the internal flag to true. All coroutines waiting for it to become - true are awakened. Coroutine that call :meth:`wait` once the flag is true - will not block at all. - - .. method:: wait() - - Block until the internal flag is true. - - If the internal flag is true on entry, return ``True`` immediately. - Otherwise, block until another coroutine calls :meth:`set` to set the - flag to true, then return ``True``. - - This method returns a :ref:`coroutine `. - - -.. class:: Condition(\*, loop=None) - - A Condition implementation, asynchronous equivalent to - :class:`threading.Condition`. - - This class implements condition variable objects. A condition variable - allows one or more coroutines to wait until they are notified by another - coroutine. - - A new :class:`Lock` object is created and used as the underlying lock. - - .. method:: notify(n=1) - - By default, wake up one coroutine waiting on this condition, if any. - If the calling coroutine has not acquired the lock when this method is - called, a :exc:`RuntimeError` is raised. - - This method wakes up at most *n* of the coroutines waiting for the - condition variable; it is a no-op if no coroutines are waiting. - - .. note:: - - An awakened coroutine does not actually return from its :meth:`wait` - call until it can reacquire the lock. Since :meth:`notify` does not - release the lock, its caller should. - - .. method:: notify_all() - - Wake up all threads waiting on this condition. This method acts like - :meth:`notify`, but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, a - :exc:`RuntimeError` is raised. - - .. method:: wait() - - Wait until notified. - - If the calling coroutine has not acquired the lock when this method is - called, a :exc:`RuntimeError` is raised. - - This method releases the underlying lock, and then blocks until it is - awakened by a :meth:`notify` or :meth:`notify_all` call for the same - condition variable in another coroutine. Once awakened, it re-acquires - the lock and returns ``True``. - - This method returns a :ref:`coroutine `. - - .. method:: wait_for(predicate) - - Wait until a predicate becomes true. - - The predicate should be a callable which result will be interpreted as a - boolean value. The final predicate value is the return value. - - This method returns a :ref:`coroutine `. - - -Semaphores ----------- - -.. class:: Semaphore(value=1, \*, loop=None) - - A Semaphore implementation. - - A semaphore manages an internal counter which is decremented by each - :meth:`acquire` call and incremented by each :meth:`release` call. The - counter can never go below zero; when :meth:`acquire` finds that it is zero, - it blocks, waiting until some other thread calls :meth:`release`. - - Semaphores also support the context manager protocol. - - The optional argument gives the initial value for the internal counter; it - defaults to ``1``. If the value given is less than ``0``, :exc:`ValueError` - is raised. - - .. method:: acquire() - - Acquire a semaphore. - - If the internal counter is larger than zero on entry, decrement it by one - and return ``True`` immediately. If it is zero on entry, block, waiting - until some other coroutine has called :meth:`release` to make it larger - than ``0``, and then return ``True``. - - This method returns a :ref:`coroutine `. - - .. method:: locked() - - Returns ``True`` if semaphore can not be acquired immediately. - - .. method:: release() - - Release a semaphore, incrementing the internal counter by one. When it - was zero on entry and another coroutine is waiting for it to become - larger than zero again, wake up that coroutine. - - -.. class:: BoundedSemaphore(value=1, \*, loop=None) - - A bounded semaphore implementation. Inherit from :class:`Semaphore`. - - This raises :exc:`ValueError` in :meth:`~Semaphore.release` if it would - increase the value above the initial value. - - -Queues ------- - -.. class:: Queue(maxsize=0, \*, loop=None) - - A queue, useful for coordinating producer and consumer coroutines. - - If *maxsize* is less than or equal to zero, the queue size is infinite. If - it is an integer greater than ``0``, then ``yield from put()`` will block - when the queue reaches *maxsize*, until an item is removed by :meth:`get`. - - Unlike the standard library :mod:`queue`, you can reliably know this Queue's - size with :meth:`qsize`, since your single-threaded Tulip application won't - be interrupted between calling :meth:`qsize` and doing an operation on the - Queue. - - .. method:: empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. - - .. method:: full() - - Return ``True`` if there are maxsize items in the queue. - - .. note:: - - If the Queue was initialized with ``maxsize=0`` (the default), then - :meth:`full()` is never ``True``. - - .. method:: get() - - Remove and return an item from the queue. - - If you yield from :meth:`get()`, wait until a item is available. - - This method returns a :ref:`coroutine `. - - .. method:: get_nowait() - - Remove and return an item from the queue. - - Return an item if one is immediately available, else raise - :exc:`~queue.Empty`. - - .. method:: put(item) - - Put an item into the queue. - - If you yield from ``put()``, wait until a free slot is available before - adding item. - - This method returns a :ref:`coroutine `. - - .. method:: put_nowait(item) - - Put an item into the queue without blocking. - - If no free slot is immediately available, raise :exc:`~queue.Full`. - - .. method:: qsize() - - Number of items in the queue. - - .. attribute:: maxsize - - Number of items allowed in the queue. - - -.. class:: PriorityQueue - - A subclass of :class:`Queue`; retrieves entries in priority order (lowest - first). - - Entries are typically tuples of the form: (priority number, data). - - -.. class:: LifoQueue - - A subclass of :class:`Queue` that retrieves most recently added entries - first. - - -.. class:: JoinableQueue - - A subclass of :class:`Queue` with :meth:`task_done` and :meth:`join` - methods. - - .. method:: join() - - Block until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer thread calls - :meth:`task_done` to indicate that the item was retrieved and all work on - it is complete. When the count of unfinished tasks drops to zero, - :meth:`join` unblocks. - - This method returns a :ref:`coroutine `. - - .. method:: task_done() - - Indicate that a formerly enqueued task is complete. - - Used by queue consumers. For each :meth:`~Queue.get` used to fetch a task, a - subsequent call to :meth:`task_done` tells the queue that the processing - on the task is complete. - - If a :meth:`join` is currently blocking, it will resume when all items - have been processed (meaning that a :meth:`task_done` call was received - for every item that had been :meth:`~Queue.put` into the queue). - - Raises :exc:`ValueError` if called more times than there were items - placed in the queue. - diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/asyncio-task.rst --- a/Doc/library/asyncio-task.rst Tue Dec 03 19:17:25 2013 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,410 +0,0 @@ -.. module:: asyncio - -Tasks and coroutines -==================== - -.. _coroutine: - -Coroutines ----------- - -A coroutine is a generator that follows certain conventions. For -documentation purposes, all coroutines should be decorated with -``@asyncio.coroutine``, but this cannot be strictly enforced. - -Coroutines use the ``yield from`` syntax introduced in :pep:`380`, -instead of the original ``yield`` syntax. - -The word "coroutine", like the word "generator", is used for two -different (though related) concepts: - -- The function that defines a coroutine (a function definition - decorated with ``asyncio.coroutine``). If disambiguation is needed - we will call this a *coroutine function*. - -- The object obtained by calling a coroutine function. This object - represents a computation or an I/O operation (usually a combination) - that will complete eventually. If disambiguation is needed we will - call it a *coroutine object*. - -Things a coroutine can do: - -- ``result = yield from future`` -- suspends the coroutine until the - future is done, then returns the future's result, or raises an - exception, which will be propagated. (If the future is cancelled, - it will raise a ``CancelledError`` exception.) Note that tasks are - futures, and everything said about futures also applies to tasks. - -- ``result = yield from coroutine`` -- wait for another coroutine to - produce a result (or raise an exception, which will be propagated). - The ``coroutine`` expression must be a *call* to another coroutine. - -- ``return expression`` -- produce a result to the coroutine that is - waiting for this one using ``yield from``. - -- ``raise exception`` -- raise an exception in the coroutine that is - waiting for this one using ``yield from``. - -Calling a coroutine does not start its code running -- it is just a -generator, and the coroutine object returned by the call is really a -generator object, which doesn't do anything until you iterate over it. -In the case of a coroutine object, there are two basic ways to start -it running: call ``yield from coroutine`` from another coroutine -(assuming the other coroutine is already running!), or convert it to a -:class:`Task`. - -Coroutines (and tasks) can only run when the event loop is running. - - -InvalidStateError ------------------ - -.. exception:: InvalidStateError - - The operation is not allowed in this state. - - -Future ------- - -.. class:: Future(\*, loop=None) - - This class is *almost* compatible with :class:`concurrent.futures.Future`. - - Differences: - - - :meth:`result` and :meth:`exception` do not take a timeout argument and - raise an exception when the future isn't done yet. - - - Callbacks registered with :meth:`add_done_callback` are always called - via the event loop's :meth:`~BaseEventLoop.call_soon_threadsafe`. - - - This class is not compatible with the :func:`~concurrent.futures.wait` and - :func:`~concurrent.futures.as_completed` functions in the - :mod:`concurrent.futures` package. - - .. method:: cancel() - - Cancel the future and schedule callbacks. - - If the future is already done or cancelled, return ``False``. Otherwise, - change the future's state to cancelled, schedule the callbacks and return - ``True``. - - .. method:: cancelled() - - Return ``True`` if the future was cancelled. - - .. method:: done() - - Return True if the future is done. - - Done means either that a result / exception are available, or that the - future was cancelled. - - .. method:: result() - - Return the result this future represents. - - If the future has been cancelled, raises :exc:`CancelledError`. If the - future's result isn't yet available, raises :exc:`InvalidStateError`. If - the future is done and has an exception set, this exception is raised. - - .. method:: exception() - - Return the exception that was set on this future. - - The exception (or ``None`` if no exception was set) is returned only if - the future is done. If the future has been cancelled, raises - :exc:`CancelledError`. If the future isn't done yet, raises - :exc:`InvalidStateError`. - - .. method:: add_done_callback(fn) - - Add a callback to be run when the future becomes done. - - The callback is called with a single argument - the future object. If the - future is already done when this is called, the callback is scheduled - with :meth:`~BaseEventLoop.call_soon`. - - .. method:: remove_done_callback(fn) - - Remove all instances of a callback from the "call when done" list. - - Returns the number of callbacks removed. - - .. method:: set_result(result) - - Mark the future done and set its result. - - If the future is already done when this method is called, raises - :exc:`InvalidStateError`. - - .. method:: set_exception(exception) - - Mark the future done and set an exception. - - If the future is already done when this method is called, raises - :exc:`InvalidStateError`. - - -Task ----- - -.. class:: Task(coro, \*, loop=None) - - A coroutine wrapped in a :class:`Future`. Subclass of :class:`Future`. - - .. classmethod:: all_tasks(loop=None) - - Return a set of all tasks for an event loop. - - By default all tasks for the current event loop are returned. - - .. method:: cancel() - - Cancel the task. - - .. method:: get_stack(self, \*, limit=None) - - Return the list of stack frames for this task's coroutine. - - If the coroutine is active, this returns the stack where it is suspended. - If the coroutine has completed successfully or was cancelled, this - returns an empty list. If the coroutine was terminated by an exception, - this returns the list of traceback frames. - - The frames are always ordered from oldest to newest. - - The optional limit gives the maximum nummber of frames to return; by - default all available frames are returned. Its meaning differs depending - on whether a stack or a traceback is returned: the newest frames of a - stack are returned, but the oldest frames of a traceback are returned. - (This matches the behavior of the traceback module.) - - For reasons beyond our control, only one stack frame is returned for a - suspended coroutine. - - .. method:: print_stack(\*, limit=None, file=None) - - Print the stack or traceback for this task's coroutine. - - This produces output similar to that of the traceback module, for the - frames retrieved by get_stack(). The limit argument is passed to - get_stack(). The file argument is an I/O stream to which the output - goes; by default it goes to sys.stderr. - - -Task functions --------------- - -.. function:: as_completed(fs, \*, loop=None, timeout=None) - - Return an iterator whose values, when waited for, are :class:`Future` - instances. - - Raises :exc:`TimeoutError` if the timeout occurs before all Futures are done. - - Example:: - - for f in as_completed(fs): - result = yield from f # The 'yield from' may raise - # Use result - - .. note:: - - The futures ``f`` are not necessarily members of fs. - -.. function:: async(coro_or_future, \*, loop=None) - - Wrap a :ref:`coroutine ` in a future. - - If the argument is a :class:`Future`, it is returned directly. - -.. function:: gather(\*coros_or_futures, loop=None, return_exceptions=False) - - Return a future aggregating results from the given coroutines or futures. - - All futures must share the same event loop. If all the tasks are done - successfully, the returned future's result is the list of results (in the - order of the original sequence, not necessarily the order of results - arrival). If *result_exception* is True, exceptions in the tasks are - treated the same as successful results, and gathered in the result list; - otherwise, the first raised exception will be immediately propagated to the - returned future. - - Cancellation: if the outer Future is cancelled, all children (that have not - completed yet) are also cancelled. If any child is cancelled, this is - treated as if it raised :exc:`~concurrent.futures.CancelledError` -- the - outer Future is *not* cancelled in this case. (This is to prevent the - cancellation of one child to cause other children to be cancelled.) - -.. function:: tasks.iscoroutinefunction(func) - - Return ``True`` if *func* is a decorated coroutine function. - -.. function:: tasks.iscoroutine(obj) - - Return ``True`` if *obj* is a coroutine object. - -.. function:: sleep(delay, result=None, \*, loop=None) - - Create a :ref:`coroutine ` that completes after a given time - (in seconds). - -.. function:: shield(arg, \*, loop=None) - - Wait for a future, shielding it from cancellation. - - The statement:: - - res = yield from shield(something()) - - is exactly equivalent to the statement:: - - res = yield from something() - - *except* that if the coroutine containing it is cancelled, the task running - in ``something()`` is not cancelled. From the point of view of - ``something()``, the cancellation did not happen. But its caller is still - cancelled, so the yield-from expression still raises - :exc:`~concurrent.futures.CancelledError`. Note: If ``something()`` is - cancelled by other means this will still cancel ``shield()``. - - If you want to completely ignore cancellation (not recommended) you can - combine ``shield()`` with a try/except clause, as follows:: - - try: - res = yield from shield(something()) - except CancelledError: - res = None - -.. function:: wait(futures, \*, loop=None, timeout=None, return_when=ALL_COMPLETED) - - Wait for the Futures and coroutines given by the sequence *futures* to - complete. Coroutines will be wrapped in Tasks. Returns two sets of - :class:`Future`: (done, pending). - - *timeout* can be used to control the maximum number of seconds to wait before - returning. *timeout* can be an int or float. If *timeout* is not specified - or ``None``, there is no limit to the wait time. - - *return_when* indicates when this function should return. It must be one of - the following constants of the :mod:`concurrent.futures` module: - - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ - - This function returns a :ref:`coroutine `. - - Usage:: - - done, pending = yield from asyncio.wait(fs) - - .. note:: - - This does not raise :exc:`TimeoutError`! Futures that aren't done when - the timeout occurs are returned in the second set. - - -Examples --------- - - -.. _asyncio-hello-world-coroutine: - -Example: Hello World (coroutine) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Print ``Hello World`` every two seconds, using a coroutine:: - - import asyncio - - @asyncio.coroutine - def greet_every_two_seconds(): - while True: - print('Hello World') - yield from asyncio.sleep(2) - - loop = asyncio.get_event_loop() - loop.run_until_complete(greet_every_two_seconds()) - - -.. seealso:: - - :ref:`Hello World example using a callback `. - -Example: Chains coroutines and parallel execution -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example chaining coroutines and executing multiple coroutines in parallel:: - - import asyncio - - @asyncio.coroutine - def compute(x, y): - print("Start computing %s + %s" % (x, y)) - yield from asyncio.sleep(3.0) - return x + y - - @asyncio.coroutine - def print_sum(x, y): - result = yield from compute(x, y) - print("%s + %s = %s" % (x, y, result)) - - @asyncio.coroutine - def wait_task(task): - while 1: - done, pending = yield from asyncio.wait([task], timeout=1.0) - if done: - break - print("Compute in progress...") - asyncio.get_event_loop().stop() - - print("Schedule tasks") - task = asyncio.async(print_sum(1, 2)) - asyncio.async(wait_task(task)) - - print("Execute tasks") - loop = asyncio.get_event_loop() - loop.run_forever() - loop.close() - - - -Output:: - - Schedule tasks - Execute tasks - Start computing 1 + 2 - Compute in progress... - Compute in progress... - 1 + 2 = 3 - -Details: - -* ``compute()`` is chained to ``print_sum()``: ``print_sum()`` coroutine waits - until ``compute()`` is complete. Coroutines are executed in parallel: - ``wait_task()`` is executed while ``compute()`` is blocked in - ``asyncio.sleep(3.0)``. - -* Coroutines are not executed before the loop is running: ``"Execute tasks"`` - is written before ``"Start computing 1 + 2"``. - -* ``wait_task()`` stops the event loop when ``print_sum()`` is done. - diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/asyncio.rst --- a/Doc/library/asyncio.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/library/asyncio.rst Tue Dec 03 16:02:48 2013 +0100 @@ -13,6 +13,7 @@ This module provides infrastructure for writing single-threaded concurrent code using coroutines, multiplexing I/O access over sockets and other resources, running network clients and servers, and other related primitives. + Here is a more detailed list of the package contents: * a pluggable :ref:`event loop ` with various system-specific @@ -24,8 +25,8 @@ * concrete support for TCP, UDP, SSL, subprocess pipes, delayed calls, and others (some may be system-dependent); -* a :class:`Future` class that mimicks the one in the :mod:`concurrent.futures` - module, but adapted for use with the event loop; +* a Future class that mimicks the one in the :mod:`concurrent.futures` module, + but adapted for use with the event loop; * coroutines and tasks based on ``yield from`` (:PEP:`380`), to help write concurrent code in a sequential fashion; @@ -39,19 +40,547 @@ you absolutely, positively have to use a library that makes blocking I/O calls. -Table of content: -.. toctree:: - :maxdepth: 3 - :numbered: +Disclaimer +---------- - asyncio-eventloop.rst - asyncio-task.rst - asyncio-protocol.rst - asyncio-sync.rst +Full documentation is not yet ready; we hope to have it written +before Python 3.4 leaves beta. Until then, the best reference is +:PEP:`3156`. For a motivational primer on transports and protocols, +see :PEP:`3153`. -.. seealso:: - The :mod:`asyncio` module was designed in the :PEP:`3156`. For a - motivational primer on transports and protocols, see :PEP:`3153`. +.. XXX should the asyncio documentation come in several pages, as for logging? + +.. _event-loop: + +Event loops +----------- + +The event loop is the central execution device provided by :mod:`asyncio`. +It provides multiple facilities, amongst which: + +* Registering, executing and cancelling delayed calls (timeouts) + +* Creating client and server :ref:`transports ` for various + kinds of communication + +* Launching subprocesses and the associated :ref:`transports ` + for communication with an external program + +* Delegating costly function calls to a pool of threads + +Getting an event loop +^^^^^^^^^^^^^^^^^^^^^ + +The easiest way to get an event loop is to call the :func:`get_event_loop` +function. + +.. XXX more docs + +Delayed calls +^^^^^^^^^^^^^ + +The event loop has its own internal clock for computing timeouts. +Which clock is used depends on the (platform-specific) event loop +implementation; ideally it is a monotonic clock. This will generally be +a different clock than :func:`time.time`. + +.. method:: time() + + Return the current time, as a :class:`float` value, according to the + event loop's internal clock. + +.. method:: call_later(delay, callback, *args) + + Arrange for the *callback* to be called after the given *delay* + seconds (either an int or float). + + A "handle" is returned: an opaque object with a :meth:`cancel` method + that can be used to cancel the call. + + *callback* will be called exactly once per call to :meth:`call_later`. + If two callbacks are scheduled for exactly the same time, it is + undefined which will be called first. + + The optional positional *args* will be passed to the callback when it + is called. If you want the callback to be called with some named + arguments, use a closure or :func:`functools.partial`. + +.. method:: call_at(when, callback, *args) + + Arrange for the *callback* to be called at the given absolute timestamp + *when* (an int or float), using the same time reference as :meth:`time`. + + This method's behavior is the same as :meth:`call_later`. + +Creating connections +^^^^^^^^^^^^^^^^^^^^ + +.. method:: create_connection(protocol_factory, host=None, port=None, **options) + + Create a streaming transport connection to a given Internet *host* and + *port*. *protocol_factory* must be a callable returning a + :ref:`protocol ` instance. + + This method returns a :ref:`coroutine ` which will try to + establish the connection in the background. When successful, the + coroutine returns a ``(transport, protocol)`` pair. + + The chronological synopsis of the underlying operation is as follows: + + #. The connection is established, and a :ref:`transport ` + is created to represent it. + + #. *protocol_factory* is called without arguments and must return a + :ref:`protocol ` instance. + + #. The protocol instance is tied to the transport, and its + :meth:`connection_made` method is called. + + #. The coroutine returns successfully with the ``(transport, protocol)`` + pair. + + The created transport is an implementation-dependent bidirectional stream. + + .. note:: + *protocol_factory* can be any kind of callable, not necessarily + a class. For example, if you want to use a pre-created + protocol instance, you can pass ``lambda: my_protocol``. + + *options* are optional named arguments allowing to change how the + connection is created: + + * *ssl*: if given and not false, a SSL/TLS transport is created + (by default a plain TCP transport is created). If *ssl* is + a :class:`ssl.SSLContext` object, this context is used to create + the transport; if *ssl* is :const:`True`, a context with some + unspecified default settings is used. + + * *server_hostname*, is only for use together with *ssl*, + and sets or overrides the hostname that the target server's certificate + will be matched against. By default the value of the *host* argument + is used. If *host* is empty, there is no default and you must pass a + value for *server_hostname*. If *server_hostname* is an empty + string, hostname matching is disabled (which is a serious security + risk, allowing for man-in-the-middle-attacks). + + * *family*, *proto*, *flags* are the optional address family, protocol + and flags to be passed through to getaddrinfo() for *host* resolution. + If given, these should all be integers from the corresponding + :mod:`socket` module constants. + + * *sock*, if given, should be an existing, already connected + :class:`socket.socket` object to be used by the transport. + If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags* + and *local_addr* should be specified. + + * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used + to bind the socket to locally. The *local_host* and *local_port* + are looked up using getaddrinfo(), similarly to *host* and *port*. + + +.. _protocol: + +Protocols +--------- + +:mod:`asyncio` provides base classes that you can subclass to implement +your network protocols. Those classes are used in conjunction with +:ref:`transports ` (see below): the protocol parses incoming +data and asks for the writing of outgoing data, while the transport is +responsible for the actual I/O and buffering. + +When subclassing a protocol class, it is recommended you override certain +methods. Those methods are callbacks: they will be called by the transport +on certain events (for example when some data is received); you shouldn't +call them yourself, unless you are implementing a transport. + +.. note:: + All callbacks have default implementations, which are empty. Therefore, + you only need to implement the callbacks for the events in which you + are interested. + + +Protocol classes +^^^^^^^^^^^^^^^^ + +.. class:: Protocol + + The base class for implementing streaming protocols (for use with + e.g. TCP and SSL transports). + +.. class:: DatagramProtocol + + The base class for implementing datagram protocols (for use with + e.g. UDP transports). + +.. class:: SubprocessProtocol + + The base class for implementing protocols communicating with child + processes (through a set of unidirectional pipes). + + +Connection callbacks +^^^^^^^^^^^^^^^^^^^^ + +These callbacks may be called on :class:`Protocol` and +:class:`SubprocessProtocol` instances: + +.. method:: connection_made(transport) + + Called when a connection is made. + + The *transport* argument is the transport representing the + connection. You are responsible for storing it somewhere + (e.g. as an attribute) if you need to. + +.. method:: connection_lost(exc) + + Called when the connection is lost or closed. + + The argument is either an exception object or :const:`None`. + The latter means a regular EOF is received, or the connection was + aborted or closed by this side of the connection. + +:meth:`connection_made` and :meth:`connection_lost` are called exactly once +per successful connection. All other callbacks will be called between those +two methods, which allows for easier resource management in your protocol +implementation. + +The following callbacks may be called only on :class:`SubprocessProtocol` +instances: + +.. method:: pipe_data_received(fd, data) + + Called when the child process writes data into its stdout or stderr pipe. + *fd* is the integer file descriptor of the pipe. *data* is a non-empty + bytes object containing the data. + +.. method:: pipe_connection_lost(fd, exc) + + Called when one of the pipes communicating with the child process + is closed. *fd* is the integer file descriptor that was closed. + +.. method:: process_exited() + + Called when the child process has exited. + + +Data reception callbacks +^^^^^^^^^^^^^^^^^^^^^^^^ + +Streaming protocols +""""""""""""""""""" + +The following callbacks are called on :class:`Protocol` instances: + +.. method:: data_received(data) + + Called when some data is received. *data* is a non-empty bytes object + containing the incoming data. + + .. note:: + Whether the data is buffered, chunked or reassembled depends on + the transport. In general, you shouldn't rely on specific semantics + and instead make your parsing generic and flexible enough. However, + data is always received in the correct order. + +.. method:: eof_received() + + Calls when the other end signals it won't send any more data + (for example by calling :meth:`write_eof`, if the other end also uses + asyncio). + + This method may return a false value (including None), in which case + the transport will close itself. Conversely, if this method returns a + true value, closing the transport is up to the protocol. Since the + default implementation returns None, it implicitly closes the connection. + + .. note:: + Some transports such as SSL don't support half-closed connections, + in which case returning true from this method will not prevent closing + the connection. + +:meth:`data_received` can be called an arbitrary number of times during +a connection. However, :meth:`eof_received` is called at most once +and, if called, :meth:`data_received` won't be called after it. + +Datagram protocols +"""""""""""""""""" + +The following callbacks are called on :class:`DatagramProtocol` instances. + +.. method:: datagram_received(data, addr) + + Called when a datagram is received. *data* is a bytes object containing + the incoming data. *addr* is the address of the peer sending the data; + the exact format depends on the transport. + +.. method:: error_received(exc) + + Called when a previous send or receive operation raises an + :class:`OSError`. *exc* is the :class:`OSError` instance. + + This method is called in rare conditions, when the transport (e.g. UDP) + detects that a datagram couldn't be delivered to its recipient. + In many conditions though, undeliverable datagrams will be silently + dropped. + + +Flow control callbacks +^^^^^^^^^^^^^^^^^^^^^^ + +These callbacks may be called on :class:`Protocol` and +:class:`SubprocessProtocol` instances: + +.. method:: pause_writing() + + Called when the transport's buffer goes over the high-water mark. + +.. method:: resume_writing() + + Called when the transport's buffer drains below the low-water mark. + + +:meth:`pause_writing` and :meth:`resume_writing` calls are paired -- +:meth:`pause_writing` is called once when the buffer goes strictly over +the high-water mark (even if subsequent writes increases the buffer size +even more), and eventually :meth:`resume_writing` is called once when the +buffer size reaches the low-water mark. + +.. note:: + If the buffer size equals the high-water mark, + :meth:`pause_writing` is not called -- it must go strictly over. + Conversely, :meth:`resume_writing` is called when the buffer size is + equal or lower than the low-water mark. These end conditions + are important to ensure that things go as expected when either + mark is zero. + + +.. _transport: + +Transports +---------- + +Transports are classed provided by :mod:`asyncio` in order to abstract +various kinds of communication channels. You generally won't instantiate +a transport yourself; instead, you will call a :class:`EventLoop` method +which will create the transport and try to initiate the underlying +communication channel, calling you back when it succeeds. + +Once the communication channel is established, a transport is always +paired with a :ref:`protocol ` instance. The protocol can +then call the transport's methods for various purposes. + +:mod:`asyncio` currently implements transports for TCP, UDP, SSL, and +subprocess pipes. The methods available on a transport depend on +the transport's kind. + +Methods common to all transports +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: close(self) + + Close the transport. If the transport has a buffer for outgoing + data, buffered data will be flushed asynchronously. No more data + will be received. After all buffered data is flushed, the + protocol's :meth:`connection_lost` method will be called with + :const:`None` as its argument. + + +.. method:: get_extra_info(name, default=None) + + Return optional transport information. *name* is a string representing + the piece of transport-specific information to get, *default* is the + value to return if the information doesn't exist. + + This method allows transport implementations to easily expose + channel-specific information. + +Methods of readable streaming transports +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: pause_reading() + + Pause the receiving end of the transport. No data will be passed to + the protocol's :meth:`data_received` method until meth:`resume_reading` + is called. + +.. method:: resume_reading() + + Resume the receiving end. The protocol's :meth:`data_received` method + will be called once again if some data is available for reading. + +Methods of writable streaming transports +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: write(data) + + Write some *data* bytes to the transport. + + This method does not block; it buffers the data and arranges for it + to be sent out asynchronously. + +.. method:: writelines(list_of_data) + + Write a list (or any iterable) of data bytes to the transport. + This is functionally equivalent to calling :meth:`write` on each + element yielded by the iterable, but may be implemented more efficiently. + +.. method:: write_eof() + + Close the write end of the transport after flushing buffered data. + Data may still be received. + + This method can raise :exc:`NotImplementedError` if the transport + (e.g. SSL) doesn't support half-closes. + +.. method:: can_write_eof() + + Return :const:`True` if the transport supports :meth:`write_eof`, + :const:`False` if not. + +.. method:: abort() + + Close the transport immediately, without waiting for pending operations + to complete. Buffered data will be lost. No more data will be received. + The protocol's :meth:`connection_lost` method will eventually be + called with :const:`None` as its argument. + +.. method:: set_write_buffer_limits(high=None, low=None) + + Set the *high*- and *low*-water limits for write flow control. + + These two values control when call the protocol's + :meth:`pause_writing` and :meth:`resume_writing` methods are called. + If specified, the low-water limit must be less than or equal to the + high-water limit. Neither *high* nor *low* can be negative. + + The defaults are implementation-specific. If only the + high-water limit is given, the low-water limit defaults to a + implementation-specific value less than or equal to the + high-water limit. Setting *high* to zero forces *low* to zero as + well, and causes :meth:`pause_writing` to be called whenever the + buffer becomes non-empty. Setting *low* to zero causes + :meth:`resume_writing` to be called only once the buffer is empty. + Use of zero for either limit is generally sub-optimal as it + reduces opportunities for doing I/O and computation + concurrently. + +.. method:: get_write_buffer_size() + + Return the current size of the output buffer used by the transport. + +Methods of datagram transports +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: sendto(data, addr=None) + + Send the *data* bytes to the remote peer given by *addr* (a + transport-dependent target address). If *addr* is :const:`None`, the + data is sent to the target address given on transport creation. + + This method does not block; it buffers the data and arranges for it + to be sent out asynchronously. + +.. method:: abort() + + Close the transport immediately, without waiting for pending operations + to complete. Buffered data will be lost. No more data will be received. + The protocol's :meth:`connection_lost` method will eventually be + called with :const:`None` as its argument. + +Methods of subprocess transports +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. method:: get_pid() + + Return the subprocess process id as an integer. + +.. method:: get_returncode() + + Return the subprocess returncode as an integer or :const:`None` + if it hasn't returned, similarly to the + :attr:`subprocess.Popen.returncode` attribute. + +.. method:: get_pipe_transport(fd) + + Return the transport for the communication pipe correspondong to the + integer file descriptor *fd*. The return value can be a readable or + writable streaming transport, depending on the *fd*. If *fd* doesn't + correspond to a pipe belonging to this transport, :const:`None` is + returned. + +.. method:: send_signal(signal) + + Send the *signal* number to the subprocess, as in + :meth:`subprocess.Popen.send_signal`. + +.. method:: terminate() + + Ask the subprocess to stop, as in :meth:`subprocess.Popen.terminate`. + This method is an alias for the :meth:`close` method. + + On POSIX systems, this method sends SIGTERM to the subprocess. + On Windows, the Windows API function TerminateProcess() is called to + stop the subprocess. + +.. method:: kill(self) + + Kill the subprocess, as in :meth:`subprocess.Popen.kill` + + On POSIX systems, the function sends SIGKILL to the subprocess. + On Windows, this method is an alias for :meth:`terminate`. + + +.. _coroutine: + +Coroutines +---------- + + +.. _sync: + +Synchronization primitives +-------------------------- + + +Examples +-------- + +A :class:`Protocol` implementing an echo server:: + + class EchoServer(asyncio.Protocol): + + TIMEOUT = 5.0 + + def timeout(self): + print('connection timeout, closing.') + self.transport.close() + + def connection_made(self, transport): + print('connection made') + self.transport = transport + + # start 5 seconds timeout timer + self.h_timeout = asyncio.get_event_loop().call_later( + self.TIMEOUT, self.timeout) + + def data_received(self, data): + print('data received: ', data.decode()) + self.transport.write(b'Re: ' + data) + + # restart timeout timer + self.h_timeout.cancel() + self.h_timeout = asyncio.get_event_loop().call_later( + self.TIMEOUT, self.timeout) + + def eof_received(self): + pass + + def connection_lost(self, exc): + print('connection lost:', exc) + self.h_timeout.cancel() + diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/imaplib.rst --- a/Doc/library/imaplib.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/library/imaplib.rst Tue Dec 03 16:02:48 2013 +0100 @@ -80,10 +80,6 @@ .. versionchanged:: 3.3 *ssl_context* parameter added. - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). The second subclass allows for connections created by a child process: @@ -441,10 +437,6 @@ .. versionadded:: 3.2 - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). .. method:: IMAP4.status(mailbox, names) diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/nntplib.rst --- a/Doc/library/nntplib.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/library/nntplib.rst Tue Dec 03 16:02:48 2013 +0100 @@ -102,10 +102,6 @@ .. versionadded:: 3.2 - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). .. exception:: NNTPError @@ -245,10 +241,6 @@ .. versionadded:: 3.2 - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). .. method:: NNTP.newgroups(date, *, file=None) diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/poplib.rst --- a/Doc/library/poplib.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/library/poplib.rst Tue Dec 03 16:02:48 2013 +0100 @@ -53,10 +53,6 @@ .. versionchanged:: 3.2 *context* parameter added. - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). One exception is defined as an attribute of the :mod:`poplib` module: @@ -202,11 +198,6 @@ .. versionadded:: 3.4 - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). - Instances of :class:`POP3_SSL` have no additional methods. The interface of this subclass is identical to its parent. diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/re.rst --- a/Doc/library/re.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/library/re.rst Tue Dec 03 16:02:48 2013 +0100 @@ -801,7 +801,7 @@ >>> pattern.fullmatch("dog") # No match as "o" is not at the start of "dog". >>> pattern.fullmatch("ogre") # No match as not the full string matches. >>> pattern.fullmatch("doggie", 1, 3) # Matches within given limits. - <_sre.SRE_Match object; span=(1, 3), match='og'> + <_sre.SRE_Match object at ...> .. versionadded:: 3.4 diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/smtplib.rst --- a/Doc/library/smtplib.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/library/smtplib.rst Tue Dec 03 16:02:48 2013 +0100 @@ -90,10 +90,6 @@ .. versionchanged:: 3.3 source_address argument was added. - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). .. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, source_address=None) @@ -320,11 +316,6 @@ .. versionchanged:: 3.3 *context* was added. - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :data:`~ssl.HAS_SNI`). - .. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=[], rcpt_options=[]) diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/library/sys.rst --- a/Doc/library/sys.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/library/sys.rst Tue Dec 03 16:02:48 2013 +0100 @@ -425,7 +425,7 @@ * On Mac OS X, the encoding is ``'utf-8'``. * On Unix, the encoding is the user's preference according to the result of - nl_langinfo(CODESET). + nl_langinfo(CODESET), or ``'utf-8'`` if ``nl_langinfo(CODESET)`` failed. * On Windows NT+, file names are Unicode natively, so no conversion is performed. :func:`getfilesystemencoding` still returns ``'mbcs'``, as @@ -436,7 +436,8 @@ * On Windows 9x, the encoding is ``'mbcs'``. .. versionchanged:: 3.2 - :func:`getfilesystemencoding` result cannot be ``None`` anymore. + On Unix, use ``'utf-8'`` instead of ``None`` if ``nl_langinfo(CODESET)`` + failed. :func:`getfilesystemencoding` result cannot be ``None``. .. function:: getrefcount(object) diff -r f2c7f771bf6a -r 66c1c9f32567 Doc/whatsnew/3.4.rst --- a/Doc/whatsnew/3.4.rst Tue Dec 03 19:17:25 2013 +0100 +++ b/Doc/whatsnew/3.4.rst Tue Dec 03 16:02:48 2013 +0100 @@ -742,7 +742,7 @@ cert store. (Contributed by Christian Heimes in :issue:`18143`, :issue:`18147` and -:issue:`17134`.) + :issue:`17134`.) Support for server-side SNI using the new :meth:`ssl.SSLContext.set_servername_callback` method. diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/_compat_pickle.py --- a/Lib/_compat_pickle.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/_compat_pickle.py Tue Dec 03 16:02:48 2013 +0100 @@ -76,62 +76,6 @@ ('itertools', 'ifilterfalse'): ('itertools', 'filterfalse'), } -PYTHON2_EXCEPTIONS = ( - "ArithmeticError", - "AssertionError", - "AttributeError", - "BaseException", - "BufferError", - "BytesWarning", - "DeprecationWarning", - "EOFError", - "EnvironmentError", - "Exception", - "FloatingPointError", - "FutureWarning", - "GeneratorExit", - "IOError", - "ImportError", - "ImportWarning", - "IndentationError", - "IndexError", - "KeyError", - "KeyboardInterrupt", - "LookupError", - "MemoryError", - "NameError", - "NotImplementedError", - "OSError", - "OverflowError", - "PendingDeprecationWarning", - "ReferenceError", - "RuntimeError", - "RuntimeWarning", - # StandardError is gone in Python 3, so we map it to Exception - "StopIteration", - "SyntaxError", - "SyntaxWarning", - "SystemError", - "SystemExit", - "TabError", - "TypeError", - "UnboundLocalError", - "UnicodeDecodeError", - "UnicodeEncodeError", - "UnicodeError", - "UnicodeTranslateError", - "UnicodeWarning", - "UserWarning", - "ValueError", - "Warning", - "ZeroDivisionError", -) - -for excname in PYTHON2_EXCEPTIONS: - NAME_MAPPING[("exceptions", excname)] = ("builtins", excname) - -NAME_MAPPING[("exceptions", "StandardError")] = ("builtins", "Exception") - # Same, but for 3.x to 2.x REVERSE_IMPORT_MAPPING = dict((v, k) for (k, v) in IMPORT_MAPPING.items()) REVERSE_NAME_MAPPING = dict((v, k) for (k, v) in NAME_MAPPING.items()) diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/asyncio/events.py --- a/Lib/asyncio/events.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/asyncio/events.py Tue Dec 03 16:02:48 2013 +0100 @@ -99,7 +99,7 @@ class AbstractServer: - """Abstract server returned by create_server().""" + """Abstract server returned by create_service().""" def close(self): """Stop serving. This leaves existing connections open.""" diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/asyncio/locks.py --- a/Lib/asyncio/locks.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/asyncio/locks.py Tue Dec 03 16:02:48 2013 +0100 @@ -79,7 +79,7 @@ return '<{} [{}]>'.format(res[1:-1], extra) def locked(self): - """Return True if lock is acquired.""" + """Return true if lock is acquired.""" return self._locked @tasks.coroutine @@ -138,7 +138,7 @@ class Event: - """An Event implementation, asynchronous equivalent to threading.Event. + """An Event implementation, our equivalent to threading.Event. Class implementing event objects. An event manages a flag that can be set to true with the set() method and reset to false with the clear() method. @@ -162,7 +162,7 @@ return '<{} [{}]>'.format(res[1:-1], extra) def is_set(self): - """Return True if and only if the internal flag is true.""" + """Return true if and only if the internal flag is true.""" return self._value def set(self): @@ -204,7 +204,7 @@ class Condition: - """A Condition implementation, asynchronous equivalent to threading.Condition. + """A Condition implementation, our equivalent to threading.Condition. This class implements condition variable objects. A condition variable allows one or more coroutines to wait until they are notified by another diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/asyncio/transports.py --- a/Lib/asyncio/transports.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/asyncio/transports.py Tue Dec 03 16:02:48 2013 +0100 @@ -1,9 +1,5 @@ """Abstract Transport class.""" -import sys - -PY34 = sys.version_info >= (3, 4) - __all__ = ['ReadTransport', 'WriteTransport', 'Transport'] @@ -89,15 +85,11 @@ def writelines(self, list_of_data): """Write a list (or any iterable) of data bytes to the transport. - The default implementation concatenates the arguments and - calls write() on the result. + The default implementation just calls write() for each item in + the list/iterable. """ - if not PY34: - # In Python 3.3, bytes.join() doesn't handle memoryview. - list_of_data = ( - bytes(data) if isinstance(data, memoryview) else data - for data in list_of_data) - self.write(b''.join(list_of_data)) + for data in list_of_data: + self.write(data) def write_eof(self): """Close the write end after flushing buffered data. diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/distutils/command/build_ext.py --- a/Lib/distutils/command/build_ext.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/distutils/command/build_ext.py Tue Dec 03 16:02:48 2013 +0100 @@ -538,7 +538,7 @@ library_dirs=ext.library_dirs, runtime_library_dirs=ext.runtime_library_dirs, extra_postargs=extra_args, - export_symbols=ext.export_symbols, + export_symbols=self.get_export_symbols(ext), debug=self.debug, build_temp=self.build_temp, target_lang=language) diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/functools.py --- a/Lib/functools.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/functools.py Tue Dec 03 16:02:48 2013 +0100 @@ -338,7 +338,14 @@ def __hash__(self): return self.hashvalue -def _make_key(args, kwds, typed, +def _weaken(arg, callback=None): + """Try to make weak reference. Return the original object if it fails.""" + try: + return weakref.ref(arg, callback) + except TypeError: + return arg + +def _make_key(args, kwds, typed, remove_key=None, kwd_mark = (object(),), fasttypes = {int, str, frozenset, type(None)}, sorted=sorted, tuple=tuple, type=type, len=len): @@ -366,7 +373,41 @@ return key[0] return _HashedSeq(key) -def lru_cache(maxsize=128, typed=False): +def _make_weakref_key(args, kwds, typed, remove_key, + kwd_mark = (object(),), + fasttypes= {int, str, frozenset, type(None)}, + sorted=sorted, tuple=tuple, type=type, len=len): + """Like ``_make_key``, but use weak references to arguments. + + The function ``remove_key`` will be called back with a key when any of that + key's arguments no longer has any references. + + This function duplicates some code from ``_make_key`` for performance + reasons, to avoid additional checks and function calls in the original + function. + """ + key = None + + def weakref_callback(ref): + # The key here is stored in a closure, because we need this callback + # before we have the key, as it is used in the weakrefs for the key. + remove_key(key) + + key = tuple(_weaken(arg, weakref_callback) for arg in args) + if kwds: + sorted_items = sorted(kwds.items()) + key += kwd_mark + for key, value in sorted_items: + key += key, _weaken(value, weakref_callback) + if typed: + key += tuple(type(v) for v in args) + if kwds: + key += tuple(type(v) for k, v in sorted_items) + elif len(key) == 1 and type(key[0]) in fasttypes: + return key[0] + return _HashedSeq(key) + +def lru_cache(maxsize=128, typed=False, use_weakref=False): """Least-recently-used cache decorator. If *maxsize* is set to None, the LRU features are disabled and the cache @@ -376,6 +417,11 @@ For example, f(3.0) and f(3) will be treated as distinct calls with distinct results. + If *use_weakref* is True, the cache will not keep references to the + arguments, keeping them alive in memory forever. Instead, cache entries + related to objects that are no longer used will be automatically removed + from cache. + Arguments to the cached function must be hashable. View the cache statistics named tuple (hits, misses, maxsize, currsize) @@ -393,9 +439,13 @@ # Constants shared by all lru cache instances: sentinel = object() # unique object used to signal cache misses - make_key = _make_key # build a key from the function arguments PREV, NEXT, KEY, RESULT = 0, 1, 2, 3 # names for the link fields + if use_weakref: + make_key = _make_weakref_key + else: + make_key = _make_key # build a key from the function arguments + def decorating_function(user_function): cache = {} hits = misses = 0 @@ -404,6 +454,8 @@ lock = RLock() # because linkedlist updates aren't threadsafe root = [] # root of the circular doubly linked list root[:] = [root, root, None, None] # initialize by pointing to self + remove_key = None # only used with use_weakref + if maxsize == 0: @@ -415,11 +467,19 @@ return result elif maxsize is None: + if use_weakref: + + def remove_key(key): + # A callback for removing keys from cache + try: + del cache[key] + except KeyError: + pass def wrapper(*args, **kwds): # Simple caching without ordering or size limit nonlocal hits, misses - key = make_key(args, kwds, typed) + key = make_key(args, kwds, typed, remove_key) result = cache_get(key, sentinel) if result is not sentinel: hits += 1 @@ -430,11 +490,22 @@ return result else: + if use_weakref: + + def remove_key(key): + # A callback for removing keys from cache + with lock: + link = cache_get(key) + if link is not None: + link_prev, link_next, _key, result = link + link_prev[NEXT] = link_next + link_next[PREV] = link_prev + del cache[key] def wrapper(*args, **kwds): # Size limited caching that tracks accesses by recency nonlocal root, hits, misses, full - key = make_key(args, kwds, typed) + key = make_key(args, kwds, typed, remove_key) with lock: link = cache_get(key) if link is not None: diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/imaplib.py --- a/Lib/imaplib.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/imaplib.py Tue Dec 03 16:02:48 2013 +0100 @@ -745,9 +745,7 @@ ssl_context = ssl._create_stdlib_context() typ, dat = self._simple_command(name) if typ == 'OK': - server_hostname = self.host if ssl.HAS_SNI else None - self.sock = ssl_context.wrap_socket(self.sock, - server_hostname=server_hostname) + self.sock = ssl_context.wrap_socket(self.sock) self.file = self.sock.makefile('rb') self._tls_established = True self._get_capabilities() @@ -1218,9 +1216,7 @@ def _create_socket(self): sock = IMAP4._create_socket(self) - server_hostname = self.host if ssl.HAS_SNI else None - return self.ssl_context.wrap_socket(sock, - server_hostname=server_hostname) + return self.ssl_context.wrap_socket(sock) def open(self, host='', port=IMAP4_SSL_PORT): """Setup connection to remote server on "host:port". diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/nntplib.py --- a/Lib/nntplib.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/nntplib.py Tue Dec 03 16:02:48 2013 +0100 @@ -279,7 +279,7 @@ if _have_ssl: - def _encrypt_on(sock, context, hostname): + def _encrypt_on(sock, context): """Wrap a socket in SSL/TLS. Arguments: - sock: Socket to wrap - context: SSL context to use for the encrypted connection @@ -289,8 +289,7 @@ # Generate a default SSL context if none was passed. if context is None: context = ssl._create_stdlib_context() - server_hostname = hostname if ssl.HAS_SNI else None - return context.wrap_socket(sock, server_hostname=server_hostname) + return context.wrap_socket(sock) # The classes themselves @@ -1006,7 +1005,7 @@ resp = self._shortcmd('STARTTLS') if resp.startswith('382'): self.file.close() - self.sock = _encrypt_on(self.sock, context, self.host) + self.sock = _encrypt_on(self.sock, context) self.file = self.sock.makefile("rwb") self.tls_on = True # Capabilities may change after TLS starts up, so ask for them @@ -1066,7 +1065,7 @@ in default port and the `ssl_context` argument for SSL connections. """ self.sock = socket.create_connection((host, port), timeout) - self.sock = _encrypt_on(self.sock, ssl_context, host) + self.sock = _encrypt_on(self.sock, ssl_context) file = self.sock.makefile("rwb") _NNTPBase.__init__(self, file, host, readermode=readermode, timeout=timeout) diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/pathlib.py --- a/Lib/pathlib.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/pathlib.py Tue Dec 03 16:02:48 2013 +0100 @@ -6,12 +6,19 @@ import posixpath import re import sys -from collections import Sequence +import weakref +try: + import threading +except ImportError: + import dummy_threading as threading + +from collections import Sequence, defaultdict from contextlib import contextmanager from errno import EINVAL, ENOENT +from itertools import chain, count from operator import attrgetter from stat import S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO -from urllib.parse import quote_from_bytes as urlquote_from_bytes +from urllib.parse import quote as urlquote, quote_from_bytes as urlquote_from_bytes supports_symlinks = True @@ -932,15 +939,6 @@ # A stub for the opener argument to built-in open() return self._accessor.open(self, flags, mode) - def _raw_open(self, flags, mode=0o777): - """ - Open the file pointed by this path and return a file descriptor, - as os.open() does. - """ - if self._closed: - self._raise_closed() - return self._accessor.open(self, flags, mode) - # Public API @classmethod @@ -1047,6 +1045,15 @@ import grp return grp.getgrgid(self.stat().st_gid).gr_name + def _raw_open(self, flags, mode=0o777): + """ + Open the file pointed by this path and return a file descriptor, + as os.open() does. + """ + if self._closed: + self._raise_closed() + return self._accessor.open(self, flags, mode) + def open(self, mode='r', buffering=-1, encoding=None, errors=None, newline=None): """ diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/poplib.py --- a/Lib/poplib.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/poplib.py Tue Dec 03 16:02:48 2013 +0100 @@ -387,9 +387,7 @@ if context is None: context = ssl._create_stdlib_context() resp = self._shortcmd('STLS') - server_hostname = self.host if ssl.HAS_SNI else None - self.sock = context.wrap_socket(self.sock, - server_hostname=server_hostname) + self.sock = context.wrap_socket(self.sock) self.file = self.sock.makefile('rb') self._tls_established = True return resp @@ -430,9 +428,7 @@ def _create_socket(self, timeout): sock = POP3._create_socket(self, timeout) - server_hostname = self.host if ssl.HAS_SNI else None - sock = self.context.wrap_socket(sock, - server_hostname=server_hostname) + sock = self.context.wrap_socket(sock) return sock def stls(self, keyfile=None, certfile=None, context=None): diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/smtplib.py --- a/Lib/smtplib.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/smtplib.py Tue Dec 03 16:02:48 2013 +0100 @@ -232,7 +232,6 @@ will be used. """ - self._host = host self.timeout = timeout self.esmtp_features = {} self.source_address = source_address @@ -668,9 +667,7 @@ if context is None: context = ssl._create_stdlib_context(certfile=certfile, keyfile=keyfile) - server_hostname = self._host if ssl.HAS_SNI else None - self.sock = context.wrap_socket(self.sock, - server_hostname=server_hostname) + self.sock = context.wrap_socket(self.sock) self.file = None # RFC 3207: # The client MUST discard any knowledge obtained from @@ -895,9 +892,7 @@ print('connect:', (host, port), file=stderr) new_socket = socket.create_connection((host, port), timeout, self.source_address) - server_hostname = self._host if ssl.HAS_SNI else None - new_socket = self.context.wrap_socket(new_socket, - server_hostname=server_hostname) + new_socket = self.context.wrap_socket(new_socket) return new_socket __all__.append("SMTP_SSL") diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/ssl.py --- a/Lib/ssl.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/ssl.py Tue Dec 03 16:02:48 2013 +0100 @@ -405,7 +405,7 @@ def _create_stdlib_context(protocol=PROTOCOL_SSLv23, *, cert_reqs=None, - check_hostname=False, purpose=Purpose.SERVER_AUTH, + purpose=Purpose.SERVER_AUTH, certfile=None, keyfile=None, cafile=None, capath=None, cadata=None): """Create a SSLContext object for Python stdlib modules @@ -424,7 +424,6 @@ if cert_reqs is not None: context.verify_mode = cert_reqs - context.check_hostname = check_hostname if keyfile and not certfile: raise ValueError("certfile must be specified") diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/test/pickletester.py --- a/Lib/test/pickletester.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/test/pickletester.py Tue Dec 03 16:02:48 2013 +0100 @@ -413,71 +413,6 @@ # set([3]) pickled from 2.x with protocol 2 DATA6 = b'\x80\x02c__builtin__\nset\nq\x00]q\x01K\x03a\x85q\x02Rq\x03.' -python2_exceptions_without_args = ( - ArithmeticError, - AssertionError, - AttributeError, - BaseException, - BufferError, - BytesWarning, - DeprecationWarning, - EOFError, - EnvironmentError, - Exception, - FloatingPointError, - FutureWarning, - GeneratorExit, - IOError, - ImportError, - ImportWarning, - IndentationError, - IndexError, - KeyError, - KeyboardInterrupt, - LookupError, - MemoryError, - NameError, - NotImplementedError, - OSError, - OverflowError, - PendingDeprecationWarning, - ReferenceError, - RuntimeError, - RuntimeWarning, - # StandardError is gone in Python 3, we map it to Exception - StopIteration, - SyntaxError, - SyntaxWarning, - SystemError, - SystemExit, - TabError, - TypeError, - UnboundLocalError, - UnicodeError, - UnicodeWarning, - UserWarning, - ValueError, - Warning, - ZeroDivisionError, -) - -exception_pickle = b'\x80\x02cexceptions\n?\nq\x00)Rq\x01.' - -# Exception objects without arguments pickled from 2.x with protocol 2 -DATA7 = { - exception : - exception_pickle.replace(b'?', exception.__name__.encode("ascii")) - for exception in python2_exceptions_without_args -} - -# StandardError is mapped to Exception, test that separately -DATA8 = exception_pickle.replace(b'?', b'StandardError') - -# UnicodeEncodeError object pickled from 2.x with protocol 2 -DATA9 = (b'\x80\x02cexceptions\nUnicodeEncodeError\n' - b'q\x00(U\x05asciiq\x01X\x03\x00\x00\x00fooq\x02K\x00K\x01' - b'U\x03badq\x03tq\x04Rq\x05.') - def create_data(): c = C() @@ -1279,21 +1214,6 @@ self.assertEqual(list(loaded.keys()), ["key"]) self.assertEqual(loaded["key"].value, "Set-Cookie: key=value") - for (exc, data) in DATA7.items(): - loaded = self.loads(data) - self.assertIs(type(loaded), exc) - - loaded = self.loads(DATA8) - self.assertIs(type(loaded), Exception) - - loaded = self.loads(DATA9) - self.assertIs(type(loaded), UnicodeEncodeError) - self.assertEqual(loaded.object, "foo") - self.assertEqual(loaded.encoding, "ascii") - self.assertEqual(loaded.start, 0) - self.assertEqual(loaded.end, 1) - self.assertEqual(loaded.reason, "bad") - def test_pickle_to_2x(self): # Pickle non-trivial data with protocol 2, expecting that it yields # the same result as Python 2.x did. @@ -1416,26 +1336,6 @@ FRAME_SIZE_TARGET = 64 * 1024 - def check_frame_opcodes(self, pickled): - """ - Check the arguments of FRAME opcodes in a protocol 4+ pickle. - """ - frame_opcode_size = 9 - last_arg = last_pos = None - for op, arg, pos in pickletools.genops(pickled): - if op.name != 'FRAME': - continue - if last_pos is not None: - # The previous frame's size should be equal to the number - # of bytes up to the current frame. - frame_size = pos - last_pos - frame_opcode_size - self.assertEqual(frame_size, last_arg) - last_arg, last_pos = arg, pos - # The last frame's size should be equal to the number of bytes up - # to the pickle's end. - frame_size = len(pickled) - last_pos - frame_opcode_size - self.assertEqual(frame_size, last_arg) - def test_framing_many_objects(self): obj = list(range(10**5)) for proto in range(4, pickle.HIGHEST_PROTOCOL + 1): @@ -1449,7 +1349,6 @@ self.FRAME_SIZE_TARGET / 2) self.assertLessEqual(bytes_per_frame, self.FRAME_SIZE_TARGET * 1) - self.check_frame_opcodes(pickled) def test_framing_large_objects(self): N = 1024 * 1024 @@ -1461,7 +1360,6 @@ self.assertEqual(obj, unpickled) n_frames = count_opcode(pickle.FRAME, pickled) self.assertGreaterEqual(n_frames, len(obj)) - self.check_frame_opcodes(pickled) def test_optional_frames(self): if pickle.HIGHEST_PROTOCOL < 4: diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/test/test_asyncio/test_transports.py --- a/Lib/test/test_asyncio/test_transports.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/test/test_asyncio/test_transports.py Tue Dec 03 16:02:48 2013 +0100 @@ -24,18 +24,12 @@ transport = transports.Transport() transport.write = unittest.mock.Mock() - transport.writelines([b'line1', - bytearray(b'line2'), - memoryview(b'line3')]) - self.assertEqual(1, transport.write.call_count) - transport.write.assert_called_with(b'line1line2line3') + transport.writelines(['line1', 'line2', 'line3']) + self.assertEqual(3, transport.write.call_count) def test_not_implemented(self): transport = transports.Transport() - self.assertRaises(NotImplementedError, - transport.set_write_buffer_limits) - self.assertRaises(NotImplementedError, transport.get_write_buffer_size) self.assertRaises(NotImplementedError, transport.write, 'data') self.assertRaises(NotImplementedError, transport.write_eof) self.assertRaises(NotImplementedError, transport.can_write_eof) diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/test/test_imaplib.py --- a/Lib/test/test_imaplib.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/test/test_imaplib.py Tue Dec 03 16:02:48 2013 +0100 @@ -20,7 +20,6 @@ ssl = None CERTFILE = None -CAFILE = None class TestImaplib(unittest.TestCase): @@ -349,25 +348,6 @@ server_class = SecureTCPServer imap_class = IMAP4_SSL - @reap_threads - def test_ssl_verified(self): - ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - ssl_context.verify_mode = ssl.CERT_REQUIRED - ssl_context.check_hostname = True - ssl_context.load_verify_locations(CAFILE) - - with self.assertRaisesRegex(ssl.CertificateError, - "hostname '127.0.0.1' doesn't match 'localhost'"): - with self.reaped_server(SimpleIMAPHandler) as server: - client = self.imap_class(*server.server_address, - ssl_context=ssl_context) - client.shutdown() - - with self.reaped_server(SimpleIMAPHandler) as server: - client = self.imap_class("localhost", server.server_address[1], - ssl_context=ssl_context) - client.shutdown() - class RemoteIMAPTest(unittest.TestCase): host = 'cyrus.andrew.cmu.edu' @@ -480,15 +460,11 @@ if support.is_resource_enabled('network'): if ssl: - global CERTFILE, CAFILE + global CERTFILE CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, - "keycert3.pem") + "keycert.pem") if not os.path.exists(CERTFILE): raise support.TestFailed("Can't read certificate files!") - CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir, - "pycacert.pem") - if not os.path.exists(CAFILE): - raise support.TestFailed("Can't read CA file!") tests.extend([ ThreadedNetworkedTests, ThreadedNetworkedTestsSSL, RemoteIMAPTest, RemoteIMAP_SSLTest, RemoteIMAP_STARTTLSTest, diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/test/test_logging.py --- a/Lib/test/test_logging.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/test/test_logging.py Tue Dec 03 16:02:48 2013 +0100 @@ -590,7 +590,6 @@ for _ in range(tries): try: os.unlink(fname) - self.deletion_time = time.time() except OSError: pass time.sleep(0.004 * random.randint(0, 4)) @@ -598,9 +597,6 @@ del_count = 500 log_count = 500 - self.handle_time = None - self.deletion_time = None - for delay in (False, True): fd, fn = tempfile.mkstemp('.log', 'test_logging-3-') os.close(fd) @@ -614,14 +610,7 @@ for _ in range(log_count): time.sleep(0.005) r = logging.makeLogRecord({'msg': 'testing' }) - try: - self.handle_time = time.time() - h.handle(r) - except Exception: - print('Deleted at %s, ' - 'opened at %s' % (self.deletion_time, - self.handle_time)) - raise + h.handle(r) finally: remover.join() h.close() @@ -923,21 +912,19 @@ @unittest.skipUnless(threading, 'Threading required for this test.') class SMTPHandlerTest(BaseTest): - TIMEOUT = 8.0 def test_basic(self): sockmap = {} server = TestSMTPServer(('localhost', 0), self.process_message, 0.001, sockmap) server.start() addr = ('localhost', server.port) - h = logging.handlers.SMTPHandler(addr, 'me', 'you', 'Log', - timeout=self.TIMEOUT) + h = logging.handlers.SMTPHandler(addr, 'me', 'you', 'Log', timeout=5.0) self.assertEqual(h.toaddrs, ['you']) self.messages = [] r = logging.makeLogRecord({'msg': 'Hello'}) self.handled = threading.Event() h.handle(r) - self.handled.wait(self.TIMEOUT) # 14314: don't wait forever + self.handled.wait(5.0) # 14314: don't wait forever server.stop() self.assertTrue(self.handled.is_set()) self.assertEqual(len(self.messages), 1) diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/test/test_pathlib.py --- a/Lib/test/test_pathlib.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/test/test_pathlib.py Tue Dec 03 16:02:48 2013 +0100 @@ -1093,21 +1093,20 @@ with open(join('dirC', 'dirD', 'fileD'), 'wb') as f: f.write(b"this is file D\n") if not symlink_skip_reason: + if os.name == 'nt': + # Workaround for http://bugs.python.org/issue13772 + def dirlink(src, dest): + os.symlink(src, dest, target_is_directory=True) + else: + def dirlink(src, dest): + os.symlink(src, dest) # Relative symlinks os.symlink('fileA', join('linkA')) os.symlink('non-existing', join('brokenLink')) - self.dirlink('dirB', join('linkB')) - self.dirlink(os.path.join('..', 'dirB'), join('dirA', 'linkC')) + dirlink('dirB', join('linkB')) + dirlink(os.path.join('..', 'dirB'), join('dirA', 'linkC')) # This one goes upwards but doesn't create a loop - self.dirlink(os.path.join('..', 'dirB'), join('dirB', 'linkD')) - - if os.name == 'nt': - # Workaround for http://bugs.python.org/issue13772 - def dirlink(self, src, dest): - os.symlink(src, dest, target_is_directory=True) - else: - def dirlink(self, src, dest): - os.symlink(src, dest) + dirlink(os.path.join('..', 'dirB'), join('dirB', 'linkD')) def assertSame(self, path_a, path_b): self.assertTrue(os.path.samefile(str(path_a), str(path_b)), @@ -1270,16 +1269,6 @@ p = P(BASE, 'dirA', 'linkX', 'linkY', 'fileB') self._check_resolve_absolute(p, P(BASE, 'dirB', 'fileB')) - @with_symlinks - def test_resolve_dot(self): - # See https://bitbucket.org/pitrou/pathlib/issue/9/pathresolve-fails-on-complex-symlinks - p = self.cls(BASE) - self.dirlink('.', join('0')) - self.dirlink(os.path.join('0', '0'), join('1')) - self.dirlink(os.path.join('1', '1'), join('2')) - q = p / '2' - self.assertEqual(q.resolve(), p) - def test_with(self): p = self.cls(BASE) it = p.iterdir() diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/test/test_poplib.py --- a/Lib/test/test_poplib.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/test/test_poplib.py Tue Dec 03 16:02:48 2013 +0100 @@ -23,8 +23,7 @@ import ssl SUPPORTS_SSL = True - CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert3.pem") - CAFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "pycacert.pem") + CERTFILE = os.path.join(os.path.dirname(__file__) or os.curdir, "keycert.pem") requires_ssl = skipUnless(SUPPORTS_SSL, 'SSL not supported') # the dummy data returned by server when LIST and RETR commands are issued @@ -333,12 +332,6 @@ def test_stls_context(self): expected = b'+OK Begin TLS negotiation' ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) - ctx.load_verify_locations(CAFILE) - ctx.verify_mode = ssl.CERT_REQUIRED - ctx.check_hostname = True - with self.assertRaises(ssl.CertificateError): - resp = self.client.stls(context=ctx) - self.client = poplib.POP3("localhost", self.server.port, timeout=3) resp = self.client.stls(context=ctx) self.assertEqual(resp, expected) diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/test/test_smtpnet.py --- a/Lib/test/test_smtpnet.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/test/test_smtpnet.py Tue Dec 03 16:02:48 2013 +0100 @@ -3,35 +3,23 @@ import unittest from test import support import smtplib -import socket ssl = support.import_module("ssl") support.requires("network") -def check_ssl_verifiy(host, port): - context = ssl.create_default_context() - with socket.create_connection((host, port)) as sock: - try: - sock = context.wrap_socket(sock, server_hostname=host) - except Exception: - return False - else: - sock.close() - return True - class SmtpTest(unittest.TestCase): testServer = 'smtp.gmail.com' remotePort = 25 + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) def test_connect_starttls(self): support.get_attribute(smtplib, 'SMTP_SSL') - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) with support.transient_internet(self.testServer): server = smtplib.SMTP(self.testServer, self.remotePort) try: - server.starttls(context=context) + server.starttls(context=self.context) except smtplib.SMTPException as e: if e.args[0] == 'STARTTLS extension not supported by server.': unittest.skip(e.args[0]) @@ -44,7 +32,7 @@ class SmtpSSLTest(unittest.TestCase): testServer = 'smtp.gmail.com' remotePort = 465 - can_verify = check_ssl_verifiy(testServer, remotePort) + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) def test_connect(self): support.get_attribute(smtplib, 'SMTP_SSL') @@ -61,19 +49,9 @@ server.quit() def test_connect_using_sslcontext(self): - context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) support.get_attribute(smtplib, 'SMTP_SSL') with support.transient_internet(self.testServer): - server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=context) - server.ehlo() - server.quit() - - @unittest.skipUnless(can_verify, "SSL certificate can't be verified") - def test_connect_using_sslcontext_verified(self): - support.get_attribute(smtplib, 'SMTP_SSL') - context = ssl.create_default_context() - with support.transient_internet(self.testServer): - server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=context) + server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=self.context) server.ehlo() server.quit() diff -r f2c7f771bf6a -r 66c1c9f32567 Lib/test/test_ssl.py --- a/Lib/test/test_ssl.py Tue Dec 03 19:17:25 2013 +0100 +++ b/Lib/test/test_ssl.py Tue Dec 03 16:02:48 2013 +0100 @@ -1032,11 +1032,9 @@ self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1, - cert_reqs=ssl.CERT_REQUIRED, - check_hostname=True) + cert_reqs=ssl.CERT_REQUIRED) self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) - self.assertTrue(ctx.check_hostname) self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) diff -r f2c7f771bf6a -r 66c1c9f32567 Misc/NEWS --- a/Misc/NEWS Tue Dec 03 19:17:25 2013 +0100 +++ b/Misc/NEWS Tue Dec 03 16:02:48 2013 +0100 @@ -18,20 +18,6 @@ Library ------- -- Issue #19785: smtplib now supports SSLContext.check_hostname and server name - indication for TLS/SSL connections. - -- Issue #19784: poplib now supports SSLContext.check_hostname and server name - indication for TLS/SSL connections. - -- Issue #19783: nntplib now supports SSLContext.check_hostname and server name - indication for TLS/SSL connections. - -- Issue #19782: imaplib now supports SSLContext.check_hostname and server name - indication for TLS/SSL connections. - -- Issue #19834: Support unpickling of exceptions pickled by Python 2. - - Issue #19781: ftplib now supports SSLContext.check_hostname and server name indication for TLS/SSL connections. diff -r f2c7f771bf6a -r 66c1c9f32567 Modules/_decimal/libmpdec/mpdecimal.c --- a/Modules/_decimal/libmpdec/mpdecimal.c Tue Dec 03 19:17:25 2013 +0100 +++ b/Modules/_decimal/libmpdec/mpdecimal.c Tue Dec 03 16:02:48 2013 +0100 @@ -392,42 +392,42 @@ /* Dynamic decimal */ ALWAYS_INLINE int -mpd_isdynamic(const mpd_t *dec) +mpd_isdynamic(mpd_t *dec) { return !(dec->flags & MPD_STATIC); } /* Static decimal */ ALWAYS_INLINE int -mpd_isstatic(const mpd_t *dec) +mpd_isstatic(mpd_t *dec) { return dec->flags & MPD_STATIC; } /* Data of decimal is dynamic */ ALWAYS_INLINE int -mpd_isdynamic_data(const mpd_t *dec) +mpd_isdynamic_data(mpd_t *dec) { return !(dec->flags & MPD_DATAFLAGS); } /* Data of decimal is static */ ALWAYS_INLINE int -mpd_isstatic_data(const mpd_t *dec) +mpd_isstatic_data(mpd_t *dec) { return dec->flags & MPD_STATIC_DATA; } /* Data of decimal is shared */ ALWAYS_INLINE int -mpd_isshared_data(const mpd_t *dec) +mpd_isshared_data(mpd_t *dec) { return dec->flags & MPD_SHARED_DATA; } /* Data of decimal is const */ ALWAYS_INLINE int -mpd_isconst_data(const mpd_t *dec) +mpd_isconst_data(mpd_t *dec) { return dec->flags & MPD_CONST_DATA; } @@ -597,7 +597,7 @@ /* Copy sign from another decimal */ ALWAYS_INLINE void -mpd_signcpy(mpd_t *result, const mpd_t *a) +mpd_signcpy(mpd_t *result, mpd_t *a) { uint8_t sign = a->flags&MPD_NEG; diff -r f2c7f771bf6a -r 66c1c9f32567 Modules/_decimal/libmpdec/mpdecimal.h --- a/Modules/_decimal/libmpdec/mpdecimal.h Tue Dec 03 19:17:25 2013 +0100 +++ b/Modules/_decimal/libmpdec/mpdecimal.h Tue Dec 03 16:02:48 2013 +0100 @@ -752,12 +752,12 @@ /* 1 if dec is positive, -1 if dec is negative */ EXTINLINE int mpd_arith_sign(const mpd_t *dec); EXTINLINE long mpd_radix(void); -EXTINLINE int mpd_isdynamic(const mpd_t *dec); -EXTINLINE int mpd_isstatic(const mpd_t *dec); -EXTINLINE int mpd_isdynamic_data(const mpd_t *dec); -EXTINLINE int mpd_isstatic_data(const mpd_t *dec); -EXTINLINE int mpd_isshared_data(const mpd_t *dec); -EXTINLINE int mpd_isconst_data(const mpd_t *dec); +EXTINLINE int mpd_isdynamic(mpd_t *dec); +EXTINLINE int mpd_isstatic(mpd_t *dec); +EXTINLINE int mpd_isdynamic_data(mpd_t *dec); +EXTINLINE int mpd_isstatic_data(mpd_t *dec); +EXTINLINE int mpd_isshared_data(mpd_t *dec); +EXTINLINE int mpd_isconst_data(mpd_t *dec); EXTINLINE mpd_ssize_t mpd_trail_zeros(const mpd_t *dec); @@ -769,7 +769,7 @@ EXTINLINE void mpd_setdigits(mpd_t *result); EXTINLINE void mpd_set_sign(mpd_t *result, uint8_t sign); /* copy sign from another decimal */ -EXTINLINE void mpd_signcpy(mpd_t *result, const mpd_t *a); +EXTINLINE void mpd_signcpy(mpd_t *result, mpd_t *a); EXTINLINE void mpd_set_infinity(mpd_t *result); EXTINLINE void mpd_set_qnan(mpd_t *result); EXTINLINE void mpd_set_snan(mpd_t *result);