diff --git a/Lib/ftplib.py b/Lib/ftplib.py --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -36,10 +36,12 @@ # Modified by Giampaolo Rodola' to add TLS support. # +import io import os import sys import socket import warnings +import select from socket import _GLOBAL_DEFAULT_TIMEOUT __all__ = ["FTP", "Netrc"] @@ -102,6 +104,7 @@ welcome = None passiveserver = 1 encoding = "latin-1" + use_sendfile = hasattr(os, 'sendfile') # Initialization method (called by class instantiation). # Initialize host to localhost, port to standard ftp port @@ -492,15 +495,86 @@ Returns: The response code. """ - self.voidcmd('TYPE I') - with self.transfercmd(cmd, rest) as conn: - while 1: + class _GiveupOnSendfile(Exception): + pass + + def _use_sendfile(): + offset = 0 + sockno = conn.fileno() + try: + fileno = fp.fileno() + except (AttributeError, io.UnsupportedOperation) as err: + raise _GiveupOnSendfile(err) # not a regular mmap-like file + + # set socket in non-blocking mode + conn.settimeout(0) + timeout = self.timeout if self.timeout != \ + _GLOBAL_DEFAULT_TIMEOUT else None + # whenever possible use poll() instead of select() in + # order to avoid running out of fds + if hasattr(select, 'poll'): + if timeout != None: + timeout *= 1000 + pollster = select.poll() + pollster.register(sockno, select.POLLOUT) + def wait_for_fd(): + if pollster.poll(timeout) == []: + raise socket.timeout('timed out') + else: + # call select() once in order to solicit EMFILE, in + # which case we'll exit immediately + try: + select.select([], [sockno], [], 0) + except OSError as err: + raise _GiveupOnSendfile(err) + + def wait_for_fd(): + fds = select.select([], [sockno], [], timeout) + if fds == ([], [], []): + raise socket.timeout('timed out') + + while True: + wait_for_fd() # block until socket is writable + try: + sent = os.sendfile(sockno, fileno, offset, blocksize) + except BlockingIOError: + continue + except OSError as err: + if offset == 0: + # We can get here for different reasons, the main + # one being fp is not a regular mmap(2)-like file, + # in which case we'll fall back on using plain + # send(). + conn.settimeout(self.timeout) + raise _GiveupOnSendfile(err) + else: + raise + else: + if sent == 0: + break + offset += sent + + def _use_send(): + while True: buf = fp.read(blocksize) if not buf: break conn.sendall(buf) if callback: callback(buf) + + self.voidcmd('TYPE I') + with self.transfercmd(cmd, rest) as conn: + if callback or not self.use_sendfile or isinstance(conn, _SSLSocket): + _use_send() + else: + try: + _use_sendfile() + except _GiveupOnSendfile as err: + if self.debugging: + print("couldn't use sendfile() %r; falling back on " \ + "using send" % err) + _use_send() # shutdown ssl layer if isinstance(conn, _SSLSocket): conn.unwrap() diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -11,6 +11,7 @@ import errno import os import time +import unittest try: import ssl except ImportError: @@ -18,7 +19,7 @@ from unittest import TestCase from test import support -from test.support import HOST +from test.support import HOST, TESTFN threading = support.import_module('threading') # the dummy data returned by server over the data channel when @@ -578,6 +579,17 @@ self.client.storbinary('stor', f, rest=r) self.assertEqual(self.server.handler_instance.rest, str(r)) + @unittest.skipUnless(hasattr(os, 'sendfile'), 'os.sendfile() not available') + def test_storbinary_sendfile(self): + with open(TESTFN, 'wb+') as f: + test_data = 'abcde12345\r\n' * 100000 + f.write(test_data.encode('ascii')) + f = open(TESTFN, 'rb') + f.seek(0) + self.client.storbinary('stor', f) + self.assertEqual(self.server.handler_instance.last_received_data, + test_data) + def test_storlines(self): f = io.BytesIO(RETR_DATA.replace('\r\n', '\n').encode('ascii')) self.client.storlines('stor', f) @@ -1009,6 +1021,7 @@ support.run_unittest(*tests) finally: support.threading_cleanup(*thread_info) + support.unlink(TESTFN) if __name__ == '__main__':