diff --git a/Lib/ftplib.py b/Lib/ftplib.py --- a/Lib/ftplib.py +++ b/Lib/ftplib.py @@ -40,6 +40,7 @@ import sys import socket import warnings +import select from socket import _GLOBAL_DEFAULT_TIMEOUT __all__ = ["FTP","Netrc"] @@ -102,6 +103,7 @@ welcome = None passiveserver = 1 encoding = "latin-1" + use_sendfile = hasattr(os, 'sendfile') # Initialization method (called by class instantiation). # Initialize host to localhost, port to standard ftp port @@ -465,7 +467,8 @@ cmd: A STOR command. fp: A file-like object with a read(num_bytes) method. blocksize: The maximum data size to read from fp and send over - the connection at once. [default: 8192] + the connection at once. Ignored when using sendfile(). + [default: 8192] callback: An optional single parameter callable that is called on on each block of data after it is sent. [default: None] rest: Passed to transfercmd(). [default: None] @@ -473,13 +476,52 @@ Returns: The response code. """ + class _GiveupOnSendfile(Exception): + pass + + def _use_sendfile(): + offset = 0 + try: + sockno = conn.fileno() + fileno = fp.fileno() + blocksize = os.fstat(fileno).st_size or blocksize + while True: + # block until socket is writable + select.select([], [sockno], []) + sent = os.sendfile(sockno, fileno, offset, blocksize) + if sent == 0: + break + offset += sent + except Exception as err: + if offset == 0: + # we might get here for different reasons, the main + # one being fp was not a regular mmap(2)-like file, + # in which case we'll fall back on using plain send() + if self.debugging: + print("couldn't use sendfile() %r; falling back on " \ + "using send" % err) + raise _GiveupOnSendfile() + raise + + def _use_send(): + while True: + buf = fp.read(blocksize) + if not buf: + break + conn.sendall(buf) + if callback: + callback(buf) + self.voidcmd('TYPE I') with self.transfercmd(cmd, rest) as conn: - while 1: - buf = fp.read(blocksize) - if not buf: break - conn.sendall(buf) - if callback: callback(buf) + if callback or not self.use_sendfile: + _use_send() + else: + try: + _use_sendfile() + except _GiveupOnSendfile: + _use_send() + return self.voidresp() def storlines(self, cmd, fp, callback=None): diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -11,6 +11,7 @@ import errno import os import time +import unittest try: import ssl except ImportError: @@ -18,7 +19,7 @@ from unittest import TestCase from test import support -from test.support import HOST +from test.support import HOST, TESTFN threading = support.import_module('threading') # the dummy data returned by server over the data channel when @@ -578,6 +579,17 @@ self.client.storbinary('stor', f, rest=r) self.assertEqual(self.server.handler_instance.rest, str(r)) + @unittest.skipUnless(hasattr(os, 'sendfile'), 'os.sendfile() not available') + def test_storbinary_sendfile(self): + with open(TESTFN, 'wb+') as f: + test_data = 'abcde12345\r\n' * 100000 + f.write(test_data.encode('ascii')) + f = open(TESTFN, 'rb') + f.seek(0) + self.client.storbinary('stor', f) + self.assertEqual(self.server.handler_instance.last_received_data, + test_data) + def test_storlines(self): f = io.BytesIO(RETR_DATA.replace('\r\n', '\n').encode('ascii')) self.client.storlines('stor', f) @@ -1009,6 +1021,7 @@ support.run_unittest(*tests) finally: support.threading_cleanup(*thread_info) + support.unlink(TESTFN) if __name__ == '__main__':