Index: Lib/mailbox.py =================================================================== --- Lib/mailbox.py (revision 53467) +++ Lib/mailbox.py (working copy) @@ -20,6 +20,8 @@ import email.Generator import rfc822 import StringIO +import shutil +import warnings try: if sys.platform == 'os2emx': # OS/2 EMX fcntl() not adequate @@ -556,6 +558,13 @@ if not self._locked: _lock_file(self._file) self._locked = True + if self._pending: + warnings.warn("mailbox .lock() method called with pending changes; " + "should have been locked before making changes", + stacklevel=2) + else: + # Force re-read of toc to ensure we're up-to-date. + self._toc = None def unlock(self): """Unlock the mailbox if it is locked.""" @@ -582,6 +591,15 @@ '(expected %i, found %i)' % (self._file_length, cur_len)) + if fcntl and self._locked and not hasattr(self._file, 'truncate'): + warnings.warn('as file.truncate() is unavailable, flush() may ' + 'momentarily release the fcntl lock; if you depend ' + 'on fcntl locking, you should regard flush() as ' + 'invalidating the message keys', RuntimeWarning, + stacklevel=2) + + orig_file = self._file + remove_temp_file = True new_file = _create_temporary(self._path) try: new_toc = {} @@ -599,27 +617,45 @@ new_file.write(buffer) new_toc[key] = (new_start, new_file.tell()) self._post_message_hook(new_file) - except: - new_file.close() - os.remove(new_file.name) - raise - _sync_close(new_file) - # self._file is about to get replaced, so no need to sync. - self._file.close() - try: - os.rename(new_file.name, self._path) - except OSError, e: - if e.errno == errno.EEXIST or \ - (os.name == 'os2' and e.errno == errno.EACCES): - os.remove(self._path) - os.rename(new_file.name, self._path) - else: - raise - self._file = open(self._path, 'rb+') - self._toc = new_toc - self._pending = False - if self._locked: - _lock_file(self._file, dotlock=False) + new_len = new_file.tell() + _sync_flush(new_file) + new_file.seek(0) + self._file.seek(0) + if new_len < cur_len and not hasattr(self._file, 'truncate'): + try: + if not os.path.samestat(os.fstat(self._file.fileno()), + os.stat(self._path)): + raise ExternalClashError("Mailbox has been replaced: " + "%s" % self._path) + except OSError, e: + if e.errno == errno.ENOENT: + raise NoSuchMailboxError(self._path) + raise + except AttributeError: + # No stat(), etc. + pass + # *** race condition *** + remove_temp_file = False + self._file = open(self._path, 'wb+') + print 'doing open' + remove_temp_file = False + shutil.copyfileobj(new_file, self._file) + if hasattr(self._file, 'truncate'): + self._file.truncate(new_len) + self._toc = new_toc + _sync_flush(self._file) + remove_temp_file = True + self._pending = False + finally: + try: + new_file.close() + if remove_temp_file: + os.remove(new_file.name) + finally: + if self._file is not orig_file: + orig_file.close() + if self._locked: + _lock_file(self._file, dotlock=False) def _pre_mailbox_hook(self, f): """Called before writing the mailbox to file f.""" Index: Lib/test/test_mailbox.py =================================================================== --- Lib/test/test_mailbox.py (revision 53467) +++ Lib/test/test_mailbox.py (working copy) @@ -416,6 +416,68 @@ self.assertRaises(TypeError, lambda: self._box._dump_message(None, output)) + def test_concurrent_add(self): + # Simple test of concurrent addition to a mailbox. + # This exercises the add() and flush() methods, based on bug #1599254. + # This bug affected only the classes based on _singlefileMailbox + # (mbox, MMDF, Babyl), but this test can apply to any mailbox type. + + # If fork isn't available, skip this test. + # XXX Could someone please port this to Windows? + if not hasattr(os, 'fork'): + return + + def random_message (): + # Generate a random message body + import random + body = "" + for i in range(random.randint(1, 10)): + line = "a" * random.randint(0, 75) + '\n' + body += line + + return body + + def add_25_messages (): + "Helper function to add 25 messages to a mailbox." + mbox = self._box + + for i in range(25): + msg = """Subject: %i, pid %i +From: sender@example.com + +Content goes here. +%s""" % (i, os.getpid(), random_message()) + while True: + try: + mbox.lock() + except mailbox.ExternalClashError: + # In case of conflict, wait a bit and try again. + time.sleep(0.01) + else: + break + mbox.add(msg) + mbox.flush() + mbox.unlock() + + # Fire off a subprocess that will add 25 messages to a mailbox + # file, locking and unlocking it each time. The parent process + # will do the same. The resulting mailbox should contain 50 messages. + pid = os.fork() + if pid == 0: + try: + add_25_messages() + finally: + os._exit(0) + + # Add our 25 messages, and wait for the child to exit. + add_25_messages() + os.wait() + + # We expect the mailbox to contain 50 messages. + self._box.lock() + self.assert_(len(self._box) == 50) + self._box.unlock() + def _get_lock_path(self): # Return the path of the dot lock file. May be overridden. return self._path + '.lock'