diff -r fca745bc70be Lib/multiprocessing/heap.py --- a/Lib/multiprocessing/heap.py Sat Jun 25 16:31:06 2011 +0200 +++ b/Lib/multiprocessing/heap.py Sun Jun 26 19:57:56 2011 +0200 @@ -101,6 +101,8 @@ self._stop_to_block = {} self._allocated_blocks = set() self._arenas = [] + # list of pending blocks to free - see free() comment below + self._pending_free_blocks = [] @staticmethod def _roundup(n, alignment): @@ -175,15 +177,39 @@ return start, stop + def _free_pending_blocks(self): + # Free all the blocks in the pending list - called with the lock held. + while True: + try: + block = self._pending_free_blocks.pop() + except IndexError: + break + self._free(block) + self._allocated_blocks.remove(block) + def free(self, block): # free a block returned by malloc() + # Since free() can be called asynchronously by the GC, it could happen + # that it's called while self._lock is held: in that case, + # self._lock.acquire() would deadlock (issue #12352). To avoid that, a + # trylock is used instead, and if the lock can't be acquired + # immediately, the block is added to a list of blocks to be freed + # synchronously sometimes later from malloc() or free(), by calling + # _free_pending_blocks() (appending and retrieving from a list is not + # strictly thread-safe but under cPython it's atomic thanks to the GIL). assert os.getpid() == self._lastpid - self._lock.acquire() - try: - self._allocated_blocks.remove(block) - self._free(block) - finally: - self._lock.release() + if not self._lock.acquire(0): + # can't acquire the lock, add it to the list of pending blocks to + # free + self._pending_free_blocks.append(block) + else: + # we hold the lock + try: + self._free_pending_blocks() + self._allocated_blocks.remove(block) + self._free(block) + finally: + self._lock.release() def malloc(self, size): # return a block of right size (possibly rounded up) @@ -191,6 +217,7 @@ if os.getpid() != self._lastpid: self.__init__() # reinitialize after fork self._lock.acquire() + self._free_pending_blocks() try: size = self._roundup(max(size,1), self._alignment) (arena, start, stop) = self._malloc(size) diff -r fca745bc70be Lib/test/test_multiprocessing.py --- a/Lib/test/test_multiprocessing.py Sat Jun 25 16:31:06 2011 +0200 +++ b/Lib/test/test_multiprocessing.py Sun Jun 26 19:57:56 2011 +0200 @@ -1737,7 +1737,31 @@ (narena, nstart, nstop) = all[i+1][:3] self.assertTrue((arena != narena and nstart == 0) or (stop == nstart)) - + + def test_free_from_gc(self): + # Check that freeing of blocks by the garbage collector doesn't deadlock + # (issue #12352). + # Make sure the GC is enabled, and set lower collection thresholds to + # make collections more frequent (and increase the probability of + # deadlock). + if gc.isenabled(): + thresholds = gc.get_threshold() + self.addCleanup(gc.set_threshold, *thresholds) + else: + gc.enable() + self.addCleanup(gc.disable) + gc.set_threshold(10) + + # perform numerous block allocations, with cyclic references to make + # sure objects are collected asynchronously by the gc + for i in range(5000): + a = multiprocessing.heap.BufferWrapper(1) + b = multiprocessing.heap.BufferWrapper(1) + # circular references + a.buddy = b + b.buddy = a + + # # #