from concurrent import futures import multiprocessing import math import sys, time import functools MIN = 100000000 MAX = 100100000 def is_prime(n): if n % 2 == 0: return False sqrt_n = int(math.floor(math.sqrt(n))) for i in range(3, sqrt_n + 1, 2): if n % i == 0: return False return True def fn2(fn, chunk): return [fn(*args) for args in zip(*chunk)] class FastProcessPoolExecutor(futures.ProcessPoolExecutor): def map(self, fn, *iterables, timeout=None, chunksize=None): try: iter_len = min(len(i) for i in iterables) except TypeError: # At least one iterable has no length return super().map(fn, *iterables, timeout) fn3 = functools.partial(fn2, fn) if chunksize is None: chunksize, extra = divmod(iter_len, self._max_workers * 4) if extra: chunksize += 1 if timeout is not None: end_time = timeout + time.time() fs = [self.submit(fn3, [iterable[ch:ch+chunksize] for iterable in iterables]) for ch in range(0, iter_len, chunksize)] try: for future in fs: if timeout is None: for i in future.result(): yield i else: for i in future.result(end_time - time.time()): yield i finally: for future in fs: future.cancel() print("Starting multiproc...", end="") start = time.time() p = multiprocessing.Pool(4) l = p.map(is_prime, range(MIN, MAX)) stop = time.time() print("done in {} s.".format(stop - start)) print("Starting futures...", end="") start = time.time() e = futures.ProcessPoolExecutor(4) f = e.map(is_prime, range(MIN, MAX)) l2 = list(f) stop = time.time() print("done in {} s.".format(stop - start)) print('Starting futures "fixed"...', end="") start = time.time() e = FastProcessPoolExecutor(4) f = e.map(is_prime, range(MIN, MAX)) l3 = list(f) stop = time.time() print("done in {} s.".format(stop - start)) print("l == l2:", l == l2) print("l == l3:", l == l3)