import asyncio import concurrent.futures import os.path import subprocess import sys import traceback @asyncio.coroutine def getcommandoutput(*args): # Slight modification of the function getstatusoutput present in: # https://docs.python.org/3/library/asyncio-subprocess.html#example proc = yield from asyncio.create_subprocess_exec( *args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) try: stdout, stderr = yield from proc.communicate() except: proc.kill() yield from proc.wait() raise exitcode = yield from proc.wait() return (exitcode, stdout, stderr) MAX_PROCESSES = 10 ITERATIONS = 10**4 def check_results(done): errors = 0 for future in done: try: future.result() except Exception as e: errors += 1 traceback.print_exc() return errors @asyncio.coroutine def test(): running = set() errors = 0 for i in range(ITERATIONS): filename = __file__ if i % 100 == 0: print(i) while len(running) > MAX_PROCESSES: done, pending = yield from asyncio.wait( running, timeout=1.0, return_when=concurrent.futures.FIRST_COMPLETED) errors += check_results(done) if errors: print('errors') return running = pending if not done: yield from asyncio.sleep(1.0) running.add(getcommandoutput('du', '--bytes', filename)) done, pending = yield from asyncio.wait(running) errors += check_results(done) assert errors == 0 if __name__ == '__main__': if len(sys.argv) > 1: MAX_PROCESSES = int(sys.argv[1]) if len(sys.argv) > 2: ITERATIONS = int(sys.argv[2]) loop = asyncio.get_event_loop() loop.run_until_complete(test())