from multiprocessing import Pool
import os
import logging
import sys
import time

stdout_handler = logging.StreamHandler(sys.stdout)
handlers = [stdout_handler]

logging.basicConfig(
    level=logging.DEBUG,
    format='[%(asctime)s] {%(filename)s:%(lineno)d} %(levelname)s - %(message)s',
    handlers=handlers
)

log = logging.getLogger('LOGGER_NAME')

i = 0

def f(x):
  log.info("pid: %s got: %s"% (os.getpid(), x))
  return [x, x+1]

def cb(r):
  global i
  i -= 1
  log.info("got result: %s"%(r))

if __name__ == '__main__':
  pool = Pool(processes=5, maxtasksperchild=5)
  keys = range(1000)
  for key in keys:
    i += 1
    while i > 10:
        log.warn("Max job limit reached. Sleeping...")
        time.sleep(0.1)
    result = pool.apply_async(f, (key,), callback=cb)
  pool.close()
  pool.join()
  log.info("i=%d"%i)