import os from multiprocessing import Pool def read_log(location): # r=root, d=directories, f = files ''' Get file path contain 'log' ''' for r, d, f in os.walk(location): for file in f: if "log" in file: yield os.path.join(r, file) def process_file(log_files): ''' Read every line from log_files ''' for log_file in log_files: with open(log_file, 'r') as file: try: for f in file: yield (log_file, f) except UnicodeDecodeError: pass def foo(lines): for line in lines: return line if __name__ == '__main__': with Pool(os.cpu_count()) as p: p.map(foo, process_file(read_log('./')))