30 wiersze
834 B
Python
30 wiersze
834 B
Python
"""
|
|
Execute a process in parallel where individual runs
|
|
may spawn more runs.
|
|
"""
|
|
|
|
def parallel(fn, start, processes=6):
|
|
with multiprocessing.Manager() as manager:
|
|
results = manager.dict()
|
|
unchecked = manager.dict({key: None for key in start })
|
|
|
|
with multiprocessing.Pool(processes) as pool:
|
|
pool.map(execute_process, [(results, unchecked, fn) for _ in range(processes)])
|
|
|
|
return dict(results)
|
|
|
|
def execute_process(args):
|
|
results, unchecked, fn = args
|
|
|
|
while len(unchecked) != 0:
|
|
key, _ = unchecked.popitem()
|
|
|
|
try:
|
|
results[key], unchecked_ = fn(key, results=results, unchecked=unchecked)
|
|
|
|
for k in unchecked_:
|
|
if k not in results:
|
|
unchecked[k] = None
|
|
except Exception as e:
|
|
continue
|