better way to handle mpi
This commit is contained in:
+4
-7
@@ -4,16 +4,13 @@ import numpy as np
|
||||
from functools import partial
|
||||
from baseprocessor import Rule
|
||||
import snapshotprocessor
|
||||
|
||||
from mypool import MyPool
|
||||
|
||||
try:
|
||||
from mpi4py.futures import MPIPoolExecutor
|
||||
|
||||
mpi = True
|
||||
mpi_loaded = True
|
||||
except ModuleNotFoundError:
|
||||
from mypool import MyPool
|
||||
|
||||
mpi = False
|
||||
mpi_loaded = False
|
||||
|
||||
|
||||
def _map_aux(fun, path, path_out, params, run_num, **kwargs):
|
||||
@@ -57,7 +54,7 @@ class Aggregator:
|
||||
map_fn = partial(
|
||||
_map_aux, func, self.path, self.path_out, self.params, **kwargs
|
||||
)
|
||||
if mpi:
|
||||
if mpi_loaded and self.params.process.mpi:
|
||||
executor = MPIPoolExecutor(max_workers=num_process)
|
||||
try:
|
||||
result = list(executor.map(map_fn, run_num, unordered=True))
|
||||
|
||||
Reference in New Issue
Block a user