better way to handle mpi

This commit is contained in:
Noe Brucy
2022-08-26 15:50:11 +02:00
parent 08b7345036
commit 7025694969
2 changed files with 7 additions and 7 deletions
+4 -7
View File
@@ -4,16 +4,13 @@ import numpy as np
from functools import partial from functools import partial
from baseprocessor import Rule from baseprocessor import Rule
import snapshotprocessor import snapshotprocessor
from mypool import MyPool
try: try:
from mpi4py.futures import MPIPoolExecutor from mpi4py.futures import MPIPoolExecutor
mpi_loaded = True
mpi = True
except ModuleNotFoundError: except ModuleNotFoundError:
from mypool import MyPool mpi_loaded = False
mpi = False
def _map_aux(fun, path, path_out, params, run_num, **kwargs): def _map_aux(fun, path, path_out, params, run_num, **kwargs):
@@ -57,7 +54,7 @@ class Aggregator:
map_fn = partial( map_fn = partial(
_map_aux, func, self.path, self.path_out, self.params, **kwargs _map_aux, func, self.path, self.path_out, self.params, **kwargs
) )
if mpi: if mpi_loaded and self.params.process.mpi:
executor = MPIPoolExecutor(max_workers=num_process) executor = MPIPoolExecutor(max_workers=num_process)
try: try:
result = list(executor.map(map_fn, run_num, unordered=True)) result = list(executor.map(map_fn, run_num, unordered=True))
+3
View File
@@ -96,6 +96,8 @@ out: # Parameters for post processing
process: # General setting of the post-processor module process: # General setting of the post-processor module
verbose : True # Give more infos on what is going on verbose : True # Give more infos on what is going on
num_process : 1 # Number of forks num_process : 1 # Number of forks
mpi : False # If available, use mpi4py
allow_error : False # Catch all errors in rules involving the disk
save_cells : True # Save cells structure on disk save_cells : True # Save cells structure on disk
save_parts : True # Save particles on disk save_parts : True # Save particles on disk
unload_cells : True # Save memory usage unload_cells : True # Save memory usage
@@ -105,6 +107,7 @@ rules: # Specific rules parameters
astrophysix: # Parameters for astrophysix and galactica astrophysix: # Parameters for astrophysix and galactica
generate : False # Whether to generate astrophysix objects
simu_fmt : "{tag}_{run}" # Format of the name of simulation simu_fmt : "{tag}_{run}" # Format of the name of simulation
descr_fmt : "{tag}_{run}" # Format of the default description descr_fmt : "{tag}_{run}" # Format of the default description
# The following keys are accepted # The following keys are accepted