better way to handle mpi
This commit is contained in:
+4
-7
@@ -4,16 +4,13 @@ import numpy as np
|
|||||||
from functools import partial
|
from functools import partial
|
||||||
from baseprocessor import Rule
|
from baseprocessor import Rule
|
||||||
import snapshotprocessor
|
import snapshotprocessor
|
||||||
|
from mypool import MyPool
|
||||||
|
|
||||||
try:
|
try:
|
||||||
from mpi4py.futures import MPIPoolExecutor
|
from mpi4py.futures import MPIPoolExecutor
|
||||||
|
mpi_loaded = True
|
||||||
mpi = True
|
|
||||||
except ModuleNotFoundError:
|
except ModuleNotFoundError:
|
||||||
from mypool import MyPool
|
mpi_loaded = False
|
||||||
|
|
||||||
mpi = False
|
|
||||||
|
|
||||||
|
|
||||||
def _map_aux(fun, path, path_out, params, run_num, **kwargs):
|
def _map_aux(fun, path, path_out, params, run_num, **kwargs):
|
||||||
@@ -57,7 +54,7 @@ class Aggregator:
|
|||||||
map_fn = partial(
|
map_fn = partial(
|
||||||
_map_aux, func, self.path, self.path_out, self.params, **kwargs
|
_map_aux, func, self.path, self.path_out, self.params, **kwargs
|
||||||
)
|
)
|
||||||
if mpi:
|
if mpi_loaded and self.params.process.mpi:
|
||||||
executor = MPIPoolExecutor(max_workers=num_process)
|
executor = MPIPoolExecutor(max_workers=num_process)
|
||||||
try:
|
try:
|
||||||
result = list(executor.map(map_fn, run_num, unordered=True))
|
result = list(executor.map(map_fn, run_num, unordered=True))
|
||||||
|
|||||||
@@ -96,6 +96,8 @@ out: # Parameters for post processing
|
|||||||
process: # General setting of the post-processor module
|
process: # General setting of the post-processor module
|
||||||
verbose : True # Give more infos on what is going on
|
verbose : True # Give more infos on what is going on
|
||||||
num_process : 1 # Number of forks
|
num_process : 1 # Number of forks
|
||||||
|
mpi : False # If available, use mpi4py
|
||||||
|
allow_error : False # Catch all errors in rules involving the disk
|
||||||
save_cells : True # Save cells structure on disk
|
save_cells : True # Save cells structure on disk
|
||||||
save_parts : True # Save particles on disk
|
save_parts : True # Save particles on disk
|
||||||
unload_cells : True # Save memory usage
|
unload_cells : True # Save memory usage
|
||||||
@@ -105,6 +107,7 @@ rules: # Specific rules parameters
|
|||||||
|
|
||||||
|
|
||||||
astrophysix: # Parameters for astrophysix and galactica
|
astrophysix: # Parameters for astrophysix and galactica
|
||||||
|
generate : False # Whether to generate astrophysix objects
|
||||||
simu_fmt : "{tag}_{run}" # Format of the name of simulation
|
simu_fmt : "{tag}_{run}" # Format of the name of simulation
|
||||||
descr_fmt : "{tag}_{run}" # Format of the default description
|
descr_fmt : "{tag}_{run}" # Format of the default description
|
||||||
# The following keys are accepted
|
# The following keys are accepted
|
||||||
|
|||||||
Reference in New Issue
Block a user