Huge refactoring

This commit is contained in:
Noe Brucy
2021-06-22 12:23:03 +02:00
parent a286c08b72
commit 0dc9e8fc7b
12 changed files with 343 additions and 361 deletions
+14 -14
View File
@@ -2,7 +2,7 @@
import numpy as np
from functools import partial
import postprocessor
import snapshotprocessor
try:
from mpi4py.futures import MPIPoolExecutor
@@ -14,44 +14,44 @@ except ModuleNotFoundError:
mpi = False
def _map_aux(fun, path, path_out, pp_params, run_num, **kwargs):
def _map_aux(fun, path, path_out, params, run_num, **kwargs):
try:
pp = postprocessor.PostProcessor(
path + "/" + run_num[0], run_num[1], path_out + "/" + run_num[0], pp_params
snap = snapshotprocessor.SnapshotProcessor(
path + "/" + run_num[0], run_num[1], path_out + "/" + run_num[0], params
)
except Exception as e:
print(e)
raise
return fun(pp, **kwargs)
return fun(snap, **kwargs)
def _map_rule(pp, rule, arg, overwrite, overwrite_dep):
return pp.process(rule, arg, overwrite, overwrite_dep)
def _map_rule(snap, rule, arg, overwrite, overwrite_dep):
return snap.process(rule, arg, overwrite, overwrite_dep)
class Aggregator:
def get_pp_list(self, select=None):
def get_snap_list(self, select=None):
if select is not None:
runs, nums = self.selector.select(**select)
else:
runs = self.runs
nums = self.nums
return [self.pp[run][num] for run in runs for num in nums[run]]
return [self.snaps[run][num] for run in runs for num in nums[run]]
def map(self, func, select=None, num_process=None, **kwargs):
pp_list = self.get_pp_list(select)
snaps = self.get_snap_list(select)
if num_process is None:
num_process = self.pp_params.process.num_process
num_process = self.params.process.num_process
if num_process == 1:
result = [func(pp, **kwargs) for pp in pp_list]
result = [func(snap, **kwargs) for snap in snaps]
else:
run_num = [(pp.run, pp.num) for pp in pp_list]
run_num = [(snap.run, snap.num) for snap in snaps]
map_fn = partial(
_map_aux, func, self.path, self.path_out, self.pp_params, **kwargs
_map_aux, func, self.path, self.path_out, self.params, **kwargs
)
if mpi:
executor = MPIPoolExecutor(max_workers=num_process)