Huge refactoring
This commit is contained in:
+14
-14
@@ -2,7 +2,7 @@
|
||||
|
||||
import numpy as np
|
||||
from functools import partial
|
||||
import postprocessor
|
||||
import snapshotprocessor
|
||||
|
||||
try:
|
||||
from mpi4py.futures import MPIPoolExecutor
|
||||
@@ -14,44 +14,44 @@ except ModuleNotFoundError:
|
||||
mpi = False
|
||||
|
||||
|
||||
def _map_aux(fun, path, path_out, pp_params, run_num, **kwargs):
|
||||
def _map_aux(fun, path, path_out, params, run_num, **kwargs):
|
||||
try:
|
||||
pp = postprocessor.PostProcessor(
|
||||
path + "/" + run_num[0], run_num[1], path_out + "/" + run_num[0], pp_params
|
||||
snap = snapshotprocessor.SnapshotProcessor(
|
||||
path + "/" + run_num[0], run_num[1], path_out + "/" + run_num[0], params
|
||||
)
|
||||
except Exception as e:
|
||||
print(e)
|
||||
raise
|
||||
return fun(pp, **kwargs)
|
||||
return fun(snap, **kwargs)
|
||||
|
||||
|
||||
def _map_rule(pp, rule, arg, overwrite, overwrite_dep):
|
||||
return pp.process(rule, arg, overwrite, overwrite_dep)
|
||||
def _map_rule(snap, rule, arg, overwrite, overwrite_dep):
|
||||
return snap.process(rule, arg, overwrite, overwrite_dep)
|
||||
|
||||
|
||||
class Aggregator:
|
||||
def get_pp_list(self, select=None):
|
||||
def get_snap_list(self, select=None):
|
||||
|
||||
if select is not None:
|
||||
runs, nums = self.selector.select(**select)
|
||||
else:
|
||||
runs = self.runs
|
||||
nums = self.nums
|
||||
return [self.pp[run][num] for run in runs for num in nums[run]]
|
||||
return [self.snaps[run][num] for run in runs for num in nums[run]]
|
||||
|
||||
def map(self, func, select=None, num_process=None, **kwargs):
|
||||
|
||||
pp_list = self.get_pp_list(select)
|
||||
snaps = self.get_snap_list(select)
|
||||
|
||||
if num_process is None:
|
||||
num_process = self.pp_params.process.num_process
|
||||
num_process = self.params.process.num_process
|
||||
|
||||
if num_process == 1:
|
||||
result = [func(pp, **kwargs) for pp in pp_list]
|
||||
result = [func(snap, **kwargs) for snap in snaps]
|
||||
else:
|
||||
run_num = [(pp.run, pp.num) for pp in pp_list]
|
||||
run_num = [(snap.run, snap.num) for snap in snaps]
|
||||
map_fn = partial(
|
||||
_map_aux, func, self.path, self.path_out, self.pp_params, **kwargs
|
||||
_map_aux, func, self.path, self.path_out, self.params, **kwargs
|
||||
)
|
||||
if mpi:
|
||||
executor = MPIPoolExecutor(max_workers=num_process)
|
||||
|
||||
Reference in New Issue
Block a user