Possible to call rule directly
This commit is contained in:
Noe Brucy
+251
-90
@@ -16,6 +16,9 @@ from pymses.sources.hop.file_formats import *
|
||||
from pymses.analysis import Camera, raytracing, slicing, splatting
|
||||
from pymses.filters import CellsToPoints
|
||||
from pymses.analysis import ScalarOperator, FractionOperator, MaxLevelOperator
|
||||
import subprocess
|
||||
|
||||
import module_extract as me
|
||||
|
||||
from mypool import MyPool as Pool
|
||||
from functools import partial
|
||||
@@ -54,18 +57,17 @@ class Rule:
|
||||
return self.process_fn(**kwargs)
|
||||
|
||||
def is_valid(self, arg):
|
||||
save = self.postproc.save
|
||||
valid = True
|
||||
for dep in self.dependencies:
|
||||
if dep in self.postproc.rules:
|
||||
rule_dep = self.postproc.rules[dep]
|
||||
if not arg is None:
|
||||
valid = (
|
||||
valid and rule_dep.group + "/" + dep + "_" + str(arg) in save
|
||||
)
|
||||
else:
|
||||
valid = valid and rule_dep.group + "/" + dep in save
|
||||
return valid and self.is_valid_add(arg)
|
||||
# save = self.postproc.save
|
||||
# valid = True
|
||||
# for dep in self.dependencies:
|
||||
# if dep in self.postproc.rules:
|
||||
# rule_dep = self.postproc.rules[dep]
|
||||
# if not arg is None:
|
||||
# valid = valid and rule_dep.group + '/' + dep + '_' + str(arg) in save
|
||||
# else:
|
||||
# valid = valid and rule_dep.group + '/' + dep in save
|
||||
# return valid and self.is_valid_add(arg)
|
||||
return self.is_valid_add(arg)
|
||||
|
||||
|
||||
class BaseProcessor:
|
||||
@@ -142,17 +144,22 @@ class BaseProcessor:
|
||||
def _solve_and_process_rule(self, name, rule, arg, overwrite=False, **kwargs):
|
||||
updated = self._solve_dependencies(name, rule, arg, overwrite, **kwargs)
|
||||
overwrite_rule = overwrite or updated
|
||||
self._process_rule(name, rule, arg, overwrite, **kwargs)
|
||||
self._process_rule(name, rule, arg, overwrite_rule, **kwargs)
|
||||
|
||||
def _solve_dependencies(self, name, rule, arg, overwrite=False, **kwargs):
|
||||
|
||||
self.done_before_dep = len(self.just_done)
|
||||
if type(rule.dependencies) == dict:
|
||||
dep_arg = rule.dependencies[dep]
|
||||
else:
|
||||
dep_arg = arg
|
||||
|
||||
# Solve dependencies
|
||||
for dep in rule.dependencies:
|
||||
try:
|
||||
dep_arg = rule.dependencies[dep]
|
||||
except:
|
||||
dep_arg = arg
|
||||
|
||||
if dep_arg == "__parent__":
|
||||
dep_arg = arg
|
||||
|
||||
if self.solve_self_dep and dep in self.rules:
|
||||
rule_dep = self.rules[dep]
|
||||
self._solve_and_process_rule(dep, rule_dep, dep_arg, self.overwrite_dep)
|
||||
@@ -189,6 +196,53 @@ class BaseProcessor:
|
||||
else:
|
||||
self._log("{} is not valid in this context".format(name_full), "ERROR")
|
||||
|
||||
def def_rules(self):
|
||||
for rule in self.rules:
|
||||
setattr(self, rule, partial(self.process, rule))
|
||||
|
||||
|
||||
class HDF5Container(BaseProcessor):
|
||||
|
||||
filename = ""
|
||||
save = None
|
||||
opened = False
|
||||
|
||||
def open(self):
|
||||
if not self.opened:
|
||||
self.save = tables.open_file(self.filename, mode="a")
|
||||
self.opened = True
|
||||
|
||||
def close(self):
|
||||
if self.opened:
|
||||
self.save.close()
|
||||
self.opened = False
|
||||
|
||||
def _needs_computation(self, overwrite, name_full):
|
||||
return overwrite or not (name_full in self.save)
|
||||
|
||||
def _process_rule(self, name, rule, arg, overwrite, **kwargs):
|
||||
self.open()
|
||||
try:
|
||||
super(HDF5Container, self)._process_rule(
|
||||
name, rule, arg, overwrite, **kwargs
|
||||
)
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
def get_value(self, node_name):
|
||||
self.open()
|
||||
try:
|
||||
node = self.save.get_node(node_name)
|
||||
if node._v_attrs.CLASS == "GROUP":
|
||||
value = {}
|
||||
for child_name in node._v_children:
|
||||
value[child_name] = self.get_value(node_name + "/" + child_name)
|
||||
else:
|
||||
value = node.read()
|
||||
finally:
|
||||
self.close()
|
||||
return value
|
||||
|
||||
def _save_data(self, name_full, data, description, unit):
|
||||
"""
|
||||
Save data in the HDF5 structure, overwrite if necessary
|
||||
@@ -243,53 +297,13 @@ class BaseProcessor:
|
||||
)
|
||||
self.save.get_node(name_full).attrs.unit = unit
|
||||
|
||||
@abstractmethod
|
||||
def def_rules(self):
|
||||
pass
|
||||
|
||||
|
||||
class HDF5Container(BaseProcessor):
|
||||
|
||||
filename = ""
|
||||
save = None
|
||||
opened = False
|
||||
|
||||
def open(self):
|
||||
if not self.opened:
|
||||
self.save = tables.open_file(self.filename, mode="a")
|
||||
self.opened = True
|
||||
|
||||
def close(self):
|
||||
if self.opened:
|
||||
self.save.close()
|
||||
self.opened = False
|
||||
|
||||
def _needs_computation(self, overwrite, name_full):
|
||||
return overwrite or not (name_full in self.save)
|
||||
|
||||
def _process_rule(self, name, rule, arg, overwrite, **kwargs):
|
||||
def set_value(self, node_name, data, description, unit):
|
||||
self.open()
|
||||
try:
|
||||
super(HDF5Container, self)._process_rule(
|
||||
name, rule, arg, overwrite, **kwargs
|
||||
)
|
||||
self._save_data(node_name, data, description, unit)
|
||||
finally:
|
||||
self.close()
|
||||
|
||||
def get_value(self, node_name):
|
||||
self.open()
|
||||
try:
|
||||
node = self.save.get_node(node_name)
|
||||
if node._v_attrs.CLASS == "GROUP":
|
||||
value = {}
|
||||
for child_name in node._v_children:
|
||||
value[child_name] = self.get_value(node_name + "/" + child_name)
|
||||
else:
|
||||
value = node.read()
|
||||
finally:
|
||||
self.close()
|
||||
return value
|
||||
|
||||
def get_attribute(self, node_name, attr_name):
|
||||
self.open()
|
||||
try:
|
||||
@@ -301,9 +315,12 @@ class HDF5Container(BaseProcessor):
|
||||
|
||||
|
||||
def _map_rule(rule, arg, overwrite, path, path_out, pp_params, run_num):
|
||||
pp = PostProcessor(
|
||||
path + "/" + run_num[0], run_num[1], path_out + "/" + run_num[0], pp_params
|
||||
)
|
||||
try:
|
||||
pp = PostProcessor(
|
||||
path + "/" + run_num[0], run_num[1], path_out + "/" + run_num[0], pp_params
|
||||
)
|
||||
except pymses.RamsesIOError as e:
|
||||
print(e)
|
||||
return pp.process(rule, arg, overwrite)
|
||||
|
||||
|
||||
@@ -395,9 +412,9 @@ class PostProcessor(HDF5Container):
|
||||
self._rt = raytracing.RayTracer(self._amr, self._ro.info, self._rho_op)
|
||||
|
||||
# Set the extend of the image
|
||||
self._radius = 0.5 / pp_params.out.zoom
|
||||
self._radius = 0.5 / pp_params.pymses.zoom
|
||||
self._lbox = self.info["boxlen"]
|
||||
center = pp_params.out.center
|
||||
center = pp_params.pymses.center
|
||||
im_extent = [
|
||||
(-self._radius + center[0]) * self._lbox,
|
||||
(self._radius + center[0]) * self._lbox,
|
||||
@@ -428,13 +445,13 @@ class PostProcessor(HDF5Container):
|
||||
ax_v = self._axes_v[ax_los]
|
||||
|
||||
self._cam[ax_los] = Camera(
|
||||
center=pp_params.out.center,
|
||||
center=pp_params.pymses.center,
|
||||
line_of_sight_axis=ax_los,
|
||||
region_size=[2.0 * self._radius, 2.0 * self._radius],
|
||||
distance=self._radius,
|
||||
far_cut_depth=self._radius,
|
||||
up_vector=ax_v,
|
||||
map_max_size=pp_params.out.map_size,
|
||||
map_max_size=pp_params.pymses.map_size,
|
||||
)
|
||||
|
||||
self.close()
|
||||
@@ -673,7 +690,7 @@ class PostProcessor(HDF5Container):
|
||||
Computes the radius from the center
|
||||
"""
|
||||
im_extent = self.save.root.maps._v_attrs.im_extent
|
||||
map_size = self.pp_params.out.map_size
|
||||
map_size = self.pp_params.pymses.map_size
|
||||
pos_star = self.pp_params.disk.pos_star
|
||||
|
||||
x = np.linspace(im_extent[0], im_extent[1], map_size)
|
||||
@@ -775,6 +792,31 @@ class PostProcessor(HDF5Container):
|
||||
pdf.attrs.var = np.var
|
||||
return True
|
||||
|
||||
def _clumps(self):
|
||||
name = self.path_out + "/" + self.tag + "_" + str(self.num).zfill(5)
|
||||
hop_save = name + "_hop" + "_prop_struct.save"
|
||||
|
||||
me.make_clump_hop(
|
||||
self.path,
|
||||
self.num,
|
||||
name + "_hop",
|
||||
self.pp_params.hop.rho_thres,
|
||||
self.pp_params.hop.lvl_thres,
|
||||
[0.5, 0.5, 0.5],
|
||||
1,
|
||||
path_out=path_out + "/",
|
||||
path_hop="./",
|
||||
force=True,
|
||||
gcomp=False,
|
||||
)
|
||||
hop_save = me.clump_properties(
|
||||
name + "_hop", path, num, path_out=path_out + "/", gcomp=False
|
||||
)
|
||||
f = open(path_out + "/" + hop_save)
|
||||
hop_data = pickle.load(f)
|
||||
f.close()
|
||||
return hop_data
|
||||
|
||||
def _sinks(self):
|
||||
csv_name = (
|
||||
self.path
|
||||
@@ -814,6 +856,56 @@ class PostProcessor(HDF5Container):
|
||||
|
||||
return sinks_dict
|
||||
|
||||
def _transform(self, name, transform_fn, group="/maps", **kwargs):
|
||||
src = self.save.get_node(group + "/" + name).read()
|
||||
return transform_fn(src, **kwargs)
|
||||
|
||||
def _gen_rule_transform(
|
||||
self,
|
||||
rule_src_name,
|
||||
transform_fn,
|
||||
transform_name,
|
||||
subarray_name=None,
|
||||
group=None,
|
||||
):
|
||||
|
||||
rule_src = self.rules[rule_src_name]
|
||||
|
||||
if subarray_name is None:
|
||||
src_name = rule_src_name
|
||||
group_src = rule_src.group
|
||||
unit = rule_src.unit
|
||||
description = rule_src.description
|
||||
else:
|
||||
src_name = subarray_name
|
||||
group_src = rule_src.group + "/" + rule_src_name
|
||||
unit = rule_src.unit[subarray_name]
|
||||
description = rule_src.description[subarray_name]
|
||||
|
||||
def fn(arg=None, **kwargs):
|
||||
if arg is None:
|
||||
return self._transform(
|
||||
src_name, transform_fn, group=group_src, **kwargs
|
||||
)
|
||||
else:
|
||||
return self._transform(
|
||||
src_name + "_" + str(arg), transform_fn, group=group_src, **kwargs
|
||||
)
|
||||
|
||||
if group is None:
|
||||
group = group_src
|
||||
|
||||
name = transform_name + "_" + rule_src_name
|
||||
|
||||
self.rules[name] = Rule(
|
||||
self,
|
||||
fn,
|
||||
group=group,
|
||||
unit=unit,
|
||||
description=description,
|
||||
dependencies=[rule_src_name],
|
||||
)
|
||||
|
||||
def def_rules(self):
|
||||
|
||||
self.rules = {
|
||||
@@ -904,6 +996,7 @@ class PostProcessor(HDF5Container):
|
||||
"Teff": cst.K,
|
||||
},
|
||||
),
|
||||
"clumps": Rule(self, self._clumps, group="/datasets"),
|
||||
# Helpers
|
||||
"radial_bins": Rule(self, self._radial_bins, "Radial bins", "/radial"),
|
||||
"rr": Rule(self, self._rr, "Coordinate map", "/maps"),
|
||||
@@ -980,6 +1073,10 @@ class PostProcessor(HDF5Container):
|
||||
dependencies=["pdf_" + name],
|
||||
)
|
||||
|
||||
self._gen_rule_transform("fluct_coldens", np.max, "max", group="/globals")
|
||||
|
||||
super(PostProcessor, self).def_rules()
|
||||
|
||||
|
||||
class Comparator(Aggregator, HDF5Container):
|
||||
"""
|
||||
@@ -1033,6 +1130,7 @@ class Comparator(Aggregator, HDF5Container):
|
||||
)
|
||||
|
||||
self.namelist = selector.namelist
|
||||
# Get info from one output. TODO Avoid using pymses for that
|
||||
self.info = self.pp_runs[self.runs[0]][self.nums[self.runs[0]][0]].info
|
||||
|
||||
# log info
|
||||
@@ -1066,19 +1164,22 @@ class Comparator(Aggregator, HDF5Container):
|
||||
super(Comparator, self)._save_data(name_full, data, description, unit)
|
||||
self.save.get_node(name_full)._v_attrs.nums = self.nums
|
||||
|
||||
def _time_series(self, getter):
|
||||
def _time_series(self, getter, arg=None):
|
||||
series = {}
|
||||
for run in self.runs:
|
||||
series[run] = np.zeros(len(self.nums[run]))
|
||||
for i, num in enumerate(self.nums[run]):
|
||||
series[run][i] = getter(run, num)
|
||||
series[run][i] = getter(run, num, arg=arg)
|
||||
return series
|
||||
|
||||
def _comp(self, getter):
|
||||
def _comp(self, getter, use_num=True):
|
||||
prop = np.zeros(len(self.runs))
|
||||
for i, run in enumerate(self.runs):
|
||||
num = self.nums[run][0]
|
||||
prop[i] = getter(run, num)
|
||||
if use_num:
|
||||
num = self.nums[run][0]
|
||||
prop[i] = getter(run, num)
|
||||
else:
|
||||
prop[i] = getter(run)
|
||||
return prop
|
||||
|
||||
def _time_avg(self, name, start=None, end=None, span=None, group="/series"):
|
||||
@@ -1108,21 +1209,22 @@ class Comparator(Aggregator, HDF5Container):
|
||||
std[i] = np.nanstd(serie[mask])
|
||||
return {"runs": self.runs, "mean": mean, "std": std, "median": median}
|
||||
|
||||
def get_attr(self, attr_name, run, num, node_name="/"):
|
||||
def get_attr(self, attr_name, run, num, node_name="/", arg=None):
|
||||
pp = self.pp_runs[run][num]
|
||||
if not arg is None:
|
||||
node_name = node_name + "_" + str(arg)
|
||||
return pp.get_attribute(node_name, attr_name)
|
||||
|
||||
def get_global(self, node_name, run, num, args=[None], unload_cells=False):
|
||||
def get_global(self, node_name, run, num, arg=None, unload_cells=False):
|
||||
if not arg is None:
|
||||
node_name = node_name + "_" + str(arg)
|
||||
pp = self.pp_runs[run][num]
|
||||
if unload_cells:
|
||||
pp.unload_cells()
|
||||
value = pp.get_value(node_name)
|
||||
return value
|
||||
|
||||
def get_nml(self, nml_key, run, num=0):
|
||||
"""
|
||||
num parameter is ignored
|
||||
"""
|
||||
def get_nml(self, nml_key, run):
|
||||
res = self.namelist[run]
|
||||
for key in nml_key.split("/"):
|
||||
res = res[key]
|
||||
@@ -1135,7 +1237,7 @@ class Comparator(Aggregator, HDF5Container):
|
||||
return slope
|
||||
|
||||
def _extract_sinks_from_log(self, series, log_filename, run):
|
||||
cmd_grep = "grep 'Number of sink' {} -A 2".format(log_filename)
|
||||
cmd_grep = "sed '/cpu.*/d' {} | grep 'Number of sink' -A 2".format(log_filename)
|
||||
content = os.popen(cmd_grep).readlines()
|
||||
for i in range(0, len(content), 4):
|
||||
series["nb_sink"][run].append(np.int(content[i].split("=")[1]))
|
||||
@@ -1154,11 +1256,18 @@ class Comparator(Aggregator, HDF5Container):
|
||||
return series
|
||||
|
||||
def _extract_rms_from_log(self, series, log_filename, run):
|
||||
cmd_grep = "grep 'turbulent rms' {} -B 1".format(log_filename)
|
||||
cmd_grep = "grep 'turbulent rms' {} -C 1".format(log_filename)
|
||||
content = os.popen(cmd_grep).readlines()
|
||||
for i in range(0, len(content), 3):
|
||||
for i in range(0, len(content), 4):
|
||||
series["time"][run].append(np.float(content[i].split("=")[2].split()[0]))
|
||||
series["turb_rms"][run].append(np.float(content[i + 1].split(":")[1]))
|
||||
try:
|
||||
turb_energy = np.float(content[i + 2].split(":")[1])
|
||||
threshold = self.pp_params.rules.turb_energy_threshold
|
||||
assert threshold < 0 or abs(turb_energy) < threshold
|
||||
series["turb_energy"][run].append(turb_energy)
|
||||
except (ValueError, IndexError, AssertionError):
|
||||
series["turb_energy"][run].append(np.nan)
|
||||
return series
|
||||
|
||||
def _from_log(self, keys, extractor):
|
||||
@@ -1223,6 +1332,33 @@ class Comparator(Aggregator, HDF5Container):
|
||||
ssfr[run][shift:] = sfr / surface
|
||||
return ssfr
|
||||
|
||||
def _gen_rule_time_global(
|
||||
self,
|
||||
glob_name,
|
||||
name=None,
|
||||
glob_group="/globals",
|
||||
subarray_name=None,
|
||||
unload_cells=True,
|
||||
unit=cst.none,
|
||||
description="",
|
||||
):
|
||||
|
||||
if name is None:
|
||||
name = "time_" + glob_name
|
||||
|
||||
self.rules[name] = Rule(
|
||||
self,
|
||||
partial(
|
||||
self._time_series,
|
||||
partial(
|
||||
self.get_global, glob_group + "/" + glob_name, unload_cells=True
|
||||
),
|
||||
),
|
||||
group="/series",
|
||||
unit=unit,
|
||||
dependencies={"time": None, glob_name: "__parent__"},
|
||||
)
|
||||
|
||||
def _gen_rule_avg(self, rule_src_name, subarray_name=None):
|
||||
|
||||
rule_src = self.rules[rule_src_name]
|
||||
@@ -1307,11 +1443,21 @@ class Comparator(Aggregator, HDF5Container):
|
||||
"rms_from_log": Rule(
|
||||
self,
|
||||
partial(
|
||||
self._from_log, ["time", "turb_rms"], self._extract_rms_from_log
|
||||
self._from_log,
|
||||
["time", "turb_rms", "turb_energy"],
|
||||
self._extract_rms_from_log,
|
||||
),
|
||||
group="/series",
|
||||
unit={"time": self.info["unit_time"], "turb_rms": cst.none},
|
||||
description={"time": "Time", "turb_rms": "Computed turbulent RMS"},
|
||||
unit={
|
||||
"time": self.info["unit_time"],
|
||||
"turb_rms": cst.none,
|
||||
"turb_energy": cst.none,
|
||||
},
|
||||
description={
|
||||
"time": "Time",
|
||||
"turb_rms": "Computed turbulent RMS",
|
||||
"turb_energy": "Injected turbulent energy",
|
||||
},
|
||||
),
|
||||
# Read from outputs
|
||||
"time": Rule(
|
||||
@@ -1323,31 +1469,46 @@ class Comparator(Aggregator, HDF5Container):
|
||||
unit=self.info["unit_time"],
|
||||
dependencies=["time_num"],
|
||||
),
|
||||
"time_sigma": Rule(
|
||||
"time_pdf_slope_coldens": Rule(
|
||||
self,
|
||||
partial(
|
||||
self._time_series,
|
||||
partial(self.get_global, "/globals/mwa_sigma", unload_cells=True),
|
||||
partial(
|
||||
self.get_attr,
|
||||
"slope",
|
||||
node_name="/hist/pdf_coldens_z",
|
||||
),
|
||||
),
|
||||
group="/series",
|
||||
unit=self.info["unit_velocity"],
|
||||
dependencies=["time", "mwa_sigma"],
|
||||
dependencies={"time": None, "fit_pdf_coldens": "z"},
|
||||
),
|
||||
# namelist
|
||||
"nml": Rule(
|
||||
self,
|
||||
lambda nml_key: self._comp(partial(self.get_nml, nml_key)),
|
||||
lambda nml_key: self._comp(
|
||||
partial(self.get_nml, nml_key), use_num=False
|
||||
),
|
||||
group="/comp",
|
||||
),
|
||||
}
|
||||
|
||||
for name in ["issfr", "time_sigma"]:
|
||||
self._gen_rule_time_global(
|
||||
"mwa_sigma", "time_sigma", unit=self.info["unit_velocity"]
|
||||
)
|
||||
self._gen_rule_time_global("max_fluct_coldens")
|
||||
|
||||
for name in ["issfr", "time_sigma", "time_pdf_slope_coldens"]:
|
||||
self._gen_rule_avg(name)
|
||||
|
||||
self._gen_rule_avg("sinks_from_log", "mass_sink")
|
||||
self._gen_rule_avg("sinks_from_log", "nb_sink")
|
||||
self._gen_rule_avg("sfr_from_log", "sfr")
|
||||
|
||||
self._gen_rule_avg("rms_from_log", "turb_rms")
|
||||
self._gen_rule_avg("rms_from_log", "turb_energy")
|
||||
|
||||
super(Comparator, self).def_rules()
|
||||
|
||||
|
||||
def get_time(path, num):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user