noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

improve io checks

Browse Source
This commit is contained in:
Noe Brucy
2023-04-05 16:50:43 +02:00
parent 301d08dce7
commit bd520d7189
5 changed files with 105 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
galturb/sectors_extraction_ramses.py
+1 -1
View File
@@ -103,6 +103,6 @@ if __name__ == "__main__":
from snapshotprocessor import SnapshotProcessor from snapshotprocessor import SnapshotProcessor
pp = SnapshotProcessor( pp = SnapshotProcessor(
"/home/nbrucy/simus/F20H_alfven_frig", num=1, params="params_gal.yml" "/home/nbrucy/ecogal/galturb/F20H_7_1pc_frig", num=80, params="params_gal.yml"
) )
data = load_fields(pp) data = load_fields(pp)
params.yml
+2 -1
View File
@@ -47,7 +47,8 @@ pymses: # Parameters for Pymses reader
# Source settings # Source settings
variables : ["rho","vel","Br","Bl","P", "g", "phi"] # Read these grid variables variables : ["rho","vel","Br","Bl","P", "g", "phi"] # Read these grid variables
part_variables : ["vel","mass","id","level","epoch"] # Read these particles variables part_variables : ["vel","mass","id","level","epoch"] # Read these particles variables
check_variables : True # Whether to check if the variables are indeed in the data
order : '<' # Bit order order : '<' # Bit order
snapshotprocessor.py
+39 -28
View File
@@ -287,6 +287,9 @@ class SnapshotProcessor(HDF5Container):
"phi": "unit_gravpot", "phi": "unit_gravpot",
"mass": "unit_mass", "mass": "unit_mass",
"epoch": "unit_time", "epoch": "unit_time",
"size": "unit_length",
"dx": "unit_length",
"pos": "unit_length",
"id": U.none, "id": U.none,
"level": U.none, "level": U.none,
} }
@@ -422,20 +425,7 @@ class SnapshotProcessor(HDF5Container):
self.def_rules() self.def_rules()
def init_pymses(self): def check_variables(self, path):
# If ratarmount was used
if os.path.exists(f"{self.path}/output_{self.num:05}/output_{self.num:05}"):
path = f"{self.path}/output_{self.num:05}"
else:
path = self.path
self._ro = pymses.RamsesOutput(
path,
self.num,
order=self.params.pymses.order,
verbose=self.params.pymses.verbose,
check_endianness=False,
)
# Check if variables are in output # Check if variables are in output
name_conv = { name_conv = {
"rho": "density", "rho": "density",
@@ -479,16 +469,16 @@ class SnapshotProcessor(HDF5Container):
def is_available(available_vars, pymsesrc, var): def is_available(available_vars, pymsesrc, var):
if var in ["g", "phi"]: if var in ["g", "phi"]:
if not has_grav: if not has_grav:
self.logger.warning(f"Variable {var} not in output") self.logger.debug(f"Variable {var} not in output")
if var not in pymsesrc: if var not in pymsesrc:
self.logger.warning(f"Variable {var} not in pymsesrc") self.logger.debug(f"Variable {var} not in pymsesrc")
return has_grav and var in pymsesrc return has_grav and var in pymsesrc
else: else:
if var in name_conv: if var in name_conv:
if name_conv[var] not in available_vars: if name_conv[var] not in available_vars:
self.logger.warning(f"Variable {var} not in output") self.logger.debug(f"Variable {var} not in output")
if var not in pymsesrc: if var not in pymsesrc:
self.logger.warning(f"Variable {var} not in pymsesrc") self.logger.debug(f"Variable {var} not in pymsesrc")
return name_conv[var] in available_vars and var in pymsesrc return name_conv[var] in available_vars and var in pymsesrc
else: else:
self.logger.warning(f"Variable {var} is unknown") self.logger.warning(f"Variable {var} is unknown")
@@ -528,6 +518,24 @@ class SnapshotProcessor(HDF5Container):
pymses.rcConfig.Ramses.amr_fields.remove_field("g") pymses.rcConfig.Ramses.amr_fields.remove_field("g")
pymses.rcConfig.Ramses.amr_fields.remove_field("phi") pymses.rcConfig.Ramses.amr_fields.remove_field("phi")
def init_pymses(self):
# If ratarmount was used
if os.path.exists(f"{self.path}/output_{self.num:05}/output_{self.num:05}"):
path = f"{self.path}/output_{self.num:05}"
else:
path = self.path
self._ro = pymses.RamsesOutput(
path,
self.num,
order=self.params.pymses.order,
verbose=self.params.pymses.verbose,
check_endianness=False,
)
if self.params.pymses.check_variables:
self.check_variables(path)
self._amr = self._ro.amr_source(self.params.pymses.variables) self._amr = self._ro.amr_source(self.params.pymses.variables)
self._part = self._ro.particle_source(self.params.pymses.part_variables) self._part = self._ro.particle_source(self.params.pymses.part_variables)
@@ -676,12 +684,12 @@ class SnapshotProcessor(HDF5Container):
if len(data[key] > 0): if len(data[key] > 0):
if f"/{group}/{key}" in hdf5 and overwrite: if f"/{group}/{key}" in hdf5 and overwrite:
hdf5.remove_node(f"/{group}/{key}") hdf5.remove_node(f"/{group}/{key}")
hdf5.create_array( hdf5.create_array(
f"/{group}", key, data[key], "", createparents=True f"/{group}", key, data[key], "", createparents=True
) )
unit = self._get_units(self.unit_key[key]) unit = self._get_units(self.unit_key[key])
hdf5.get_node("/{group}/{key}").unit = unit hdf5.get_node(f"/{group}/{key}").unit = unit
nb_written += 1 nb_written += 1
else: else:
self.logger.warning("Empty key") self.logger.warning("Empty key")
if "namelist" not in hdf5.root._v_attrs: if "namelist" not in hdf5.root._v_attrs:
@@ -739,12 +747,12 @@ class SnapshotProcessor(HDF5Container):
self.cells = self.load_data( self.cells = self.load_data(
cells_src, cells_src,
filename, filename,
self.params.process.save_cells and save, True,
keys=keys, keys=keys,
group="cells", group="cells",
) )
self.cells_loaded = True self.cells_loaded = True
self.logger.info("Cells loaded") self.logger.info("Cells loaded")
def unload_cells(self): def unload_cells(self):
""" """
@@ -754,7 +762,7 @@ class SnapshotProcessor(HDF5Container):
if self.cells_loaded: if self.cells_loaded:
del self.cells del self.cells
self.cells_loaded = False self.cells_loaded = False
self.logger.info("Cells unloaded") self.logger.info("Cells unloaded")
def load_destructured(self, save=True): def load_destructured(self, save=True):
self.load_cells(save=save) self.load_cells(save=save)
@@ -1032,6 +1040,7 @@ class SnapshotProcessor(HDF5Container):
unit=None, unit=None,
logbins=False, logbins=False,
weight_func=vol_func, weight_func=vol_func,
**kwargs,
): ):
self.load_cells() self.load_cells()
data = getter(self.cells) data = getter(self.cells)
@@ -1043,7 +1052,9 @@ class SnapshotProcessor(HDF5Container):
if self.params.process.unload_cells: if self.params.process.unload_cells:
self.unload_cells() self.unload_cells()
values, edges = np.histogram(data, bins, weights=weights, density=True) values, edges = np.histogram(
data, bins, weights=weights, density=True, **kwargs
)
centers = 0.5 * (edges[1:] + edges[:-1]) centers = 0.5 * (edges[1:] + edges[:-1])
return (np.stack([values, centers]), {"logbins": logbins}) return (np.stack([values, centers]), {"logbins": logbins})
studyprocessor.py
+43 -33
View File
@@ -132,7 +132,10 @@ class StudyProcessor(Aggregator, HDF5Container):
def _save_data(self, name_full, data, description, unit): def _save_data(self, name_full, data, description, unit):
super(StudyProcessor, self)._save_data(name_full, data, description, unit) super(StudyProcessor, self)._save_data(name_full, data, description, unit)
self.save.get_node(name_full)._v_attrs.nums = self.nums if name_full in self.save:
self.save.get_node(name_full)._v_attrs.nums = self.nums
else:
self.logger.warning(f"{name_full} was not written")
def time_series(self, getter, arg=None): def time_series(self, getter, arg=None):
series = {} series = {}
@@ -380,7 +383,7 @@ class StudyProcessor(Aggregator, HDF5Container):
def _extract_fine_step_from_log(self, series, log_filename, run): def _extract_fine_step_from_log(self, series, log_filename, run):
cmd_grep = "grep 'Fine step' {} ".format(log_filename) cmd_grep = "grep 'Fine step' {} ".format(log_filename)
content = os.popen(cmd_grep).readlines() content = os.popen(cmd_grep).readlines()
block_err = [] # Block that will ill parsed block_err = [] # Blocks that are ill parsed
for i in range(0, len(content)): for i in range(0, len(content)):
try: try:
data = content[i].replace("=", " ").split() data = content[i].replace("=", " ").split()
@@ -412,38 +415,48 @@ class StudyProcessor(Aggregator, HDF5Container):
log_filename, nlines - 1 log_filename, nlines - 1
) )
content = os.popen(cmd_grep).readlines() content = os.popen(cmd_grep).readlines()
block_err = [] # Blocks that are ill parsed
for j in range(0, len(content), 2 * (nlines + 1)): for j in range(0, len(content), 2 * (nlines + 1)):
i = j + nlines + 1 # Index for the "Main step" grep i = j + nlines + 1 # Index for the "Main step" grep
if i + nlines - 1 < len(content): if i + nlines - 1 < len(content):
series["time"][run].append( try:
np.float(content[i + nlines - 1].split("=")[2].split()[0]) series["time"][run].append(
) np.float(content[i + nlines - 1].split("=")[2].split()[0])
series["step"][run].append(np.int(content[i].split("=")[1].split()[0])) )
series["mcons"][run].append( series["step"][run].append(
np.float(content[i].split("=")[2].split()[0]) np.int(content[i].split("=")[1].split()[0])
) )
series["econs"][run].append( series["mcons"][run].append(
np.float(content[i].split("=")[3].split()[0]) np.float(content[i].split("=")[2].split()[0])
) )
series["epot"][run].append( series["econs"][run].append(
np.float(content[i].split("=")[4].split()[0]) np.float(content[i].split("=")[3].split()[0])
) )
series["ekin"][run].append( series["epot"][run].append(
np.float(content[i].split("=")[5].split()[0]) np.float(content[i].split("=")[4].split()[0])
) )
if rism: series["ekin"][run].append(
eint = np.float(content[i].split("=")[6].split()[0]) np.float(content[i].split("=")[5].split()[0])
emag = np.float(content[i + 1].split("=")[1].split()[0]) )
else: if rism:
eint = 0.0 eint = np.float(content[i].split("=")[6].split()[0])
emag = 0.0 emag = np.float(content[i + 1].split("=")[1].split()[0])
series["eint"][run].append(eint) else:
series["emag"][run].append(emag) eint = 0.0
series["elapsed"][run].append( emag = 0.0
np.float(content[j].split(":")[1].split()[0]) series["eint"][run].append(eint)
) series["emag"][run].append(emag)
series["memory"][run].append(content[j + 1].split(":")[1]) series["elapsed"][run].append(
np.float(content[j].split(":")[1].split()[0])
)
series["memory"][run].append(content[j + 1].split(":")[1])
except (ValueError, IndexError):
block_err.append(i)
if len(block_err) > 0:
self.logger.warning(
f"Error encountered in parsing {log_filename} (grepped blocks {block_err})"
)
return series return series
def _extract_rms_from_log(self, series, log_filename, run): def _extract_rms_from_log(self, series, log_filename, run):
@@ -656,10 +669,7 @@ class StudyProcessor(Aggregator, HDF5Container):
glob_name, glob_name,
name=None, name=None,
glob_group="/globals", glob_group="/globals",
subarray_name=None,
unload_cells=True,
unit=U.none, unit=U.none,
description="",
): ):
if name is None: if name is None:
utils/runselector.py
+20 -20
View File
@@ -352,26 +352,6 @@ class RunSelector:
def try_load_info(num): def try_load_info(num):
return True return True
# -- Time getter according to unit_time
if unit_time is None:
def get_time(num):
return self.info[run][num]["time"]
elif isinstance(unit_time, str):
factor = self.get_nml_value(unit_time, run)
def get_time(num):
time_code = self.info[run][num]["time"]
return time_code / factor
else:
def get_time(num):
time_code = self.info[run][num]["time"]
return time_code * self.info[run][num]["unit_time"].express(unit_time)
# -- A function to search a given time using dichotomy # -- A function to search a given time using dichotomy
def search(nums, time, position="closest"): def search(nums, time, position="closest"):
@@ -460,6 +440,26 @@ class RunSelector:
path = f"{self.path_in}/{run}/output_{nums[0]:05}/namelist.txt" path = f"{self.path_in}/{run}/output_{nums[0]:05}/namelist.txt"
self.namelist[run] = self.load_namelist(run, path=path) self.namelist[run] = self.load_namelist(run, path=path)
# -- Time getter according to unit_time
if unit_time is None:
def get_time(num):
return self.info[run][num]["time"]
elif isinstance(unit_time, str):
factor = self.get_nml_value(unit_time, run)
def get_time(num):
time_code = self.info[run][num]["time"]
return time_code / factor
else:
def get_time(num):
time_code = self.info[run][num]["time"]
return time_code * self.info[run][num]["unit_time"].express(unit_time)
# -- Select according to time -- # -- Select according to time --
if time_min is not None and len(nums) > 0: if time_min is not None and len(nums) > 0:
imin = search(nums, time_min, "right") imin = search(nums, time_min, "right")