noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

improve io checks

Browse Source
This commit is contained in:
Noe Brucy
2023-04-05 16:50:43 +02:00
parent 301d08dce7
commit bd520d7189
5 changed files with 105 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
studyprocessor.py
+43 -33
View File
@@ -132,7 +132,10 @@ class StudyProcessor(Aggregator, HDF5Container):
def _save_data(self, name_full, data, description, unit):
super(StudyProcessor, self)._save_data(name_full, data, description, unit)
self.save.get_node(name_full)._v_attrs.nums = self.nums
if name_full in self.save:
self.save.get_node(name_full)._v_attrs.nums = self.nums
else:
self.logger.warning(f"{name_full} was not written")
def time_series(self, getter, arg=None):
series = {}
@@ -380,7 +383,7 @@ class StudyProcessor(Aggregator, HDF5Container):
def _extract_fine_step_from_log(self, series, log_filename, run):
cmd_grep = "grep 'Fine step' {} ".format(log_filename)
content = os.popen(cmd_grep).readlines()
block_err = [] # Block that will ill parsed
block_err = [] # Blocks that are ill parsed
for i in range(0, len(content)):
try:
data = content[i].replace("=", " ").split()
@@ -412,38 +415,48 @@ class StudyProcessor(Aggregator, HDF5Container):
log_filename, nlines - 1
)
content = os.popen(cmd_grep).readlines()
block_err = [] # Blocks that are ill parsed
for j in range(0, len(content), 2 * (nlines + 1)):
i = j + nlines + 1 # Index for the "Main step" grep
if i + nlines - 1 < len(content):
series["time"][run].append(
np.float(content[i + nlines - 1].split("=")[2].split()[0])
)
series["step"][run].append(np.int(content[i].split("=")[1].split()[0]))
series["mcons"][run].append(
np.float(content[i].split("=")[2].split()[0])
)
series["econs"][run].append(
np.float(content[i].split("=")[3].split()[0])
)
series["epot"][run].append(
np.float(content[i].split("=")[4].split()[0])
)
series["ekin"][run].append(
np.float(content[i].split("=")[5].split()[0])
)
if rism:
eint = np.float(content[i].split("=")[6].split()[0])
emag = np.float(content[i + 1].split("=")[1].split()[0])
else:
eint = 0.0
emag = 0.0
series["eint"][run].append(eint)
series["emag"][run].append(emag)
series["elapsed"][run].append(
np.float(content[j].split(":")[1].split()[0])
)
series["memory"][run].append(content[j + 1].split(":")[1])
try:
series["time"][run].append(
np.float(content[i + nlines - 1].split("=")[2].split()[0])
)
series["step"][run].append(
np.int(content[i].split("=")[1].split()[0])
)
series["mcons"][run].append(
np.float(content[i].split("=")[2].split()[0])
)
series["econs"][run].append(
np.float(content[i].split("=")[3].split()[0])
)
series["epot"][run].append(
np.float(content[i].split("=")[4].split()[0])
)
series["ekin"][run].append(
np.float(content[i].split("=")[5].split()[0])
)
if rism:
eint = np.float(content[i].split("=")[6].split()[0])
emag = np.float(content[i + 1].split("=")[1].split()[0])
else:
eint = 0.0
emag = 0.0
series["eint"][run].append(eint)
series["emag"][run].append(emag)
series["elapsed"][run].append(
np.float(content[j].split(":")[1].split()[0])
)
series["memory"][run].append(content[j + 1].split(":")[1])
except (ValueError, IndexError):
block_err.append(i)
if len(block_err) > 0:
self.logger.warning(
f"Error encountered in parsing {log_filename} (grepped blocks {block_err})"
)
return series
def _extract_rms_from_log(self, series, log_filename, run):
@@ -656,10 +669,7 @@ class StudyProcessor(Aggregator, HDF5Container):
glob_name,
name=None,
glob_group="/globals",
subarray_name=None,
unload_cells=True,
unit=U.none,
description="",
):
if name is None: