noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve logging

Browse Source
This commit is contained in:
Noe Brucy
2022-09-02 16:16:17 +02:00
parent f1eb57a2b0
commit 6e0f5b4297
3 changed files with 40 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
baseprocessor.py
+2 -13
View File
@@ -76,7 +76,7 @@ class BaseProcessor:
# Initialize logger
self.logger = logging.getLogger(self.log_id)
self.logger.propagate = False
logging_format = '%(levelname)s | %(asctime)s | %(name)s | %(message)s' # %(funcName)s(%(lineno)d)
logging_format = '%(levelname)s | %(asctime)s | %(name)s.%(funcName)s:%(lineno)d | %(message)s'
formatter = logging.Formatter(logging_format, datefmt = '%H:%M:%S')
if not self.logger.hasHandlers():
@@ -96,17 +96,6 @@ class BaseProcessor:
for handler in self.logger.handlers:
handler.setFormatter(formatter)
def _log(self, string, status=""):
# self.logger.warning("Use of _log is deprecated, use logger instead")
if status == "WARNING":
self.logger.warning(string)
elif status == "ERROR":
self.logger.error(string)
elif status == "SUCCESS":
self.logger.info(string)
else:
self.logger.debug(f"{string}")
def process(
self,
to_process,
@@ -232,7 +221,7 @@ class BaseProcessor:
return data
else:
self.logger.info(
"Data for {} is already computed, skipping...".format(name_full)
"Data for {} is already computed.".format(name_full)
)
def def_rules(self):
snapshotprocessor.py
+10 -7
View File
@@ -313,7 +313,7 @@ class SnapshotProcessor(HDF5Container):
self.path = path
self.run = os.path.basename(path)
self.num = num
self.log_id = "{}, {}".format(self.run, self.num)
self.log_id = "snap({}, {})".format(self.run, self.num)
super(SnapshotProcessor, self).__init__(path, path_out, params, tag)
@@ -400,8 +400,9 @@ class SnapshotProcessor(HDF5Container):
try:
self.init_pymses()
except FileNotFoundError:
self._log("Pymses not initialized", "WARNING")
except:
self.logger.error("Pymses not initialized", exc_info=1)
self.def_rules()
@@ -485,16 +486,18 @@ class SnapshotProcessor(HDF5Container):
map_max_size=self.params.pymses.map_size,
)
# Initialize HDF5 group
try:
self.open()
if "/maps" not in self.save:
self.save.create_group("/", "maps", "2D maps")
if "/hist" not in self.save:
self.save.create_group("/", "hist", "Histograms")
if "/datasets" not in self.save:
self.save.create_group("/", "datasets", "Complex datasets")
self.save.root.maps._v_attrs.center = center
self.save.root.maps._v_attrs.radius = self._radius
self.save.root.maps._v_attrs.im_extent = im_extent
except:
self.logger.error("Error in HDF5", exc_info=1)
raise
finally:
self.close()
def load_data(self, points_src, filename, save, keys=None):
studyprocessor.py
+20 -17
View File
@@ -40,7 +40,7 @@ class StudyProcessor(Aggregator, HDF5Container):
# log id
self.log_id = "study {}".format(tag)
self.log_id = "study({})".format(tag)
@@ -156,7 +156,7 @@ class StudyProcessor(Aggregator, HDF5Container):
prop[run] = getter(run, num)
else:
prop[run] = getter(run)
return np.array(list(prop.keys()))
return np.array(list(prop.values()))
def time_avg(self, name, start=None, end=None, span=None, unit_time=U.Myr, group="/series", select=None):
"""Do the time average and quantiles of a time series
@@ -295,6 +295,7 @@ class StudyProcessor(Aggregator, HDF5Container):
def _extract_sinks_from_log(self, series, log_filename, run):
cmd_grep = f"grep 'Number of sink' {log_filename} -A 2"
content = os.popen(cmd_grep).readlines()
block_err = [] # Block that will ill parsed
for i in range(0, len(content), 4):
try:
nb_sink = np.int(content[i].split("=")[1])
@@ -309,11 +310,11 @@ class StudyProcessor(Aggregator, HDF5Container):
series["time"][run].append(time)
except (ValueError, IndexError):
self._log(
"Error encountered in parsing {} (grepped block {})".format(
log_filename, i
),
"WARNING",
block_err.append(i)
if len(block_err) > 0:
self.logger.warning(
f"Errors encountered in parsing {log_filename} (grepped blocks {block_err})"
)
return series
@@ -323,6 +324,7 @@ class StudyProcessor(Aggregator, HDF5Container):
nb_stellar = list(map(lambda s: int(s.split()[1]), content))
line_numbers = list(map(lambda s: int(s.split(":")[0]), content))
current_line = 0
block_err = [] # Block that will ill parsed
logfile = open(log_filename)
for i in range(0, len(line_numbers)):
try:
@@ -346,11 +348,11 @@ class StudyProcessor(Aggregator, HDF5Container):
stellar_objects["id"][run].append(id)
except (ValueError, IndexError):
self._log(
"[stellar] Error encountered in parsing {} (grepped block {})".format(
log_filename, i
),
"WARNING",
block_err.append(i)
if len(block_err) > 0:
self.logger.warning(
f"Errors encountered in parsing {log_filename} (grepped blocks {block_err})"
)
logfile.close()
return stellar_objects
@@ -368,6 +370,7 @@ class StudyProcessor(Aggregator, HDF5Container):
def _extract_fine_step_from_log(self, series, log_filename, run):
cmd_grep = "grep 'Fine step' {} ".format(log_filename)
content = os.popen(cmd_grep).readlines()
block_err = [] # Block that will ill parsed
for i in range(0, len(content)):
try:
data = content[i].replace("=", " ").split()
@@ -384,11 +387,11 @@ class StudyProcessor(Aggregator, HDF5Container):
series["mem_cells"][run].append(mempc1)
series["mem_parts"][run].append(mempc2)
except (ValueError, IndexError):
self._log(
"[fine step] Error encountered in parsing {} (grepped block {})".format(
log_filename, i
),
"WARNING",
block_err.append(i)
if len(block_err) > 0:
self.logger.warning(
f"Error encountered in parsing {log_filename} (grepped blocks {block_err})"
)
return series