noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve logging

Browse Source
This commit is contained in:
Noe Brucy
2022-09-02 16:16:17 +02:00
parent f1eb57a2b0
commit 6e0f5b4297
3 changed files with 40 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
baseprocessor.py
+2 -13
View File
@@ -76,7 +76,7 @@ class BaseProcessor:
# Initialize logger # Initialize logger
self.logger = logging.getLogger(self.log_id) self.logger = logging.getLogger(self.log_id)
self.logger.propagate = False self.logger.propagate = False
logging_format = '%(levelname)s | %(asctime)s | %(name)s | %(message)s' # %(funcName)s(%(lineno)d) logging_format = '%(levelname)s | %(asctime)s | %(name)s.%(funcName)s:%(lineno)d | %(message)s'
formatter = logging.Formatter(logging_format, datefmt = '%H:%M:%S') formatter = logging.Formatter(logging_format, datefmt = '%H:%M:%S')
if not self.logger.hasHandlers(): if not self.logger.hasHandlers():
@@ -96,17 +96,6 @@ class BaseProcessor:
for handler in self.logger.handlers: for handler in self.logger.handlers:
handler.setFormatter(formatter) handler.setFormatter(formatter)
def _log(self, string, status=""):
# self.logger.warning("Use of _log is deprecated, use logger instead")
if status == "WARNING":
self.logger.warning(string)
elif status == "ERROR":
self.logger.error(string)
elif status == "SUCCESS":
self.logger.info(string)
else:
self.logger.debug(f"{string}")
def process( def process(
self, self,
to_process, to_process,
@@ -232,7 +221,7 @@ class BaseProcessor:
return data return data
else: else:
self.logger.info( self.logger.info(
"Data for {} is already computed, skipping...".format(name_full) "Data for {} is already computed.".format(name_full)
) )
def def_rules(self): def def_rules(self):
snapshotprocessor.py
+18 -15
View File
@@ -313,7 +313,7 @@ class SnapshotProcessor(HDF5Container):
self.path = path self.path = path
self.run = os.path.basename(path) self.run = os.path.basename(path)
self.num = num self.num = num
self.log_id = "{}, {}".format(self.run, self.num) self.log_id = "snap({}, {})".format(self.run, self.num)
super(SnapshotProcessor, self).__init__(path, path_out, params, tag) super(SnapshotProcessor, self).__init__(path, path_out, params, tag)
@@ -400,8 +400,9 @@ class SnapshotProcessor(HDF5Container):
try: try:
self.init_pymses() self.init_pymses()
except FileNotFoundError: except:
self._log("Pymses not initialized", "WARNING") self.logger.error("Pymses not initialized", exc_info=1)
self.def_rules() self.def_rules()
@@ -483,19 +484,21 @@ class SnapshotProcessor(HDF5Container):
far_cut_depth=distance, far_cut_depth=distance,
up_vector=ax_v, up_vector=ax_v,
map_max_size=self.params.pymses.map_size, map_max_size=self.params.pymses.map_size,
) )
self.open() # Initialize HDF5 group
if "/maps" not in self.save: try:
self.save.create_group("/", "maps", "2D maps") self.open()
if "/hist" not in self.save: if "/maps" not in self.save:
self.save.create_group("/", "hist", "Histograms") self.save.create_group("/", "maps", "2D maps")
if "/datasets" not in self.save: self.save.root.maps._v_attrs.center = center
self.save.create_group("/", "datasets", "Complex datasets") self.save.root.maps._v_attrs.radius = self._radius
self.save.root.maps._v_attrs.center = center self.save.root.maps._v_attrs.im_extent = im_extent
self.save.root.maps._v_attrs.radius = self._radius except:
self.save.root.maps._v_attrs.im_extent = im_extent self.logger.error("Error in HDF5", exc_info=1)
self.close() raise
finally:
self.close()
def load_data(self, points_src, filename, save, keys=None): def load_data(self, points_src, filename, save, keys=None):
""" """
studyprocessor.py
+20 -17
View File
@@ -40,7 +40,7 @@ class StudyProcessor(Aggregator, HDF5Container):
# log id # log id
self.log_id = "study {}".format(tag) self.log_id = "study({})".format(tag)
@@ -156,7 +156,7 @@ class StudyProcessor(Aggregator, HDF5Container):
prop[run] = getter(run, num) prop[run] = getter(run, num)
else: else:
prop[run] = getter(run) prop[run] = getter(run)
return np.array(list(prop.keys())) return np.array(list(prop.values()))
def time_avg(self, name, start=None, end=None, span=None, unit_time=U.Myr, group="/series", select=None): def time_avg(self, name, start=None, end=None, span=None, unit_time=U.Myr, group="/series", select=None):
"""Do the time average and quantiles of a time series """Do the time average and quantiles of a time series
@@ -295,6 +295,7 @@ class StudyProcessor(Aggregator, HDF5Container):
def _extract_sinks_from_log(self, series, log_filename, run): def _extract_sinks_from_log(self, series, log_filename, run):
cmd_grep = f"grep 'Number of sink' {log_filename} -A 2" cmd_grep = f"grep 'Number of sink' {log_filename} -A 2"
content = os.popen(cmd_grep).readlines() content = os.popen(cmd_grep).readlines()
block_err = [] # Block that will ill parsed
for i in range(0, len(content), 4): for i in range(0, len(content), 4):
try: try:
nb_sink = np.int(content[i].split("=")[1]) nb_sink = np.int(content[i].split("=")[1])
@@ -309,11 +310,11 @@ class StudyProcessor(Aggregator, HDF5Container):
series["time"][run].append(time) series["time"][run].append(time)
except (ValueError, IndexError): except (ValueError, IndexError):
self._log( block_err.append(i)
"Error encountered in parsing {} (grepped block {})".format(
log_filename, i if len(block_err) > 0:
), self.logger.warning(
"WARNING", f"Errors encountered in parsing {log_filename} (grepped blocks {block_err})"
) )
return series return series
@@ -323,6 +324,7 @@ class StudyProcessor(Aggregator, HDF5Container):
nb_stellar = list(map(lambda s: int(s.split()[1]), content)) nb_stellar = list(map(lambda s: int(s.split()[1]), content))
line_numbers = list(map(lambda s: int(s.split(":")[0]), content)) line_numbers = list(map(lambda s: int(s.split(":")[0]), content))
current_line = 0 current_line = 0
block_err = [] # Block that will ill parsed
logfile = open(log_filename) logfile = open(log_filename)
for i in range(0, len(line_numbers)): for i in range(0, len(line_numbers)):
try: try:
@@ -346,11 +348,11 @@ class StudyProcessor(Aggregator, HDF5Container):
stellar_objects["id"][run].append(id) stellar_objects["id"][run].append(id)
except (ValueError, IndexError): except (ValueError, IndexError):
self._log( block_err.append(i)
"[stellar] Error encountered in parsing {} (grepped block {})".format(
log_filename, i if len(block_err) > 0:
), self.logger.warning(
"WARNING", f"Errors encountered in parsing {log_filename} (grepped blocks {block_err})"
) )
logfile.close() logfile.close()
return stellar_objects return stellar_objects
@@ -368,6 +370,7 @@ class StudyProcessor(Aggregator, HDF5Container):
def _extract_fine_step_from_log(self, series, log_filename, run): def _extract_fine_step_from_log(self, series, log_filename, run):
cmd_grep = "grep 'Fine step' {} ".format(log_filename) cmd_grep = "grep 'Fine step' {} ".format(log_filename)
content = os.popen(cmd_grep).readlines() content = os.popen(cmd_grep).readlines()
block_err = [] # Block that will ill parsed
for i in range(0, len(content)): for i in range(0, len(content)):
try: try:
data = content[i].replace("=", " ").split() data = content[i].replace("=", " ").split()
@@ -384,11 +387,11 @@ class StudyProcessor(Aggregator, HDF5Container):
series["mem_cells"][run].append(mempc1) series["mem_cells"][run].append(mempc1)
series["mem_parts"][run].append(mempc2) series["mem_parts"][run].append(mempc2)
except (ValueError, IndexError): except (ValueError, IndexError):
self._log( block_err.append(i)
"[fine step] Error encountered in parsing {} (grepped block {})".format(
log_filename, i if len(block_err) > 0:
), self.logger.warning(
"WARNING", f"Error encountered in parsing {log_filename} (grepped blocks {block_err})"
) )
return series return series