From 6e0f5b429716185f2f255dcfbd7c98b1ee011982 Mon Sep 17 00:00:00 2001 From: Noe Brucy Date: Fri, 2 Sep 2022 16:16:17 +0200 Subject: [PATCH] Improve logging --- baseprocessor.py | 15 ++------------- snapshotprocessor.py | 33 ++++++++++++++++++--------------- studyprocessor.py | 37 ++++++++++++++++++++----------------- 3 files changed, 40 insertions(+), 45 deletions(-) diff --git a/baseprocessor.py b/baseprocessor.py index 8a153fe..1fb8e9b 100644 --- a/baseprocessor.py +++ b/baseprocessor.py @@ -76,7 +76,7 @@ class BaseProcessor: # Initialize logger self.logger = logging.getLogger(self.log_id) self.logger.propagate = False - logging_format = '%(levelname)s | %(asctime)s | %(name)s | %(message)s' # %(funcName)s(%(lineno)d) + logging_format = '%(levelname)s | %(asctime)s | %(name)s.%(funcName)s:%(lineno)d | %(message)s' formatter = logging.Formatter(logging_format, datefmt = '%H:%M:%S') if not self.logger.hasHandlers(): @@ -96,17 +96,6 @@ class BaseProcessor: for handler in self.logger.handlers: handler.setFormatter(formatter) - def _log(self, string, status=""): - # self.logger.warning("Use of _log is deprecated, use logger instead") - if status == "WARNING": - self.logger.warning(string) - elif status == "ERROR": - self.logger.error(string) - elif status == "SUCCESS": - self.logger.info(string) - else: - self.logger.debug(f"{string}") - def process( self, to_process, @@ -232,7 +221,7 @@ class BaseProcessor: return data else: self.logger.info( - "Data for {} is already computed, skipping...".format(name_full) + "Data for {} is already computed.".format(name_full) ) def def_rules(self): diff --git a/snapshotprocessor.py b/snapshotprocessor.py index 0442c8b..fc245d9 100644 --- a/snapshotprocessor.py +++ b/snapshotprocessor.py @@ -313,7 +313,7 @@ class SnapshotProcessor(HDF5Container): self.path = path self.run = os.path.basename(path) self.num = num - self.log_id = "{}, {}".format(self.run, self.num) + self.log_id = "snap({}, {})".format(self.run, self.num) super(SnapshotProcessor, self).__init__(path, path_out, params, tag) @@ -400,8 +400,9 @@ class SnapshotProcessor(HDF5Container): try: self.init_pymses() - except FileNotFoundError: - self._log("Pymses not initialized", "WARNING") + except: + self.logger.error("Pymses not initialized", exc_info=1) + self.def_rules() @@ -483,19 +484,21 @@ class SnapshotProcessor(HDF5Container): far_cut_depth=distance, up_vector=ax_v, map_max_size=self.params.pymses.map_size, - ) + ) - self.open() - if "/maps" not in self.save: - self.save.create_group("/", "maps", "2D maps") - if "/hist" not in self.save: - self.save.create_group("/", "hist", "Histograms") - if "/datasets" not in self.save: - self.save.create_group("/", "datasets", "Complex datasets") - self.save.root.maps._v_attrs.center = center - self.save.root.maps._v_attrs.radius = self._radius - self.save.root.maps._v_attrs.im_extent = im_extent - self.close() + # Initialize HDF5 group + try: + self.open() + if "/maps" not in self.save: + self.save.create_group("/", "maps", "2D maps") + self.save.root.maps._v_attrs.center = center + self.save.root.maps._v_attrs.radius = self._radius + self.save.root.maps._v_attrs.im_extent = im_extent + except: + self.logger.error("Error in HDF5", exc_info=1) + raise + finally: + self.close() def load_data(self, points_src, filename, save, keys=None): """ diff --git a/studyprocessor.py b/studyprocessor.py index 747188f..d1f0c7b 100644 --- a/studyprocessor.py +++ b/studyprocessor.py @@ -40,7 +40,7 @@ class StudyProcessor(Aggregator, HDF5Container): # log id - self.log_id = "study {}".format(tag) + self.log_id = "study({})".format(tag) @@ -156,7 +156,7 @@ class StudyProcessor(Aggregator, HDF5Container): prop[run] = getter(run, num) else: prop[run] = getter(run) - return np.array(list(prop.keys())) + return np.array(list(prop.values())) def time_avg(self, name, start=None, end=None, span=None, unit_time=U.Myr, group="/series", select=None): """Do the time average and quantiles of a time series @@ -295,6 +295,7 @@ class StudyProcessor(Aggregator, HDF5Container): def _extract_sinks_from_log(self, series, log_filename, run): cmd_grep = f"grep 'Number of sink' {log_filename} -A 2" content = os.popen(cmd_grep).readlines() + block_err = [] # Block that will ill parsed for i in range(0, len(content), 4): try: nb_sink = np.int(content[i].split("=")[1]) @@ -309,11 +310,11 @@ class StudyProcessor(Aggregator, HDF5Container): series["time"][run].append(time) except (ValueError, IndexError): - self._log( - "Error encountered in parsing {} (grepped block {})".format( - log_filename, i - ), - "WARNING", + block_err.append(i) + + if len(block_err) > 0: + self.logger.warning( + f"Errors encountered in parsing {log_filename} (grepped blocks {block_err})" ) return series @@ -323,6 +324,7 @@ class StudyProcessor(Aggregator, HDF5Container): nb_stellar = list(map(lambda s: int(s.split()[1]), content)) line_numbers = list(map(lambda s: int(s.split(":")[0]), content)) current_line = 0 + block_err = [] # Block that will ill parsed logfile = open(log_filename) for i in range(0, len(line_numbers)): try: @@ -346,11 +348,11 @@ class StudyProcessor(Aggregator, HDF5Container): stellar_objects["id"][run].append(id) except (ValueError, IndexError): - self._log( - "[stellar] Error encountered in parsing {} (grepped block {})".format( - log_filename, i - ), - "WARNING", + block_err.append(i) + + if len(block_err) > 0: + self.logger.warning( + f"Errors encountered in parsing {log_filename} (grepped blocks {block_err})" ) logfile.close() return stellar_objects @@ -368,6 +370,7 @@ class StudyProcessor(Aggregator, HDF5Container): def _extract_fine_step_from_log(self, series, log_filename, run): cmd_grep = "grep 'Fine step' {} ".format(log_filename) content = os.popen(cmd_grep).readlines() + block_err = [] # Block that will ill parsed for i in range(0, len(content)): try: data = content[i].replace("=", " ").split() @@ -384,11 +387,11 @@ class StudyProcessor(Aggregator, HDF5Container): series["mem_cells"][run].append(mempc1) series["mem_parts"][run].append(mempc2) except (ValueError, IndexError): - self._log( - "[fine step] Error encountered in parsing {} (grepped block {})".format( - log_filename, i - ), - "WARNING", + block_err.append(i) + + if len(block_err) > 0: + self.logger.warning( + f"Error encountered in parsing {log_filename} (grepped blocks {block_err})" ) return series