Improve log system

baseprocessor.py

@@ -3,6 +3,7 @@
 import copy
 import os
 import time
+import logging
 from abc import ABCMeta
 from functools import partial
 
@@ -14,8 +15,10 @@ from tables.registry import class_name_dict
 
 from params import default_params, load_params
 from units import U
+import sys
 import traceback
 
+
 class Rule:
     def __init__(
         self,
@@ -48,10 +51,11 @@ class BaseProcessor:
 
     __metaclass__ = ABCMeta
 
-    log_id = ""
+    log_id = "base"
     rules = {}
     solve_self_dep = True
 
+
     def __init__(self, path, path_out=".", params=None, tag=None):
         if params is None:
             self.params = default_params()
@@ -69,12 +73,39 @@ class BaseProcessor:
         if tag is not None:
             self.params.out.tag = tag
 
-    def _log(self, string, status=""):
+        # Initialize logger
+        self.logger = logging.getLogger(self.log_id)
+        self.logger.propagate = False
+        logging_format = '%(levelname)s | %(asctime)s | %(name)s | %(message)s' #  %(funcName)s(%(lineno)d)
+        formatter = logging.Formatter(logging_format, datefmt = '%H:%M:%S')
+
+        if not self.logger.hasHandlers():
+            stream = logging.StreamHandler(sys.stdout)
+            stream.setFormatter(formatter)
+            self.logger.addHandler(stream)
+
+        if len(self.params.process.logfile) > 0:
+            fileHandler = logging.FileHandler(self.params.process.logfile)
+            fileHandler.setFormatter(formatter)
+            self.logger.addHandler(fileHandler)
         if self.params.process.verbose:
-            if len(status) > 0:
-                print(status + ": " + self.log_id + string)
-            else:
-                print(self.log_id + string)
+            self.logger.setLevel(logging.DEBUG)
+        else:
+            self.logger.setLevel(logging.WARNING)
+
+        for handler in self.logger.handlers:
+                handler.setFormatter(formatter)
+
+    def _log(self, string, status=""):
+        # self.logger.warning("Use of _log is deprecated, use logger instead")
+        if status == "WARNING":
+            self.logger.warning(string)
+        elif status == "ERROR":
+            self.logger.error(string)
+        elif status == "SUCCESS":
+            self.logger.info(string)
+        else:
+            self.logger.debug(f"{string}")
 
     def process(
         self,
@@ -117,11 +148,10 @@ class BaseProcessor:
                 rule.name, rule, arg, overwrite, skip_dep, select, **kwargs
             )
         else:
-            self._log(
+            self.logger.error(
                 "{} is unknown, allowed rules are {}".format(
                     to_process, self.rules.keys()
-                ),
-                "ERROR",
+                )
             )
 
     def _solve_and_process_rule(
@@ -181,7 +211,7 @@ class BaseProcessor:
         return len(self.just_done) > self.done_before_dep
 
     def _not_self_dep(self, name, dep, dep_arg, overwrite, select=None):
-        self._log("Dependency {} for {} is unknown".format(dep, name), "ERROR")
+        self.logger.error("Dependency {} for {} is unknown".format(dep, name))
 
     def _needs_computation(self, overwrite, name_full):
         return overwrite
@@ -194,14 +224,14 @@ class BaseProcessor:
 
         if name_full not in self.just_done:
             if self._needs_computation(overwrite, name_full):
-                self._log("Processing {}".format(name_full))
+                self.logger.debug("Processing {}".format(name_full))
                 data = rule.process(arg, **kwargs)
                 self._save_data(name_full, data, rule.description, rule.unit)
-                self._log("Data for {} computed".format(name_full), "SUCCESS")
+                self.logger.info("Data for {} computed".format(name_full))
                 self.just_done.append(name_full)
                 return data
             else:
-                self._log(
+                self.logger.info(
                     "Data for {} is already computed, skipping...".format(name_full)
                 )
 
@@ -244,8 +274,11 @@ class HDF5Container(BaseProcessor):
             )
         except Exception as e:
             if self.params.process.allow_error:
-                traceback.print_exc()
-                self._log(f"{repr(e)}", "ERROR")
+                traceback_lines = traceback.format_exc().splitlines()
+                for line in traceback_lines:
+                    if line != traceback_lines[-1]:
+                        self.logger.error(line)
+                self.logger.error(f"{repr(e)}")
                 pass
             else:
                 raise
@@ -411,9 +444,9 @@ class HDF5Container(BaseProcessor):
 
             group_name =  os.path.dirname(name_full)
             if group_name in self.save:
-                group = self.save[group_name]
+                group = self.save.get_node(group_name)
                 if not isinstance(group, class_name_dict['Group']):
-                    self._log(f"{group_name} already there and no a group, deleting", "WARNING")
+                    self.logger.warning(f"{group_name} already there and no a group, deleting")
                     self.save.remove_node(group)
             self.save.create_array(
                 group_name,