diff --git a/baseprocessor.py b/baseprocessor.py index c70b213..8a153fe 100644 --- a/baseprocessor.py +++ b/baseprocessor.py @@ -3,6 +3,7 @@ import copy import os import time +import logging from abc import ABCMeta from functools import partial @@ -14,8 +15,10 @@ from tables.registry import class_name_dict from params import default_params, load_params from units import U +import sys import traceback + class Rule: def __init__( self, @@ -48,10 +51,11 @@ class BaseProcessor: __metaclass__ = ABCMeta - log_id = "" + log_id = "base" rules = {} solve_self_dep = True + def __init__(self, path, path_out=".", params=None, tag=None): if params is None: self.params = default_params() @@ -69,12 +73,39 @@ class BaseProcessor: if tag is not None: self.params.out.tag = tag - def _log(self, string, status=""): + # Initialize logger + self.logger = logging.getLogger(self.log_id) + self.logger.propagate = False + logging_format = '%(levelname)s | %(asctime)s | %(name)s | %(message)s' # %(funcName)s(%(lineno)d) + formatter = logging.Formatter(logging_format, datefmt = '%H:%M:%S') + + if not self.logger.hasHandlers(): + stream = logging.StreamHandler(sys.stdout) + stream.setFormatter(formatter) + self.logger.addHandler(stream) + + if len(self.params.process.logfile) > 0: + fileHandler = logging.FileHandler(self.params.process.logfile) + fileHandler.setFormatter(formatter) + self.logger.addHandler(fileHandler) if self.params.process.verbose: - if len(status) > 0: - print(status + ": " + self.log_id + string) - else: - print(self.log_id + string) + self.logger.setLevel(logging.DEBUG) + else: + self.logger.setLevel(logging.WARNING) + + for handler in self.logger.handlers: + handler.setFormatter(formatter) + + def _log(self, string, status=""): + # self.logger.warning("Use of _log is deprecated, use logger instead") + if status == "WARNING": + self.logger.warning(string) + elif status == "ERROR": + self.logger.error(string) + elif status == "SUCCESS": + self.logger.info(string) + else: + self.logger.debug(f"{string}") def process( self, @@ -117,11 +148,10 @@ class BaseProcessor: rule.name, rule, arg, overwrite, skip_dep, select, **kwargs ) else: - self._log( + self.logger.error( "{} is unknown, allowed rules are {}".format( to_process, self.rules.keys() - ), - "ERROR", + ) ) def _solve_and_process_rule( @@ -181,7 +211,7 @@ class BaseProcessor: return len(self.just_done) > self.done_before_dep def _not_self_dep(self, name, dep, dep_arg, overwrite, select=None): - self._log("Dependency {} for {} is unknown".format(dep, name), "ERROR") + self.logger.error("Dependency {} for {} is unknown".format(dep, name)) def _needs_computation(self, overwrite, name_full): return overwrite @@ -194,14 +224,14 @@ class BaseProcessor: if name_full not in self.just_done: if self._needs_computation(overwrite, name_full): - self._log("Processing {}".format(name_full)) + self.logger.debug("Processing {}".format(name_full)) data = rule.process(arg, **kwargs) self._save_data(name_full, data, rule.description, rule.unit) - self._log("Data for {} computed".format(name_full), "SUCCESS") + self.logger.info("Data for {} computed".format(name_full)) self.just_done.append(name_full) return data else: - self._log( + self.logger.info( "Data for {} is already computed, skipping...".format(name_full) ) @@ -244,8 +274,11 @@ class HDF5Container(BaseProcessor): ) except Exception as e: if self.params.process.allow_error: - traceback.print_exc() - self._log(f"{repr(e)}", "ERROR") + traceback_lines = traceback.format_exc().splitlines() + for line in traceback_lines: + if line != traceback_lines[-1]: + self.logger.error(line) + self.logger.error(f"{repr(e)}") pass else: raise @@ -411,9 +444,9 @@ class HDF5Container(BaseProcessor): group_name = os.path.dirname(name_full) if group_name in self.save: - group = self.save[group_name] + group = self.save.get_node(group_name) if not isinstance(group, class_name_dict['Group']): - self._log(f"{group_name} already there and no a group, deleting", "WARNING") + self.logger.warning(f"{group_name} already there and no a group, deleting") self.save.remove_node(group) self.save.create_array( group_name, diff --git a/plotter.py b/plotter.py index 04b654b..460ea3f 100644 --- a/plotter.py +++ b/plotter.py @@ -239,6 +239,10 @@ class Plotter(Aggregator, BaseProcessor): kwargs : Keyword arguments for RunSelector. """ + # log info + self.log_id = "plot {}".format(tag) + + super(Plotter, self).__init__(path, path_out, params, tag) # Select runs @@ -273,9 +277,6 @@ class Plotter(Aggregator, BaseProcessor): # Get postprocesor objets for each run self.snaps = self.study.snaps - # Define log prefix - self.log_id = "[plot {}] ".format(self.params.out.tag) - # Define rules self.def_rules() @@ -312,7 +313,7 @@ class Plotter(Aggregator, BaseProcessor): try: value = self.study.get_nml(param.key, run) except KeyError as e: - self._log("key {} not found".format(e), "WARNING") + self.logger.warning("key {} not found".format(e)) if value is not None: try: @@ -858,7 +859,7 @@ class Plotter(Aggregator, BaseProcessor): plt.xlim(xlim) plt.ylim(ylim) - if self.params.astrophysics.generate: + if self.params.astrophysix.generate: return PlotInfo( plot_type=PlotType.IMAGE, xaxis_values=np.linspace(im_extent[0], im_extent[1], dmap.shape[0] + 1), @@ -1203,7 +1204,7 @@ class Plotter(Aggregator, BaseProcessor): ) # returns PlotInfo (for Galactica) - if self.params.astrophysics.generate: + if self.params.astrophysix.generate: edges = np.append(centers - width / 2.0, centers[-1] + width / 2.0) return PlotInfo( plot_type=PlotType.HISTOGRAM, @@ -1429,7 +1430,7 @@ class Plotter(Aggregator, BaseProcessor): if kind == "linear": if yerr is None or np.sum(np.abs(yerr)) == 0: (a, b, rho, _map_rule, stderr) = linregress(x, y) - self._log( + self.logger.info( "Linear fit y = {} x + {} with R^2 = {} and error is {}".format( a, b, rho, stderr ) @@ -1443,7 +1444,7 @@ class Plotter(Aggregator, BaseProcessor): c = fit[0] residual = fit[1][0][0] b, a = c[0], c[1] - self._log( + self.logger.info( "Linear fit y = {} x + {} with residual {}".format(a, b, residual) ) if label is None: @@ -1452,7 +1453,7 @@ class Plotter(Aggregator, BaseProcessor): elif kind == "power_law": if yerr is None or np.sum(np.abs(yerr)) == 0: (a, b, rho, _map_rule, stderr) = linregress(np.log10(x), np.log10(y)) - self._log( + self.logger.info( "Power law fit y = x^({}) * {} with R^2 = {} and error is {}".format( a, 10 ** b, rho, stderr ) @@ -1476,7 +1477,7 @@ class Plotter(Aggregator, BaseProcessor): c = out[0] b, a = c[0], c[1] residual = errfunc(c, np.log10(x), np.log10(y), yerr / y) - self._log( + self.logger.info( "Power law fit y = x^({}) * {} with residual {}".format( a, 10 ** b, residual ) diff --git a/snapshotprocessor.py b/snapshotprocessor.py index 49584eb..0442c8b 100644 --- a/snapshotprocessor.py +++ b/snapshotprocessor.py @@ -310,6 +310,11 @@ class SnapshotProcessor(HDF5Container): unit_time : Unit instance, used for astrophysix """ + self.path = path + self.run = os.path.basename(path) + self.num = num + self.log_id = "{}, {}".format(self.run, self.num) + super(SnapshotProcessor, self).__init__(path, path_out, params, tag) # Open outfile @@ -334,10 +339,6 @@ class SnapshotProcessor(HDF5Container): if not os.path.exists(f"{self.path_out}{subfolder}"): os.makedirs(f"{self.path_out}{subfolder}") - self.path = path - self.run = os.path.basename(path) - self.num = num - # Create selector object if selector is None: selector = RunSelector( @@ -373,8 +374,6 @@ class SnapshotProcessor(HDF5Container): self.save.root._v_attrs.time = self.time self.close() - self.log_id = "[{}, {}] ".format(self.run, self.num) - if os.path.exists(self.filaments_filename): with open(self.filaments_filename, "rb") as f: self.fil = pickle.load(f) diff --git a/studyprocessor.py b/studyprocessor.py index a655689..747188f 100644 --- a/studyprocessor.py +++ b/studyprocessor.py @@ -38,13 +38,18 @@ class StudyProcessor(Aggregator, HDF5Container): Creates the basic structures needed for the outputs """ + + # log id + self.log_id = "study {}".format(tag) + + + super(StudyProcessor, self).__init__(path, path_out, params, tag) # Open outfile + tag_name = self.params.out.tag if not self.params.out.tag == "": - tag_name = "_" + self.params.out.tag - else: - tag_name = "" + tag_name = "_" + tag_name if self.params.out.ext_subfolder: self.filename = f"{self.path_out}/h5/study{tag_name}.h5" @@ -100,9 +105,6 @@ class StudyProcessor(Aggregator, HDF5Container): dest = f"{self.path_out}/{run}/logs/{os.path.basename(log)}" copy_file(log, dest, update=1) - # log info - self.log_id = "[study {}] ".format(self.params.out.tag) - # Define rules self.def_rules()