noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Improve log system

Browse Source
This commit is contained in:
Noe Brucy
2022-08-26 20:06:38 +02:00
parent 07017611ee
commit 67f8f014d6
4 changed files with 74 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
baseprocessor.py
+50 -17
View File
@@ -3,6 +3,7 @@
import copy import copy
import os import os
import time import time
import logging
from abc import ABCMeta from abc import ABCMeta
from functools import partial from functools import partial
@@ -14,8 +15,10 @@ from tables.registry import class_name_dict
from params import default_params, load_params from params import default_params, load_params
from units import U from units import U
import sys
import traceback import traceback
class Rule: class Rule:
def __init__( def __init__(
self, self,
@@ -48,10 +51,11 @@ class BaseProcessor:
__metaclass__ = ABCMeta __metaclass__ = ABCMeta
log_id = "" log_id = "base"
rules = {} rules = {}
solve_self_dep = True solve_self_dep = True
def __init__(self, path, path_out=".", params=None, tag=None): def __init__(self, path, path_out=".", params=None, tag=None):
if params is None: if params is None:
self.params = default_params() self.params = default_params()
@@ -69,12 +73,39 @@ class BaseProcessor:
if tag is not None: if tag is not None:
self.params.out.tag = tag self.params.out.tag = tag
def _log(self, string, status=""): # Initialize logger
self.logger = logging.getLogger(self.log_id)
self.logger.propagate = False
logging_format = '%(levelname)s | %(asctime)s | %(name)s | %(message)s' # %(funcName)s(%(lineno)d)
formatter = logging.Formatter(logging_format, datefmt = '%H:%M:%S')
if not self.logger.hasHandlers():
stream = logging.StreamHandler(sys.stdout)
stream.setFormatter(formatter)
self.logger.addHandler(stream)
if len(self.params.process.logfile) > 0:
fileHandler = logging.FileHandler(self.params.process.logfile)
fileHandler.setFormatter(formatter)
self.logger.addHandler(fileHandler)
if self.params.process.verbose: if self.params.process.verbose:
if len(status) > 0: self.logger.setLevel(logging.DEBUG)
print(status + ": " + self.log_id + string) else:
else: self.logger.setLevel(logging.WARNING)
print(self.log_id + string)
for handler in self.logger.handlers:
handler.setFormatter(formatter)
def _log(self, string, status=""):
# self.logger.warning("Use of _log is deprecated, use logger instead")
if status == "WARNING":
self.logger.warning(string)
elif status == "ERROR":
self.logger.error(string)
elif status == "SUCCESS":
self.logger.info(string)
else:
self.logger.debug(f"{string}")
def process( def process(
self, self,
@@ -117,11 +148,10 @@ class BaseProcessor:
rule.name, rule, arg, overwrite, skip_dep, select, **kwargs rule.name, rule, arg, overwrite, skip_dep, select, **kwargs
) )
else: else:
self._log( self.logger.error(
"{} is unknown, allowed rules are {}".format( "{} is unknown, allowed rules are {}".format(
to_process, self.rules.keys() to_process, self.rules.keys()
), )
"ERROR",
) )
def _solve_and_process_rule( def _solve_and_process_rule(
@@ -181,7 +211,7 @@ class BaseProcessor:
return len(self.just_done) > self.done_before_dep return len(self.just_done) > self.done_before_dep
def _not_self_dep(self, name, dep, dep_arg, overwrite, select=None): def _not_self_dep(self, name, dep, dep_arg, overwrite, select=None):
self._log("Dependency {} for {} is unknown".format(dep, name), "ERROR") self.logger.error("Dependency {} for {} is unknown".format(dep, name))
def _needs_computation(self, overwrite, name_full): def _needs_computation(self, overwrite, name_full):
return overwrite return overwrite
@@ -194,14 +224,14 @@ class BaseProcessor:
if name_full not in self.just_done: if name_full not in self.just_done:
if self._needs_computation(overwrite, name_full): if self._needs_computation(overwrite, name_full):
self._log("Processing {}".format(name_full)) self.logger.debug("Processing {}".format(name_full))
data = rule.process(arg, **kwargs) data = rule.process(arg, **kwargs)
self._save_data(name_full, data, rule.description, rule.unit) self._save_data(name_full, data, rule.description, rule.unit)
self._log("Data for {} computed".format(name_full), "SUCCESS") self.logger.info("Data for {} computed".format(name_full))
self.just_done.append(name_full) self.just_done.append(name_full)
return data return data
else: else:
self._log( self.logger.info(
"Data for {} is already computed, skipping...".format(name_full) "Data for {} is already computed, skipping...".format(name_full)
) )
@@ -244,8 +274,11 @@ class HDF5Container(BaseProcessor):
) )
except Exception as e: except Exception as e:
if self.params.process.allow_error: if self.params.process.allow_error:
traceback.print_exc() traceback_lines = traceback.format_exc().splitlines()
self._log(f"{repr(e)}", "ERROR") for line in traceback_lines:
if line != traceback_lines[-1]:
self.logger.error(line)
self.logger.error(f"{repr(e)}")
pass pass
else: else:
raise raise
@@ -411,9 +444,9 @@ class HDF5Container(BaseProcessor):
group_name = os.path.dirname(name_full) group_name = os.path.dirname(name_full)
if group_name in self.save: if group_name in self.save:
group = self.save[group_name] group = self.save.get_node(group_name)
if not isinstance(group, class_name_dict['Group']): if not isinstance(group, class_name_dict['Group']):
self._log(f"{group_name} already there and no a group, deleting", "WARNING") self.logger.warning(f"{group_name} already there and no a group, deleting")
self.save.remove_node(group) self.save.remove_node(group)
self.save.create_array( self.save.create_array(
group_name, group_name,
plotter.py
+11 -10
View File
@@ -239,6 +239,10 @@ class Plotter(Aggregator, BaseProcessor):
kwargs : Keyword arguments for RunSelector. kwargs : Keyword arguments for RunSelector.
""" """
# log info
self.log_id = "plot {}".format(tag)
super(Plotter, self).__init__(path, path_out, params, tag) super(Plotter, self).__init__(path, path_out, params, tag)
# Select runs # Select runs
@@ -273,9 +277,6 @@ class Plotter(Aggregator, BaseProcessor):
# Get postprocesor objets for each run # Get postprocesor objets for each run
self.snaps = self.study.snaps self.snaps = self.study.snaps
# Define log prefix
self.log_id = "[plot {}] ".format(self.params.out.tag)
# Define rules # Define rules
self.def_rules() self.def_rules()
@@ -312,7 +313,7 @@ class Plotter(Aggregator, BaseProcessor):
try: try:
value = self.study.get_nml(param.key, run) value = self.study.get_nml(param.key, run)
except KeyError as e: except KeyError as e:
self._log("key {} not found".format(e), "WARNING") self.logger.warning("key {} not found".format(e))
if value is not None: if value is not None:
try: try:
@@ -858,7 +859,7 @@ class Plotter(Aggregator, BaseProcessor):
plt.xlim(xlim) plt.xlim(xlim)
plt.ylim(ylim) plt.ylim(ylim)
if self.params.astrophysics.generate: if self.params.astrophysix.generate:
return PlotInfo( return PlotInfo(
plot_type=PlotType.IMAGE, plot_type=PlotType.IMAGE,
xaxis_values=np.linspace(im_extent[0], im_extent[1], dmap.shape[0] + 1), xaxis_values=np.linspace(im_extent[0], im_extent[1], dmap.shape[0] + 1),
@@ -1203,7 +1204,7 @@ class Plotter(Aggregator, BaseProcessor):
) )
# returns PlotInfo (for Galactica) # returns PlotInfo (for Galactica)
if self.params.astrophysics.generate: if self.params.astrophysix.generate:
edges = np.append(centers - width / 2.0, centers[-1] + width / 2.0) edges = np.append(centers - width / 2.0, centers[-1] + width / 2.0)
return PlotInfo( return PlotInfo(
plot_type=PlotType.HISTOGRAM, plot_type=PlotType.HISTOGRAM,
@@ -1429,7 +1430,7 @@ class Plotter(Aggregator, BaseProcessor):
if kind == "linear": if kind == "linear":
if yerr is None or np.sum(np.abs(yerr)) == 0: if yerr is None or np.sum(np.abs(yerr)) == 0:
(a, b, rho, _map_rule, stderr) = linregress(x, y) (a, b, rho, _map_rule, stderr) = linregress(x, y)
self._log( self.logger.info(
"Linear fit y = {} x + {} with R^2 = {} and error is {}".format( "Linear fit y = {} x + {} with R^2 = {} and error is {}".format(
a, b, rho, stderr a, b, rho, stderr
) )
@@ -1443,7 +1444,7 @@ class Plotter(Aggregator, BaseProcessor):
c = fit[0] c = fit[0]
residual = fit[1][0][0] residual = fit[1][0][0]
b, a = c[0], c[1] b, a = c[0], c[1]
self._log( self.logger.info(
"Linear fit y = {} x + {} with residual {}".format(a, b, residual) "Linear fit y = {} x + {} with residual {}".format(a, b, residual)
) )
if label is None: if label is None:
@@ -1452,7 +1453,7 @@ class Plotter(Aggregator, BaseProcessor):
elif kind == "power_law": elif kind == "power_law":
if yerr is None or np.sum(np.abs(yerr)) == 0: if yerr is None or np.sum(np.abs(yerr)) == 0:
(a, b, rho, _map_rule, stderr) = linregress(np.log10(x), np.log10(y)) (a, b, rho, _map_rule, stderr) = linregress(np.log10(x), np.log10(y))
self._log( self.logger.info(
"Power law fit y = x^({}) * {} with R^2 = {} and error is {}".format( "Power law fit y = x^({}) * {} with R^2 = {} and error is {}".format(
a, 10 ** b, rho, stderr a, 10 ** b, rho, stderr
) )
@@ -1476,7 +1477,7 @@ class Plotter(Aggregator, BaseProcessor):
c = out[0] c = out[0]
b, a = c[0], c[1] b, a = c[0], c[1]
residual = errfunc(c, np.log10(x), np.log10(y), yerr / y) residual = errfunc(c, np.log10(x), np.log10(y), yerr / y)
self._log( self.logger.info(
"Power law fit y = x^({}) * {} with residual {}".format( "Power law fit y = x^({}) * {} with residual {}".format(
a, 10 ** b, residual a, 10 ** b, residual
) )
snapshotprocessor.py
+5 -6
View File
@@ -310,6 +310,11 @@ class SnapshotProcessor(HDF5Container):
unit_time : Unit instance, used for astrophysix unit_time : Unit instance, used for astrophysix
""" """
self.path = path
self.run = os.path.basename(path)
self.num = num
self.log_id = "{}, {}".format(self.run, self.num)
super(SnapshotProcessor, self).__init__(path, path_out, params, tag) super(SnapshotProcessor, self).__init__(path, path_out, params, tag)
# Open outfile # Open outfile
@@ -334,10 +339,6 @@ class SnapshotProcessor(HDF5Container):
if not os.path.exists(f"{self.path_out}{subfolder}"): if not os.path.exists(f"{self.path_out}{subfolder}"):
os.makedirs(f"{self.path_out}{subfolder}") os.makedirs(f"{self.path_out}{subfolder}")
self.path = path
self.run = os.path.basename(path)
self.num = num
# Create selector object # Create selector object
if selector is None: if selector is None:
selector = RunSelector( selector = RunSelector(
@@ -373,8 +374,6 @@ class SnapshotProcessor(HDF5Container):
self.save.root._v_attrs.time = self.time self.save.root._v_attrs.time = self.time
self.close() self.close()
self.log_id = "[{}, {}] ".format(self.run, self.num)
if os.path.exists(self.filaments_filename): if os.path.exists(self.filaments_filename):
with open(self.filaments_filename, "rb") as f: with open(self.filaments_filename, "rb") as f:
self.fil = pickle.load(f) self.fil = pickle.load(f)
studyprocessor.py
+8 -6
View File
@@ -38,13 +38,18 @@ class StudyProcessor(Aggregator, HDF5Container):
Creates the basic structures needed for the outputs Creates the basic structures needed for the outputs
""" """
# log id
self.log_id = "study {}".format(tag)
super(StudyProcessor, self).__init__(path, path_out, params, tag) super(StudyProcessor, self).__init__(path, path_out, params, tag)
# Open outfile # Open outfile
tag_name = self.params.out.tag
if not self.params.out.tag == "": if not self.params.out.tag == "":
tag_name = "_" + self.params.out.tag tag_name = "_" + tag_name
else:
tag_name = ""
if self.params.out.ext_subfolder: if self.params.out.ext_subfolder:
self.filename = f"{self.path_out}/h5/study{tag_name}.h5" self.filename = f"{self.path_out}/h5/study{tag_name}.h5"
@@ -100,9 +105,6 @@ class StudyProcessor(Aggregator, HDF5Container):
dest = f"{self.path_out}/{run}/logs/{os.path.basename(log)}" dest = f"{self.path_out}/{run}/logs/{os.path.basename(log)}"
copy_file(log, dest, update=1) copy_file(log, dest, update=1)
# log info
self.log_id = "[study {}] ".format(self.params.out.tag)
# Define rules # Define rules
self.def_rules() self.def_rules()