Correct bugs, add extractor for turb_rms

plotter.py

@@ -680,6 +680,17 @@ class Plotter(Aggregator, BaseProcessor):
                     kind="series",
                     dependencies=["issfr"],
                 ),
+                "turb_rms": PlotRule(
+                    self,
+                    partial(
+                        self._plot,
+                        "/series/rms_from_log/time",
+                        "/series/rms_from_log/turb_rms",
+                        xunit=cst.Myr,
+                    ),
+                    kind="series",
+                    dependencies=["rms_from_log"],
+                ),
                 "sigma": PlotRule(
                     self,
                     partial(
@@ -691,7 +702,7 @@ class Plotter(Aggregator, BaseProcessor):
                         yunit=cst.km_s,
                     ),
                     kind="comp",
-                    dependencies=["time", "time_sigma"],
+                    dependencies=["time_sigma"],
                 ),
                 "plot": PlotRule(
                     self, lambda arg, **kwargs: self._plot(*arg, **kwargs), kind="comp"

postprocessor.py

@@ -20,7 +20,7 @@ from pymses.analysis import ScalarOperator, FractionOperator, MaxLevelOperator
 from mypool import MyPool as Pool
 from functools import partial
 from abc import ABCMeta, abstractmethod
-
+import contextlib
 import bunch
 
 from run_selector import *
@@ -77,7 +77,7 @@ class BaseProcessor:
     __metaclass__ = ABCMeta
 
     log_id = ""
-    rules = dict()
+    rules = {}
     solve_self_dep = True
 
     def __init__(self, path, path_out=None, pp_params=None, tag=None):
@@ -98,6 +98,7 @@ class BaseProcessor:
             self.path_out = path_out
 
     def _log(self, string, status=""):
+        if self.pp_params.process.verbose:
             if len(status) > 0:
                 print(status + ": " + self.log_id + string)
             else:
@@ -165,7 +166,7 @@ class BaseProcessor:
     def _not_self_dep(self, name, dep, dep_arg, overwrite, **kwargs):
         self._log("Dependency {} for {} is unknown".format(dep, name), "ERROR")
 
-    def _check_existing(self, overwrite, name_full):
+    def _needs_computation(self, overwrite, name_full):
         return overwrite or not (name_full in self.save)
 
     def _process_rule(self, name, rule, arg, overwrite=False, **kwargs):
@@ -176,7 +177,7 @@ class BaseProcessor:
 
         if rule.is_valid(arg):
             if not name_full in self.just_done:
-                if self._check_existing(overwrite, name_full):
+                if self._needs_computation(overwrite, name_full):
                     self._log("Processing {}".format(name_full))
                     data = rule.process(arg, **kwargs)
                     self._save_data(name_full, data, rule.description, rule.unit)
@@ -278,7 +279,7 @@ class HDF5Container(BaseProcessor):
         try:
             node = self.save.get_node(node_name)
             if node._v_attrs.CLASS == "GROUP":
-                value = dict()
+                value = {}
                 for child_name in node._v_children:
                     value[child_name] = self.get_value(node_name + "/" + child_name)
             else:
@@ -353,7 +354,7 @@ class PostProcessor(HDF5Container):
         path=None,
         num=None,
         path_out=None,
-        pp_params=default_params(),
+        pp_params=None,
         tag=None,
         variables=["rho", "vel", "Br", "Bl", "P", "g", "phi"],
     ):
@@ -364,8 +365,8 @@ class PostProcessor(HDF5Container):
         super(PostProcessor, self).__init__(path, path_out, pp_params, tag)
 
         # Open outfile
-        if not pp_params.out.tag == "":
+        if not self.pp_params.out.tag == "":
-            tag_name = pp_params.out.tag + "_"
+            tag_name = self.pp_params.out.tag + "_"
         else:
             tag_name = ""
 
@@ -426,7 +427,7 @@ class PostProcessor(HDF5Container):
             self.save.root.maps._v_attrs.im_extent = im_extent
 
         # Initialize cameras
-        self._cam = dict()
+        self._cam = {}
         for ax_los in self._ax_nb:  # los = line of sight
             ax_h = self._axes_h[ax_los]
             ax_v = self._axes_v[ax_los]
@@ -463,8 +464,15 @@ class PostProcessor(HDF5Container):
             self.label = self.run
         self.save.root._v_attrs.label = self.label
 
+    # def open_pymlog(self):
+    #     if self.pp_params.pymses.verbose:
+    #         return sys.stdout
+    #     else:
+    #         return open(os.devnull, "w")
+
     def load_cells(self):
         if not self.cells_loaded:
+            # with self.open_pymlog() as f, contextlib.redirect_stdout(f):
             cell_source = CellsToPoints(self._amr)
             self.cells = cell_source.flatten()
             self.cells_loaded = True
@@ -1022,18 +1030,30 @@ class Comparator(Aggregator, HDF5Container):
         # Define rules
         self.def_rules()
 
-    def _check_existing(self, overwrite, name_full):
+    def _needs_computation(self, overwrite, name_full):
+        """
+        Returns True if a new computation of the rule is needed
+        """
         if overwrite or not (name_full in self.save):
             return True
-        elif not "runs" in self.save.get_node(name_full)._v_attrs:
+        elif not "nums" in self.save.get_node(name_full)._v_attrs:
             return True
         else:
-            saved_runs = self.save.get_node(name_full)._v_attrs.runs
+            saved_nums = self.save.get_node(name_full)._v_attrs.nums
-            return len([run for run in self.runs if not run in saved_runs]) == 0
+            missing_runs = len([run for run in self.nums if not run in saved_nums]) > 0
+            missing_nums = missing_runs and all(
+                [
+                    len([num for num in self.nums[run] if not num in saved_nums[run]])
+                    > 0
+                    for run in self.nums
+                    if run in saved_nums
+                ]
+            )
+            return missing_nums
 
     def _save_data(self, name_full, data, description, unit):
         super(Comparator, self)._save_data(name_full, data, description, unit)
-        self.save.get_node(name_full)._v_attrs.runs = self.runs
+        self.save.get_node(name_full)._v_attrs.nums = self.nums
 
     def _time_series(self, getter):
         series = {}
@@ -1122,6 +1142,14 @@ class Comparator(Aggregator, HDF5Container):
             series["sfr"][run].append(sfr)
         return series
 
+    def _extract_rms_from_log(self, series, log_filename, run):
+        cmd_grep = "grep 'turbulent rms' {} -B 1".format(log_filename)
+        content = os.popen(cmd_grep).readlines()
+        for i in range(0, len(content), 3):
+            series["time"][run].append(np.float(content[i].split("=")[2].split()[0]))
+            series["turb_rms"][run].append(np.float(content[i + 1].split(":")[1]))
+        return series
+
     def _from_log(self, keys, extractor):
         nums = self.nums
 
@@ -1265,6 +1293,15 @@ class Comparator(Aggregator, HDF5Container):
                     "sfr": "Averaged surfacic star formation rate",
                 },
             ),
+            "rms_from_log": Rule(
+                self,
+                partial(
+                    self._from_log, ["time", "turb_rms"], self._extract_rms_from_log
+                ),
+                group="/series",
+                unit={"time": self.info["unit_time"], "turb_rms": cst.none},
+                description={"time": "Time", "turb_rms": "Computed turbulent RMS"},
+            ),
             # Read from outputs
             "time": Rule(
                 self,

run_selector.py

@@ -14,7 +14,6 @@ class RunSelector:
         in_runs=None,
         in_nums="all",
         pp_params=default_params(),
-        number_run="[0-9]*_",
         name_run="*",
         namelist_cond={},
         sort_run_by=None,
@@ -25,9 +24,7 @@ class RunSelector:
         self.pp_params = pp_params
 
         self.namelist = {}
-        self.runs = self.get_runs(
+        self.runs = self.get_runs(in_runs, name_run, namelist_cond, sort_run_by)
-            in_runs, number_run, name_run, namelist_cond, sort_run_by
-        )
 
         self.info = {}
         for run in self.runs:
@@ -54,14 +51,7 @@ class RunSelector:
             res = res[key]
         return res
 
-    def get_runs(
+    def get_runs(self, in_runs=None, name_run="*", namelist_cond={}, sort_run_by=None):
-        self,
-        in_runs=None,
-        number_run="[0-9]*_",
-        name_run="*",
-        namelist_cond={},
-        sort_run_by=None,
-    ):
         def try_load_nml(run):
             try:
                 self.namelist[run] = self.load_namelist(run)
@@ -71,7 +61,8 @@ class RunSelector:
             return success
 
         runs = map(
-            os.path.basename, glob.glob(self.path_in + "/" + number_run + name_run)
+            os.path.basename,
+            filter(os.path.isdir, glob.glob(self.path_in + "/" + name_run)),
         )
         if not in_runs is None:
             runs = filter(lambda n: n in runs, in_runs)
@@ -140,6 +131,8 @@ class RunSelector:
         )
         nums = map(lambda n: int(n.split("/")[-1].split("_")[1]), names)
 
+        if type(in_nums) == int:
+            in_nums = [in_nums]
         if type(in_nums) == list:
             nums = filter(lambda n: n in nums, in_nums)