Added extror from logs and namelist

2019-11-13 17:33:15 +01:00
parent 3986b1cdf4
commit ea6f9b6bdd
4 changed files with 563 additions and 249 deletions
@@ -2,25 +2,48 @@

 import sys
 import os
+import glob as glob
+

 import tables
 import pymses
 import numpy as np
 from numpy.polynomial.polynomial import polyfit
+from scipy.stats import linregress
+
 from pymses.sources.ramses import output
 from pymses.sources.hop.file_formats import *
 from pymses.analysis import Camera, raytracing, slicing, splatting
 from pymses.filters import CellsToPoints
 from pymses.analysis import ScalarOperator, FractionOperator, MaxLevelOperator
-from scipy.stats import linregress
+import pymses.utils.constants as cst
+
+import f90nml
+import bunch
+
 from functools import partial
 from abc import ABCMeta, abstractmethod

 from pp_params import *

+def unit_str(unit):
+    if unit == cst.none:
+        return ''
+    elif len(unit.name) > 0 :
+        return  ' ({})'.format(unit.name)
+    else:
+        return ' ' + str(unit)
+
+cst.ssfr = cst.create_unit('Msun/yr/pc^2',
+                           base_unit=cst.Msun/cst.year/cst.pc**2,
+                           descr="Surfacic SFR",
+                           latex='M_{\astrosun}.yr^{-1}.pc^{-2}')
+
+
 class Rule:
-    def __init__(self, postproc, process, description='', group='', dependencies=[], args_ok=['x', 'y', 'z'],
-                 is_valid=lambda arg:True):
+    def __init__(self, postproc, process, description='',
+                 group='', dependencies=[], args_ok=['x', 'y', 'z'],
+                 is_valid=lambda arg:True, kind='classic', unit=cst.none):
        self.postproc = postproc
        self.process_fn = process
        self.dependencies = dependencies
@@ -28,12 +51,14 @@ class Rule:
        self.group = group
        self.args_ok = args_ok
        self.description = description
+        self.unit=unit
+        self.kind = kind

-    def process(self, arg):
+    def process(self, arg, **kwargs):
        if not arg is None:
-            return self.process_fn(arg)
+            return self.process_fn(arg, **kwargs)
        else:
-            return self.process_fn()
+            return self.process_fn(**kwargs)

    def is_valid(self, arg):
        save = self.postproc.save
@@ -55,7 +80,9 @@ class BaseProcessor:

    log_id = ""
    rules = dict()
+    data = dict()
    filename = ""
+    save = None

    @abstractmethod
    def __init__(self):
@@ -67,39 +94,45 @@ class BaseProcessor:
        else:
            print(self.log_id + string)

-    def process(self, to_process_list, args=[None], overwrite=False, overwrite_dep=None):
+    def open(self):
+        self.save = tables.open_file(self.filename, mode="a")
+
+    def close(self):
+        self.save.close()
+
+    def process(self, to_process_list, args=[None], overwrite=False, overwrite_dep=False, **kwargs):
        """
        Render the data in to_process_list and save them
        """
-        if overwrite_dep is None:
-            overwrite_dep = overwrite
-
        self.overwrite_dep = overwrite_dep
        just_done = [] # Computations done within this call

-        self.save = tables.open_file(self.filename, mode="a")
        for name in to_process_list:
            if name in self.rules:
                rule = self.rules[name]
                for arg in args:
-                  just_done = self._process_single(name, rule, arg, overwrite, just_done)
+                  just_done = self._process_single(name, rule, arg, overwrite, just_done, **kwargs)
            else:
                self._log("{} is unknown, allowed rules are {}".format(name, self.rules.keys()), "ERROR")
-        self.save.close()

-    def _process_single(self, name, rule, arg, overwrite=False, just_done=[]):
+    def _process_single(self, name, rule, arg, overwrite=False, just_done=[], **kwargs):
        # Solve dependencies
        for dep in rule.dependencies:
            if dep in self.rules:
                rule_dep = self.rules[dep]
-                just_done = self._process_single(dep, rule_dep, arg, self.overwrite_dep, just_done)
+                just_done = self._process_single(dep, rule_dep, arg, self.overwrite_dep,
+                                                 just_done)
            else:
                self._log("Dependency {} for {} is unknown".format(dep, name), "ERROR")
        # Process rule
-        done = self._process_rule(name, rule, arg, overwrite, just_done)
+        self.open()
+        try:
+            done = self._process_rule(name, rule, arg, overwrite, just_done, **kwargs)
+        finally:
+            self.close()
        return just_done + [done]

-    def _process_rule(self, name, rule, arg, overwrite=False, just_done=[]):
+    def _process_rule(self, name, rule, arg, overwrite=False, just_done=[], **kwargs):
        if not arg is None:
            name_full = rule.group + '/' + name + '_' + str(arg)
        else:
@@ -107,10 +140,11 @@ class BaseProcessor:

        if rule.is_valid(arg):
            if not name_full in just_done:
-                if overwrite or not name_full in self.save:
+                if (overwrite or
+                    (not name_full in self.save and not name_full in self.data)):
                    self._log("Processing {}".format(name_full))
-                    data = rule.process(arg)
-                    self._save_data(name_full, data, rule.description)
+                    data = rule.process(arg, **kwargs)
+                    self._save_data(name_full, data, rule.description, rule.unit)
                    self._log("Data for {} computed".format(name_full), "SUCCESS")
                    return name_full
                else:
@@ -119,24 +153,43 @@ class BaseProcessor:
            self._log("{} is not valid in this context".format(name_full), "ERROR")


-    def _save_data(self, name_full, data, description):
+    def _save_data(self, name_full, data, description, unit):
        """
        Save data in the HDF5 structure, overwrite if necessary
        """
        if name_full in self.save:
            self.save.remove_node(name_full, recursive=True)

-        if not type(data) == dict:
-            self.save.create_array(os.path.dirname(name_full), os.path.basename(name_full),
-                                   data, description, createparents=True)
-        else:
+        if type(data) == dict:
+            if type(description) == str:
+                self.save.create_group(os.path.dirname(name_full),
+                                       os.path.basename(name_full),
+                                       description, createparents=True)
+            else:
+                self.save.create_group(os.path.dirname(name_full),
+                                       os.path.basename(name_full),
+                                       '', createparents=True)
+
+            if not type(unit) == dict:
+                self.save.get_node(name_full)._v_attrs.unit = unit
+
            for key in data:
                if type(description) == dict:
-                    self.save.create_array(name_full, key,
-                                           data[key], description[key], createparents=True)
+                    if type(unit) == dict:
+                        self._save_data(name_full + '/' + key, data[key], description[key], unit[key])
+                    else:
+                        self._save_data(name_full + '/' + key, data[key], description[key], unit)
                else:
-                    self.save.create_array(name_full, key,
-                                           data[key], description, createparents=True)
+                    if type(unit) == dict:
+                        self._save_data(name_full + '/' + key, data[key], '', unit[key])
+                    else:
+                        self._save_data(name_full + '/' + key, data[key], '', unit)
+        else:
+            self.save.create_array(os.path.dirname(name_full),
+                                   os.path.basename(name_full),
+                                   data, description, createparents=True)
+            self.save.get_node(name_full).attrs.unit = unit
+

    @abstractmethod
    def def_rules(self):
@@ -154,14 +207,18 @@ class PostProcessor(BaseProcessor):

    G = 1. # Gravitational constant

-    def __init__(self, path=None, num=None, path_out=None, pp_params=Params()):
+    def __init__(self, path=None, num=None, path_out=None, pp_params=default_params()):
        """
        Creates the basic structures needed for the outputs
        """

        if not path is None and not num is None:
            # TODO : Make possible to load the HDF5 file even without the original file
-            self.pp_params = pp_params
+
+            if type(pp_params) == str:
+                self.pp_params = load_params(pp_params)
+            else :
+                self.pp_params = pp_params

            # Determining output directory
            if (path_out is None):
@@ -175,6 +232,9 @@ class PostProcessor(BaseProcessor):

            self.filename = (path_out + '/postproc_' +
                             tag_name  + format(num,'05') + '.h5')
+
+            if not os.path.exists(path_out):
+                os.makedirs(path_out)
            self.save = tables.open_file(self.filename, mode="a",
                                         title=os.path.basename(path)+ '_' + format(num,'05'))

@@ -184,6 +244,13 @@ class PostProcessor(BaseProcessor):
            self.num  = num
            self._ro  = pymses.RamsesOutput(path, num, order=pp_params.pymses.order)
            self._amr = self._ro.amr_source(["rho","vel","P"])
+            self.info = self._ro.info.copy()
+
+            # Delete specific info
+            self.info.pop('dom_decomp_Hilbert_keys')
+            self.info.pop('nstep_coarse')
+            self.info.pop('dom_decomp')
+            self.info.pop('time')

            # Density operator
            self._rho_op = ScalarOperator(lambda dset: dset["rho"], self._ro.info["unit_density"])
@@ -196,7 +263,7 @@ class PostProcessor(BaseProcessor):

            # Set the extend of the image
            self._radius = 0.5 / pp_params.out.zoom
-            self._lbox = self._ro.info['boxlen']
+            self._lbox = self.info['boxlen']
            center = pp_params.out.center
            im_extent = [(- self._radius + center[0]) * self._lbox,
                         (  self._radius + center[0]) * self._lbox,
@@ -219,7 +286,6 @@ class PostProcessor(BaseProcessor):
                self.save.root.maps._v_attrs.radius = self._radius
                self.save.root.maps._v_attrs.im_extent = im_extent

-
            # Initialize cameras
            self._cam = dict()
            for ax_los in self._ax_nb : # los = line of sight
@@ -246,12 +312,15 @@ class PostProcessor(BaseProcessor):
        Add additional metadata to the file
        """

-        # Beta for the beta cooling
-        if not (self.pp_params.disk.beta is None or self.pp_params.disk.beta == False):
-            if type(self.pp_params.disk.beta) == int:
-                self.save.root._v_attrs.beta = self.pp_params.disk.beta
-            else:
-                self.save.root._v_attrs.beta = int(self.save.root._v_attrs.run.split('_')[1][4:])
+        # Label of the run in the label.txt file
+        label_filename = self.path + '/' + self.pp_params.input.label_filename
+        if os.path.exists(label_filename):
+            label_file = open(label_filename, 'r')
+            self.label = label_file.readline()
+            label_file.close()
+        else:
+            self.label = self.run
+        self.save.root._v_attrs.label = self.label

    def _coldens(self, ax_los):
         datamap  = self._rt.process(self._cam[ax_los], surf_qty=True)
@@ -482,27 +551,39 @@ class PostProcessor(BaseProcessor):
            # Base rules
            'coldens' : Rule(self, self._coldens, "Column density", '/maps'),
            'rho' : Rule(self, self._rho, "Density slice", '/maps'),
-            'speed_h' : Rule(self, self._speed_h, "Horizontal speed slice wrt the line of sight", '/maps'),
-            'speed_v' : Rule(self, self._speed_v, "Vertical speed slice wrt the line of sight", '/maps'),
-            'T' : Rule(self, self._temperature, "Temperature slice", '/maps', dependencies=['rho']),
-            'levels' : Rule(self, self._levels, "Max level within line of sight", '/maps'),
-            'jeans' : Rule(self, self._jeans, "Jeans lenght slice", '/maps', dependencies=['rho', 'T']),
-            'jeans_ratio' : Rule(self, self._jeans_ratio, "Jeans' lenght divided by the max resolution",
+            'speed_h' : Rule(self, self._speed_h,
+                             "Horizontal speed slice wrt the line of sight", '/maps'),
+            'speed_v' : Rule(self, self._speed_v,
+                             "Vertical speed slice wrt the line of sight", '/maps'),
+            'T' : Rule(self, self._temperature,
+                       "Temperature slice", '/maps', dependencies=['rho']),
+            'levels' : Rule(self, self._levels,
+                            "Max level within line of sight", '/maps'),
+            'jeans' : Rule(self, self._jeans,
+                           "Jeans lenght slice", '/maps', dependencies=['rho', 'T']),
+            'jeans_ratio' : Rule(self, self._jeans_ratio,
+                                 "Jeans' lenght divided by the max resolution",
                                 '/maps', dependencies=['jeans', 'levels']),
-            'Q' : Rule(self, self._toomreQ_disk, "Toomre Q parameter for a Keplerian disk", '/maps',
+            'Q' : Rule(self, self._toomreQ_disk,
+                       "Toomre Q parameter for a Keplerian disk", '/maps',
                       dependencies=['coldens'], args_ok=['z'],
                       is_valid=lambda _: self.pp_params.disk.on),
            'sinks' :  Rule(self, self._sinks, group="/datasets", args_ok=[None],
                            description={'Id': '', 'M':'[Msol]', 'dmf':'[Msol]',
-                                         'x': '', 'y': '', 'z': '', 'vx': '', 'vy': '', 'vz': '',
-                                         'rot_period':'[y]', 'lx':'|l|', 'ly':'|l|', 'lz':'|l|',
-                                         'acc_rate':'[Msol/y]', 'acc_lum':'[Lsol]', 'age':'[y]',
+                                         'x': '', 'y': '', 'z': '',
+                                         'vx': '', 'vy': '', 'vz': '',
+                                         'rot_period':'[y]',
+                                         'lx':'|l|', 'ly':'|l|', 'lz':'|l|',
+                                         'acc_rate':'[Msol/y]', 'acc_lum':'[Lsol]',
+                                         'age':'[y]',
                                         'int_lum':'[Lsol]', 'Teff':'[K]'}),

            # Helpers
-            'radial_bins' : Rule(self, self._radial_bins, "Radial bins", '/radial', args_ok=['z']),
+            'radial_bins' : Rule(self, self._radial_bins,
+                                 "Radial bins", '/radial', args_ok=['z']),
            'rr' : Rule(self, self._rr, "Coordinate map", '/maps', args_ok=['z']),
-            'bins_on_map' : Rule(self, self._bins_on_map, "Convert map coordinates to bins", '/maps',
+            'bins_on_map' : Rule(self, self._bins_on_map,
+                                 "Convert map coordinates to bins", '/maps',
                                 dependencies=['radial_bins', 'rr'], args_ok=['z'])
        }

@@ -542,12 +623,15 @@ class Comparator(BaseProcessor):
    Do comparaison between outputs and runs
    """

-    def __init__(self, path, runs, nums, path_out=None, pp_params=Params()):
+    def __init__(self, path, runs, nums, path_out=None, pp_params=default_params()):
        """
        Creates the basic structures needed for the outputs
        """

-        self.pp_params = pp_params
+        if type(pp_params) == str:
+            self.pp_params = load_params(pp_params)
+        else :
+            self.pp_params = pp_params

        # Determining output directory
        if (path_out is None):
@@ -570,16 +654,31 @@ class Comparator(BaseProcessor):
            for run in runs:
                nums[run] = nums_tmp

+        attrs = dict()
+        self.namelist = dict()
+
        for run in runs:
            path_run = path + '/' + run
            path_out_run = path_out + '/' + run
            self.pp_runs[run] = dict()
            for num in nums[run]:
-                self.pp_runs[run][num] = PostProcessor(path_run, num, path_out=path_out_run, pp_params=pp_params)
+                self.pp_runs[run][num] = PostProcessor(path_run, num, path_out=path_out_run,
+                                                       pp_params=pp_params)
+
+            h5file = tables.open_file(self.pp_runs[run][nums[run][0]].filename, 'r')
+            attrs[run] = h5file.root._v_attrs
+            h5file.close()
+            path_nml = path_run + '/' + self.pp_params.input.nml_filename
+            self.namelist[run] = bunch.bunchify(f90nml.read(path_nml))
+
+
+        self.info = self.pp_runs[runs[0]][nums[runs[0]][0]].info

        # save metadata
        self.save.root._v_attrs.runs = runs
        self.save.root._v_attrs.nums = nums
+        self.save.root._v_attrs.attrs = attrs
+        self.save.root._v_attrs.namelist = bunch.unbunchify(self.namelist)

        # log info
        self.log_id = "[comp {}] ".format(self.pp_params.out.tag)
@@ -593,7 +692,7 @@ class Comparator(BaseProcessor):
        for run in self.save.root._v_attrs.runs:
            series[run] = np.zeros(len(nums[run]))
            for i, num in enumerate(nums[run]):
-                series[run][i] = getter(self.pp_runs[run][num])
+                series[run][i] = getter(run, num)
        return series

    def _comp(self, name, getter):
@@ -602,7 +701,7 @@ class Comparator(BaseProcessor):
        prop = np.zeros(len(runs))
        for i, run in enumerate(runs):
            num = nums[run][0]
-            prop[i] = getter(self.pp_runs[run][num])
+            prop[i] = getter(run, num)
        return prop

    def _time_avg(self, name):
@@ -615,13 +714,21 @@ class Comparator(BaseProcessor):
            std[i] = np.std(serie)
        return {"mean": mean, "std": std}

-    def _get_attr(self, attr_name, pp):
+    def get_attr(self, attr_name, run, num):
+        pp = self.pp_runs[run][num]
        h5file = tables.open_file(pp.filename, "r")
        attr = h5file.root._v_attrs[attr_name]
        h5file.close()
        return attr

-    def _get_pdf_slope(self, name, pp):
+    def get_nml(self, nml_key, run):
+        res = self.namelist[run]
+        for key in nml_key.split('/'):
+            res = res[key]
+        return res
+
+    def get_pdf_slope(self, name, run, num):
+        pp = self.pp_runs[run][num]
        pp.process(['fit_pdf_' + name], ['z'], overwrite=self.overwrite_dep)
        h5file = tables.open_file(pp.filename, "r")
        pdf = h5file.get_node('/hist/pdf_' + name +'_z')
@@ -629,34 +736,136 @@ class Comparator(BaseProcessor):
        h5file.close()
        return slope

-    def _get_sinks_mass(self, pp):
+    def get_sinks_mass(self, run, num):
+        pp = self.pp_runs[run][num]
        pp.process(['sinks'], overwrite=self.overwrite_dep)
        h5file = tables.open_file(pp.filename, "r")
        sinks_mass = h5file.get_node('/datasets/sinks/M').read()
        h5file.close()
        return np.sum(sinks_mass)

+    def _extract_sinks_from_log(self, series, log_filename, run):
+        cmd_grep = "grep 'Number of sink' {} -A 2".format(log_filename)
+        content = os.popen(cmd_grep).readlines()
+        for i in range(0, len(content), 4):
+            series['nb_sink'][run].append(np.int(content[i].split('=')[1]))
+            series['mass_sink'][run].append(np.float(content[i + 1].split('=')[1]))
+            series['time'][run].append(np.float(content[i + 2].split('=')[1]))
+        return series
+
+    def _extract_sfr_from_log(self, series, log_filename, run):
+        cmd_grep = "grep '\[SFR' {} ".format(log_filename)
+        content = os.popen(cmd_grep).readlines()
+        for i in range(0, len(content)):
+            time = np.float(content[i].split(']')[0].split('=')[1].split()[0])
+            sfr = np.float(content[i].split(']')[1].split('=')[1].split()[0])
+            series['time'][run].append(time)
+            series['sfr'][run].append(sfr)
+        return series
+
+    def _from_log(self, keys, extractor):
+        nums = self.save.root._v_attrs.nums
+
+        # Initialize series
+        series = dict()
+        for key in keys:
+            series[key] = dict()
+
+        for run in self.save.root._v_attrs.runs:
+            # Initialize list
+            for key in keys:
+                series[key][run] = []
+
+            # get one preprocessor
+            num = nums[run][0]
+            pp = self.pp_runs[run][num]
+
+            # Get list of run files
+            log_files = (pp.path + '/'
+                         + self.pp_params.input.log_prefix + '*')
+
+            # Parse files
+            for log_filename in glob.glob(log_files):
+                series = extractor(series, log_filename, run)
+
+            # Numpify the lists
+            for key in series:
+                series[key][run] = np.array(series[key][run])
+
+            # Sort the arrays
+            ind_sort = series['time'][run].argsort()
+            for key in series:
+                series[key][run] = series[key][run][ind_sort]
+        return series
+
+
+    def _ssfr_from_mass_sink(self):
+        time_unit = self.save.get_node('/series/sinks_from_log/time')._v_attrs.unit
+        mass_unit = self.save.get_node('/series/sinks_from_log/mass_sink')._v_attrs.unit
+        # Surface of the box in pc^2
+        surface = (self.info['unit_length'].express(cst.pc))**2
+        # WARNING : We do not multiply by boxlen since already done in 'unit_length'
+        ssfr = dict()
+        for run in self.save.root._v_attrs.runs:
+            time = self.save.get_node('/series/sinks_from_log/time/' + run).read()
+            time = time * time_unit.express(cst.year)
+            mass_sink = self.save.get_node('/series/sinks_from_log/mass_sink/' + run).read()
+            mass_sink = mass_sink * mass_unit.express(cst.Msun)
+            sfr = (mass_sink[1:] - mass_sink[:-1]) / (time[1:] - time[:-1])
+            ssfr[run] = np.zeros(len(mass_sink))
+            ssfr[run][1:] = sfr / surface
+        return ssfr
+
+
    def def_rules(self):
        averageables = ['coldens', 'rho', 'T', 'Q']
        self.rules = {
-            'beta' : Rule(self, lambda arg: self._comp("beta", partial(self._get_attr, 'beta')), group='/comp',
-                          args_ok = [None]),
+            # Read from log
+            'sinks_from_log' : Rule(self,
+                                    partial(self._from_log,
+                                                  ['time', 'mass_sink', 'nb_sink'],
+                                                  self._extract_sinks_from_log),
+                                    group='/series', args_ok=[None],
+                                    unit={'time' : self.info['unit_time'],
+                                          'mass_sink' : cst.Msun,
+                                          'nb_sink' : cst.none}),
+            'issfr' : Rule(self,
+                           self._ssfr_from_mass_sink,
+                           group='/series/sinks_from_log', args_ok=[None],
+                           unit=cst.ssfr,
+                           description="Instantaneous surfacic star formation rate",
+                           dependencies=['sinks_from_log']),
+            'sfr_from_log' : Rule(self,
+                                  partial(self._from_log,
+                                                  ['time', 'sfr'],
+                                                  self._extract_sfr_from_log),
+                                  group='/series', args_ok=[None],
+                                  unit={'time' : cst.year,
+                                        'sfr' : cst.ssfr},
+                                  description={'time': 'Time',
+                                               'sfr' : 'Averaged surfacic star formation rate'}),
+            # Read from outputs
+            'time' : Rule(self, partial(self._time_series, "time",
+                                        partial(self.get_attr, 'time')),
+                          group='/series', args_ok=[None]),
            'time_pdf_slope' : Rule(self,
-                                    lambda name: self._time_series("pdf_slope_" + name,
-                                                                   partial(self._get_pdf_slope, name)),
+                                    lambda name:
+                                    self._time_series("pdf_slope_" + name,
+                                                      partial(self.get_pdf_slope,
+                                                              name)),
                                    group='/series', args_ok = averageables),
-            'time_sinks_mass' : Rule(self, partial(self._time_series, "sinks", self._get_sinks_mass),
-                                     group='/series', args_ok=[None]),
-            'time' : Rule(self, partial(self._time_series, "time", partial(self._get_attr, 'time')),
-                                     group='/series', args_ok=[None]),
+            # Averages and run properties
            'avg_pdf_slope' : Rule(self,
-                                   lambda name: self._time_avg("time_pdf_slope_" + name),
+                                   lambda name:
+                                   self._time_avg("time_pdf_slope_" + name),
                                   group='/comp', dependencies=['time_pdf_slope'],
                                   args_ok=averageables,
-                                   description={"mean": "Temporal average", "std": "Standard deviation"})
+                                   description={"mean": "Temporal average",
+                                                "std": "Standard deviation"}),
        }


+
 def get_time(path, num):
    """
    Return the time of the output (code units)