From aeb1f22e2cc0406e2144ec077ea703a36ecf5a20 Mon Sep 17 00:00:00 2001 From: Noe Brucy Date: Sun, 25 Jul 2021 15:31:13 +0200 Subject: [PATCH] add an option to save all important files to path_out --- params.yml | 1 + run_selector.py | 11 +++++++---- snapshotprocessor.py | 15 +++++++++++++-- studyprocessor.py | 34 ++++++++++++++++++++++++++++------ 4 files changed, 49 insertions(+), 12 deletions(-) diff --git a/params.yml b/params.yml index 71fbb9d..a815790 100644 --- a/params.yml +++ b/params.yml @@ -75,6 +75,7 @@ input: # Parameters on how to look for input files (= output from Ramses) ramses_ism : True # If ramses-ism is used out: # Parameters for post processing + copy_info : True # Copy logs, nml and info files to the outdir tag : "" # Tag for the image interactive : False # Interactive mode (keep figures open) save : True # Save the plots on the disk diff --git a/run_selector.py b/run_selector.py index 0546ee7..0e0a25a 100644 --- a/run_selector.py +++ b/run_selector.py @@ -205,8 +205,7 @@ class RunSelector: return selected_runs, selected_nums def load_namelist(self, run): - path_run = self.path_in + "/" + run - path_nml = path_run + "/" + self.nml_filename + path_nml = f"{self.path_in}/{run}/{self.nml_filename}" return NamelistRecursive(f90nml.read(path_nml)) def get_nml_value(self, nml_key, run): @@ -281,8 +280,12 @@ class RunSelector: return runs def load_info(self, run, num): - info_filename = f"{self.path_in}/{run}/output_{num:05}/info_{num:05}.txt" - info = read_ramses_info_file(info_filename) + info_filename_output = f"{self.path_in}/{run}/output_{num:05}/info_{num:05}.txt" + info_filename_folder = f"{self.path_in}/{run}/info/info_{num:05}.txt" + if os.path.exists(info_filename_output): + info = read_ramses_info_file(info_filename_output) + else: + info = read_ramses_info_file(info_filename_folder) return info def get_nums( diff --git a/snapshotprocessor.py b/snapshotprocessor.py index 21477e2..4d20f1e 100644 --- a/snapshotprocessor.py +++ b/snapshotprocessor.py @@ -9,6 +9,8 @@ import pandas as pd from skimage.morphology import medial_axis import os +from distutils.file_util import copy_file + from functools import partial from scipy.stats import linregress @@ -343,8 +345,17 @@ class SnapshotProcessor(HDF5Container): self.info = selector.info[self.run][self.num] self.namelist = selector.namelist[self.run] - self.lbox = self.info["boxlen"] + # Save important info files + if self.params.out.copy_info: + info_src = f"{self.path}/output_{self.num:05}/info_{self.num:05}.txt" + info_dest = f"{self.path_out}/info/info_{self.num:05}.txt" + if os.path.exists(info_src): + os.makedirs(os.path.dirname(info_dest), exist_ok=True) + copy_file(info_src, info_dest, update=1) + + # Get box length + self.lbox = self.info["boxlen"] # Get time self.time = self.info["time"] @@ -352,7 +363,7 @@ class SnapshotProcessor(HDF5Container): self.open() self.save.root._v_attrs.dir = os.path.dirname(path) self.save.root._v_attrs.run = os.path.basename(path) - self.save.root._v_attrs.num = num + self.save.root._v_attrs.num = self.num self.save.root._v_attrs.lbox = self.lbox self.save.root._v_attrs.unit_length = self.info["unit_length"] self.save.root._v_attrs.time = self.time diff --git a/studyprocessor.py b/studyprocessor.py index e2031f5..12672fa 100644 --- a/studyprocessor.py +++ b/studyprocessor.py @@ -1,6 +1,9 @@ # coding: utf-8 + import os +from distutils.file_util import copy_file + import glob import numpy as np from functools import partial @@ -82,6 +85,20 @@ class StudyProcessor(Aggregator, HDF5Container): run0 = self.runs[0] self.info = selector.info[run0][self.nums[run0][0]] self.namelist = selector.namelist + + # Save namelist and logs + if self.params.out.copy_info: + for run in self.runs: + nml_src = f"{self.path}/{run}/{self.params.input.nml_filename}" + nml_dest = f"{self.path_out}/{run}/{self.params.input.nml_filename}" + copy_file(nml_src, nml_dest, update=1) + + logs = self.get_logs(run) + os.makedirs(f"{self.path_out}/{run}/logs", exist_ok=True) + for log in logs: + dest = f"{self.path_out}/{run}/logs/{os.path.basename(log)}" + copy_file(log, dest, update=1) + # log info self.log_id = "[study {}] ".format(self.params.out.tag) @@ -334,6 +351,14 @@ class StudyProcessor(Aggregator, HDF5Container): series["turb_energy"][run].append(np.nan) return series + def get_logs(self, run): + glob_str = f"{self.path}/{run}/{self.params.input.log_prefix}*" + logs = glob.glob(glob_str) + if len(logs) == 0: + glob_str = f"{self.path}/{run}/logs/{self.params.input.log_prefix}*" + logs = glob.glob(glob_str) + return logs + def _from_log(self, keys, extractor): # Initialize series @@ -346,14 +371,11 @@ class StudyProcessor(Aggregator, HDF5Container): for key in keys: series[key][run] = [] - # get one preprocessor - path_run = self.path + "/" + run - - # Get list of run files - log_files = path_run + "/" + self.params.input.log_prefix + "*" + # Get list of log files + log_files = self.get_logs(run) # Parse files - for log_filename in glob.glob(log_files): + for log_filename in log_files: series = extractor(series, log_filename, run) # Numpify the lists