noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

extension subfolders

Browse Source
This commit is contained in:
Noe Brucy
2021-07-24 20:53:21 +02:00
parent 35e5a767cb
commit 35cf8ff401
4 changed files with 37 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
params.yml
+8 -6
View File
@@ -74,17 +74,19 @@ input: # Parameters on how to look for input files (= output from Ramses)
ramses_ism : True # If ramses-ism is used ramses_ism : True # If ramses-ism is used
out: # Parameters for post processing out: # Parameters for post processing
tag : "" # Tag for the image tag : "" # Tag for the image
interactive : False # Interactive mode (keep figures open) interactive : False # Interactive mode (keep figures open)
save : True # Save the plots on the disk save : True # Save the plots on the disk
ext : '.jpeg' # extension for plots ext : '.jpeg' # extension for plots
fmt : "" # Format of the output filename for plots ext_subfolder : True # Separate production by extension in subfolders
fmt : "" # Format of the output filename for plots
# The following keys are accepted # The following keys are accepted
# {out} : The output directory (where hdf5 files are also stored) # {out} : The output directory (where hdf5 files are also stored)
# {run} : Name of the relevant run # {run} : Name of the relevant run
# {num} : Name of the input file (from Ramses) # {num} : Name of the input file (from Ramses)
# {ext} : Extension defined above # {ext} : Extension defined above (with the dot)
# {name} : Name of the rule # {name} : Name of the rule
# {subfolder} : replaced by /{ext[1:]}/ if ext_subfolder is true
# {tag} : Tag defined above # {tag} : Tag defined above
# {nml[nml_key]} : The value of nml_key in the namelist (ex: {nml[amr_params/levelmin]}) # {nml[nml_key]} : The value of nml_key in the namelist (ex: {nml[amr_params/levelmin]})
plotter.py
+14 -7
View File
@@ -377,8 +377,7 @@ class Plotter(Aggregator, BaseProcessor):
if movie: if movie:
for run in runs: for run in runs:
clip = ImageSequenceClip(filenames[run], fps=movie_fps) clip = ImageSequenceClip(filenames[run], fps=movie_fps)
movie_filename = self._find_filename(name_full, run=run) movie_filename = self._find_filename(name_full, run=run, ext=".mp4")
movie_filename = os.path.splitext(movie_filename)[0] + ".mp4"
clip.write_videofile(movie_filename) clip.write_videofile(movie_filename)
return datafiles return datafiles
@@ -408,22 +407,26 @@ class Plotter(Aggregator, BaseProcessor):
else: else:
self._log("Plot {} is already done, skipping...".format(plot_filename)) self._log("Plot {} is already done, skipping...".format(plot_filename))
def _find_filename(self, name_full, run=None, num=None, fmt=None): def _find_filename(self, name_full, run=None, num=None, fmt=None, ext=None):
""" """
Determine a filename based on rule name, run, output and parameters Determine a filename based on rule name, run, output and parameters
""" """
tag_name = self.params.out.tag tag_name = self.params.out.tag
if self.params.out.ext_subfolder:
subfolder = "/{ext[1:]}/"
else:
subfolder = ""
if fmt is None and self.params.out.fmt == "": if fmt is None and self.params.out.fmt == "":
if not self.params.out.tag == "": if not self.params.out.tag == "":
tag_name = "_" + tag_name tag_name = "_" + tag_name
if run is not None and num is not None: if run is not None and num is not None:
fmt = "{out}/{run}/{name}{tag}_{run}_{num:05}{ext}" fmt = "{out}/{run}/{subfolder}{name}{tag}_{run}_{num:05}{ext}"
elif run is not None: elif run is not None:
fmt = "{out}/{run}/{name}{tag}_{run}{ext}" fmt = "{out}/{run}/{subfolder}{name}{tag}_{run}{ext}"
else: else:
fmt = "{out}/{name}{tag}{ext}" fmt = "{out}/{name}{subfolder}{tag}{ext}"
elif fmt is None: elif fmt is None:
fmt = self.params.out.fmt fmt = self.params.out.fmt
@@ -431,6 +434,9 @@ class Plotter(Aggregator, BaseProcessor):
if run is not None: if run is not None:
nml = self.study.namelist[run] nml = self.study.namelist[run]
if ext is None:
ext = self.params.out.ext
return fmt.format( return fmt.format(
run=run, run=run,
name=name_full, name=name_full,
@@ -438,7 +444,8 @@ class Plotter(Aggregator, BaseProcessor):
num=num, num=num,
nml=nml, nml=nml,
out=self.path_out, out=self.path_out,
ext=self.params.out.ext, ext=ext,
subfolder=subfolder,
) )
def get_label_run(self, run, label=None, nml_key=None, time=None): def get_label_run(self, run, label=None, nml_key=None, time=None):
snapshotprocessor.py
+10 -18
View File
@@ -312,24 +312,17 @@ class SnapshotProcessor(HDF5Container):
else: else:
tag_name = "" tag_name = ""
self.filename = ( if self.params.out.subfolder_ext:
self.path_out + "/postproc_" + tag_name + format(num, "05") + ".h5" subfolder = "/h5/"
) else:
subfolder = ""
self.cells_filename = (
self.path_out + "/cells_" + tag_name + format(num, "05") + ".h5"
)
self.parts_filename = (
self.path_out + "/parts_" + tag_name + format(num, "05") + ".h5"
)
self.pspec_filename = (
self.path_out + "/pspec_" + tag_name + format(num, "05") + ".h5"
)
self.filename = f"{self.path_out}/{subfolder}postproc_{tag_name}{num:05}.h5"
self.cells_filename = f"{self.path_out}/{subfolder}cells_{tag_name}{num:05}.h5"
self.parts_filename = f"{self.path_out}/{subfolder}parts_{tag_name}{num:05}.h5"
self.pspec_filename = f"{self.path_out}/{subfolder}pspec_{tag_name}{num:05}.h5"
self.filaments_filename = ( self.filaments_filename = (
self.path_out + "/filaments_" + tag_name + format(num, "05") + ".pickle" f"{self.path_out}/{subfolder}filaments_{tag_name}{num:05}.h5"
) )
if not os.path.exists(self.path_out): if not os.path.exists(self.path_out):
@@ -507,7 +500,7 @@ class SnapshotProcessor(HDF5Container):
if len(data[key] > 0): if len(data[key] > 0):
hdf5.create_array( hdf5.create_array(
"/data", key, data[key], "", createparents=True "/data", key, data[key], "", createparents=True
) )
finally: finally:
hdf5.close() hdf5.close()
return data return data
@@ -1374,7 +1367,6 @@ class SnapshotProcessor(HDF5Container):
df = pd.read_csv(csv_name, header=None, names=header) df = pd.read_csv(csv_name, header=None, names=header)
return {key: df[key].values for key in df} return {key: df[key].values for key in df}
def _pspec(self, **kwargs): def _pspec(self, **kwargs):
outfile = self.pspec_filename outfile = self.pspec_filename
pspec_new.pspec(repo=self.path, iouts=[self.num], outfile=outfile, **kwargs) pspec_new.pspec(repo=self.path, iouts=[self.num], outfile=outfile, **kwargs)
studyprocessor.py
+5 -3
View File
@@ -29,7 +29,7 @@ class StudyProcessor(Aggregator, HDF5Container):
selector=None, selector=None,
tag=None, tag=None,
unit_time=U.year, unit_time=U.year,
**kwargs **kwargs,
): ):
""" """
Creates the basic structures needed for the outputs Creates the basic structures needed for the outputs
@@ -43,8 +43,10 @@ class StudyProcessor(Aggregator, HDF5Container):
else: else:
tag_name = "" tag_name = ""
self.filename = self.path_out + "/comp" + tag_name + ".h5" if self.params.out.ext_subfolder:
self.filename = f"{self.path_out}/study{tag_name}.h5"
else:
self.filename = f"{self.path_out}/h5/study{tag_name}.h5"
# Select runs # Select runs
if selector is None: if selector is None:
selector = RunSelector( selector = RunSelector(