noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add run selector + unit handling + multiprocessing + examples

Browse Source
This commit is contained in:
Noe Brucy
2019-12-01 23:46:17 +01:00
parent e528363938
commit 7d5aa9d911
13 changed files with 2088 additions and 3117 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -0,0 +1 @@
/.ipynb_checkpoints/
auto_pp.py
+38
View File
@@ -0,0 +1,38 @@
from plotter import *
# Turb
in_dir = "/home/nbrucy/simus/ismfeed/turb"
out_dir = "/home/nbrucy/visus/ismfeed/turb"
nml_key = ["turb_params/turb_rms", "turb_params/comp_frac"]
cond1_5 = [("cloud_params/dens0", "=", 1.5), ("turb_params/turb_rms", "!=", 72000)]
cond6 = ("cloud_params/dens0", "=", 6)
nproc = 15
pl = Plotter(
in_dir,
namelist_cond=cond1_5,
in_nums="all",
sort_run_by=nml_key,
path_out=out_dir,
tag="sfr_turb",
)
pl6 = Plotter(
in_dir,
namelist_cond=cond6,
in_nums="all",
sort_run_by=nml_key,
path_out=out_dir,
tag="sfr_turb_n6",
)
pl.pp_params.process.num_process = nproc
pl6.pp_params.process.num_process = nproc
for plotter in [pl, pl6]:
plotter.process(["sigma"])
plotter.process(["coldens_l", "rho_v"], ["x, y, z"])
auto_pp.sh
Executable
+8
View File
@@ -0,0 +1,8 @@
#! /bin/bash
while true; do
date >> ~/pp.log
python auto_pp.py >> ~/pp.log 2>&1
printf "\n\n" >> ~/pp.log
sleep 2000
done
clump.py
-7
View File
@@ -1,7 +0,0 @@
import module_extract as me
import pickle
import matplotlib.pyplot as plt
import numpy as np
from matplotlib.collections import EllipseCollection
disk_postprocess.py
-1680
View File
File diff suppressed because it is too large Load Diff
mypool.py
+63
View File
@@ -0,0 +1,63 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import multiprocessing
# We must import this explicitly, it is not imported by the top-level
# multiprocessing module.
import multiprocessing.pool
import time
from random import randint
class NoDaemonProcess(multiprocessing.Process):
# make 'daemon' attribute always return False
def _get_daemon(self):
return False
def _set_daemon(self, value):
pass
daemon = property(_get_daemon, _set_daemon)
# We sub-class multiprocessing.pool.Pool instead of multiprocessing.Pool
# because the latter is only a wrapper function, not a proper class.
class MyPool(multiprocessing.pool.Pool):
Process = NoDaemonProcess
def sleepawhile(t):
print("Sleeping %i seconds..." % t)
time.sleep(t)
return t
def work(num_procs):
print("Creating %i (daemon) workers and jobs in child." % num_procs)
pool = multiprocessing.Pool(num_procs)
result = pool.map(sleepawhile, [randint(1, 5) for x in range(num_procs)])
# The following is not really needed, since the (daemon) workers of the
# child's pool are killed when the child is terminated, but it's good
# practice to cleanup after ourselves anyway.
pool.close()
pool.join()
return result
def test():
print("Creating 5 (non-daemon) workers and jobs in main process.")
pool = MyPool(5)
result = pool.map(work, [randint(1, 5) for x in range(5)])
pool.close()
pool.join()
print(result)
if __name__ == "__main__":
test()
pipeline.py
+237 -206
View File
@@ -16,163 +16,174 @@ fake_pp = PostProcessor()
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
input_args = parser.add_argument_group('input', "Input selection") input_args = parser.add_argument_group("input", "Input selection")
input_args.add_argument("runs", help='name of runs', nargs='*', input_args.add_argument("runs", help="name of runs", nargs="*", default=[])
default=[])
input_args.add_argument("-ip", "--input_path", input_args.add_argument(
help="specify input directory", "-ip", "--input_path", help="specify input directory", default="/home/nbrucy/simus/"
default="/home/nbrucy/simus/") )
input_args.add_argument("-p", "--project", input_args.add_argument(
help="specify project name (directory within the input directory)", "-p",
default="disk") "--project",
help="specify project name (directory within the input directory)",
default="disk",
)
input_args.add_argument("-c", "--config", input_args.add_argument(
help="Path of a default config file", "-c", "--config", help="Path of a default config file", default=None
default=None) )
input_args.add_argument("-wo", "--which_inputs", input_args.add_argument(
choices=['all', 'id', 'time'], "-wo",
help="Select inputs by time range, id range or all of them", "--which_inputs",
default='all') choices=["all", "id", "time"],
input_args.add_argument("-b", "--begin", help="Select inputs by time range, id range or all of them",
help="id of first input", default="all",
type=int, )
default=1) input_args.add_argument("-b", "--begin", help="id of first input", type=int, default=1)
input_args.add_argument("-e", "--end", input_args.add_argument("-e", "--end", help="id of last input", type=int, default=100)
help="id of last input", input_args.add_argument(
type=int, "-s", "--step", help="step between two input", type=int, default=1
default=100) )
input_args.add_argument("-s", "--step", input_args.add_argument(
help="step between two input", "-tb", "--time_begin", help="time of first input", type=float, default=0.0
type=int, )
default=1) input_args.add_argument(
input_args.add_argument("-tb", "--time_begin", "-te", "--time_end", help="time of last input", type=float, default=6.0
help="time of first input", )
type=float,
default=0.)
input_args.add_argument("-te", "--time_end",
help="time of last input",
type=float,
default=6.)
input_args.add_argument("-w", "--watch", input_args.add_argument(
help="wait and watch for missing inputs", "-w", "--watch", help="wait and watch for missing inputs", action="store_true"
action='store_true') )
input_args.add_argument("--skip", input_args.add_argument("--skip", help="skip failed loadings", action="store_true")
help="skip failed loadings", input_args.add_argument(
action='store_true') "-wt",
input_args.add_argument("-wt", "--waiting_time", "--waiting_time",
help="time between to successive try when watching new inputs (in second)", help="time between to successive try when watching new inputs (in second)",
type=int, type=int,
default=120) default=120,
input_args.add_argument("-af", "--allowed_failures", )
help="number of allowed failures when waiting", input_args.add_argument(
default=30) "-af",
"--allowed_failures",
help="number of allowed failures when waiting",
default=30,
)
output_args = parser.add_argument_group('output', "Output configuration") output_args = parser.add_argument_group("output", "Output configuration")
output_args.add_argument("-op", "--output_path", output_args.add_argument(
help="specify output directory", "-op",
default='/home/nbrucy/visus/') "--output_path",
help="specify output directory",
default="/home/nbrucy/visus/",
)
output_args.add_argument("--tag", output_args.add_argument(
help="Add a special tag on output filemanes", "--tag", help="Add a special tag on output filemanes", default=""
default='') )
output_args.add_argument("--interactive", output_args.add_argument(
help="Interactive mode : do not save data", "--interactive", help="Interactive mode : do not save data", action="store_true"
action='store_true') )
output_args.add_argument("-owr", "--overwrite", output_args.add_argument(
help="Overwrite outputs", "-owr", "--overwrite", help="Overwrite outputs", action="store_true"
action='store_true') )
output_args.add_argument("-owrd", "--overwrite_dependencies", output_args.add_argument(
help="Overwrite outputs for dependencies", "-owrd",
action='store_true', "--overwrite_dependencies",
default=False) help="Overwrite outputs for dependencies",
action="store_true",
default=False,
)
pp_args = parser.add_argument_group('postproc', "Post Processing configuration") pp_args = parser.add_argument_group("postproc", "Post Processing configuration")
pp_args.add_argument("--process", pp_args.add_argument(
help="Individual rules to apply", "--process",
choices=fake_pp.rules.keys(), help="Individual rules to apply",
default=[], choices=fake_pp.rules.keys(),
nargs='*') default=[],
nargs="*",
)
pp_args.add_argument("-pargs", "--process_args", pp_args.add_argument(
help="Args to give to process rules", "-pargs",
default=['x', 'y', 'z'], "--process_args",
nargs='*') help="Args to give to process rules",
default=["x", "y", "z"],
nargs="*",
)
pp_args.add_argument("--compare", pp_args.add_argument(
help="Time and inter run comparaison", "--compare", help="Time and inter run comparaison", default=[], nargs="*"
default=[], )
nargs='*')
pp_args.add_argument("-cargs", "--compare_args", pp_args.add_argument(
help="Args to give to process rules", "-cargs",
default=[None], "--compare_args",
nargs='*') help="Args to give to process rules",
default=[None],
nargs="*",
)
pp_args.add_argument("--plot", pp_args.add_argument("--plot", help="Plot rules", default=[], nargs="*")
help="Plot rules",
default=[],
nargs='*')
pp_args.add_argument("-plargs", "--plot_args", pp_args.add_argument(
help="Args to give to plot rules", "-plargs",
default=[None], "--plot_args",
nargs='*') help="Args to give to plot rules",
default=[None],
nargs="*",
)
pp_args.add_argument("-d", "--disk", pp_args.add_argument(
help="Specify this for disk simulations", "-d", "--disk", help="Specify this for disk simulations", action="store_true"
action='store_true') )
pp_args.add_argument("--fft", pp_args.add_argument(
help="use quick and dirty fft rendering", "--fft", help="use quick and dirty fft rendering", action="store_true"
action='store_true') )
pp_args.add_argument("--zoom", pp_args.add_argument("--zoom", help="zoom", type=float, default=2.0)
help="zoom", pp_args.add_argument(
type=float, "-ms",
default=2.) "--map_size",
pp_args.add_argument("-ms", "--map_size", help="size of the maps created in he map mode (in pixel)",
help="size of the maps created in he map mode (in pixel)", type=int,
type=int, default=1024,
default=1024) )
pp_args.add_argument("--nb_bin", pp_args.add_argument(
help="Number of bins for azimuthal averages", "--nb_bin", help="Number of bins for azimuthal averages", type=int, default=50
type=int, )
default=50) pp_args.add_argument(
pp_args.add_argument("--pdf_nb_bin", "--pdf_nb_bin", help="Number of bins for PDF", type=int, default=50
help="Number of bins for PDF", )
type=int, pp_args.add_argument(
default=50) "--binning",
pp_args.add_argument("--binning", help="Kind of binning (logarithmic or linear)",
help="Kind of binning (logarithmic or linear)", choices=["log", "lin"],
choices=['log', 'lin'], default="log",
default='log') )
plot_args = parser.add_argument_group('plot', "Plot configuration") plot_args = parser.add_argument_group("plot", "Plot configuration")
plot_args.add_argument("--colormap", plot_args.add_argument(
help="Colormap used", "--colormap", help="Colormap used", choices=P.colormaps(), default="plasma"
choices=P.colormaps(), )
default='plasma') plot_args.add_argument(
plot_args.add_argument("--format", "--format",
help="Format of the plot images", help="Format of the plot images",
choices=['png', 'jpeg', 'pdf', 'svg', 'ps'], choices=["png", "jpeg", "pdf", "svg", "ps"],
default='jpeg') default="jpeg",
plot_args.add_argument("--dpi", )
help="Resolution of the plot images", plot_args.add_argument(
type=int, "--dpi", help="Resolution of the plot images", type=int, default=400
default=400) )
plot_args.add_argument("--beamer", plot_args.add_argument("--beamer", help="Beamer mode", action="store_true")
help="Beamer mode",
action='store_true')
args = parser.parse_args() args = parser.parse_args()
@@ -182,16 +193,16 @@ storage_in = args.input_path
storage_out = args.output_path storage_out = args.output_path
if args.config is None: if args.config is None:
pp_params = default_params() pp_params = default_params()
else: else:
pp_params = load_params(args.config) pp_params = load_params(args.config)
pp_params.out.zoom = args.zoom pp_params.out.zoom = args.zoom
pp_params.out.tag = args.tag pp_params.out.tag = args.tag
pp_params.out.map_size = args.map_size pp_params.out.map_size = args.map_size
pp_params.out.interactive = args.interactive pp_params.out.interactive = args.interactive
pp_params.pymses.fft = args.fft pp_params.pymses.fft = args.fft
pp_params.disk.on = args.disk pp_params.disk.on = args.disk
@@ -202,94 +213,114 @@ pp_params.pdf.nb_bin = args.pdf_nb_bin
# extension for out files # extension for out files
P.style.use("seaborn-deep") P.style.use("seaborn-deep")
if args.format == 'pdf': if args.format == "pdf":
P.style.use("~/.config/matplotlib/pdf.mplstyle") P.style.use("~/.config/matplotlib/pdf.mplstyle")
if args.beamer: if args.beamer:
P.rcParams['font.family'] = 'sans-serif' P.rcParams["font.family"] = "sans-serif"
P.rcParams['figure.figsize'] = (7, 4.5) P.rcParams["figure.figsize"] = (7, 4.5)
# Plot properties # Plot properties
P.rcParams['image.cmap'] = args.colormap P.rcParams["image.cmap"] = args.colormap
P.rcParams['savefig.dpi'] = args.dpi P.rcParams["savefig.dpi"] = args.dpi
P.rcParams['lines.linewidth'] = 2 P.rcParams["lines.linewidth"] = 2
P.rcParams['lines.markersize'] = 10 P.rcParams["lines.markersize"] = 10
P.rcParams["errorbar.capsize"] = 4 P.rcParams["errorbar.capsize"] = 4
# List of id that were successfully computed # List of id that were successfully computed
nums_success = dict() nums_success = {}
# Go through all runs # Go through all runs
for run in runs: for run in runs:
path_suffix = project + '/' + run path_suffix = project + "/" + run
path_in = storage_in + path_suffix path_in = storage_in + path_suffix
path_out = storage_out + path_suffix path_out = storage_out + path_suffix
if args.tag == '': if args.tag == "":
tag_run = run tag_run = run
else: else:
tag_run = run + '_' + args.tag tag_run = run + "_" + args.tag
if not os.path.exists(path_out): if not os.path.exists(path_out):
os.makedirs(path_out) os.makedirs(path_out)
try: try:
copy(path_in + '/disk.nml', path_out) copy(path_in + "/disk.nml", path_out)
copy(path_in + '/output_00001/compilation.txt', path_out) copy(path_in + "/output_00001/compilation.txt", path_out)
except: except:
pass pass
nums_success[run] = [] nums_success[run] = []
if args.which_inputs in ['all', 'time'] : if args.which_inputs in ["all", "time"]:
names = glob.glob(path_in + "/output_[0-9][0-9][0-9][0-9][0-9]") names = glob.glob(path_in + "/output_[0-9][0-9][0-9][0-9][0-9]")
nums_all = [int(n.split('/')[-1].split('_')[1]) for n in names] nums_all = [int(n.split("/")[-1].split("_")[1]) for n in names]
nums_all = np.sort(nums_all) nums_all = np.sort(nums_all)
if args.which_inputs == 'all': if args.which_inputs == "all":
nums = nums_all nums = nums_all
else: else:
time = [get_time(path_in, n) for n in nums_all] time = [get_time(path_in, n) for n in nums_all]
nums = [n for i,n in enumerate(nums_all) if time[i] >= args.time_begin nums = [
and time[i] < args.time_end] n
else: for i, n in enumerate(nums_all)
nums = range(args.begin, args.end + 1, args.step) if time[i] >= args.time_begin and time[i] < args.time_end
]
else:
nums = range(args.begin, args.end + 1, args.step)
for num in nums: for num in nums:
failures = 0 failures = 0
success = False success = False
while not success: while not success:
try: try:
if len(args.process) > 0 : if len(args.process) > 0:
pp = PostProcessor(run, num, pp_params=pp_params) pp = PostProcessor(run, num, pp_params=pp_params)
pp.process(args.process, args.process_args, pp.process(
overwrite=args.overwrite, overwrite_dep=args.overwrite_dependencies) args.process,
args.process_args,
overwrite=args.overwrite,
overwrite_dep=args.overwrite_dependencies,
)
# If we are here, success ! # If we are here, success !
success = True success = True
nums_success[run].append(num) nums_success[run].append(num)
except (ValueError, IOError, KeyError) as e: except (ValueError, IOError, KeyError) as e:
print(e) print(e)
if(args.watch and failures < args.allowed_failures): if args.watch and failures < args.allowed_failures:
failures = failures + 1 failures = failures + 1
print("ERROR: Unable to proceed for run {} output {}.\ print(
Trying again in {} s ({} tries remaining)".format(run, num, "ERROR: Unable to proceed for run {} output {}.\
args.waiting_time, args.allowed_failures - failures)) Trying again in {} s ({} tries remaining)".format(
time.sleep(args.waiting_time) run,
elif args.skip: num,
break args.waiting_time,
else: args.allowed_failures - failures,
raise )
)
time.sleep(args.waiting_time)
elif args.skip:
break
else:
raise
path_in = storage_in + project path_in = storage_in + project
path_out = storage_out + project path_out = storage_out + project
if len(args.plot) > 0: if len(args.plot) > 0:
pl = Plotter(path_in, runs, nums_success, path_out=path_out, pp_params=pp_params) pl = Plotter(path_in, runs, nums_success, path_out=path_out, pp_params=pp_params)
pl.process(args.plot, args.plot_args, pl.process(
overwrite=args.overwrite, overwrite_dep=args.overwrite_dependencies) args.plot,
args.plot_args,
overwrite=args.overwrite,
overwrite_dep=args.overwrite_dependencies,
)
if len(args.compare) > 0: if len(args.compare) > 0:
cc = Comparator(path_in, runs, nums_success, path_out=path_out, pp_params=pp_params) cc = Comparator(path_in, runs, nums_success, path_out=path_out, pp_params=pp_params)
cc.process(args.compare, args.compare_args, cc.process(
overwrite=args.overwrite, overwrite_dep=args.overwrite_dependencies) args.compare,
args.compare_args,
overwrite=args.overwrite,
overwrite_dep=args.overwrite_dependencies,
)
pipeline_disk.py
-397
View File
@@ -1,397 +0,0 @@
# coding: utf-8
import os
import glob
from shutil import copy
import argparse
import time
import numpy as np
from functools import reduce
from pp_params import *
from plotter import *
from postprocessor import *
storage_in = '/home/nbrucy/simus/'
storage_out = '/home/nbrucy/visus/'
parser = argparse.ArgumentParser()
parser.add_argument("runs", help='name of runs', nargs='*',
default=[])
parser.add_argument("-wo", "--which_outputs",
choices=['all', 'id', 'time'],
help="Select outputs by time range, id range or all of them",
default='all')
parser.add_argument("-b", "--begin",
help="id of first output",
type=int,
default=1)
parser.add_argument("-e", "--end",
help="id of last output",
type=int,
default=100)
parser.add_argument("-s", "--step",
help="step between two output",
type=int,
default=1)
parser.add_argument("-tb", "--time_begin",
help="time of first output",
type=float,
default=0.)
parser.add_argument("-te", "--time_end",
help="time of last output",
type=float,
default=6.)
parser.add_argument("-p", "--project",
help="specify project name",
default="disk")
parser.add_argument("-fr", "--force_redo",
help="redo plots even if the files already exist",
action='store_true')
parser.add_argument("-w", "--watch",
help="wait and watch for missing outputs",
action='store_true')
parser.add_argument("--skip",
help="skip failed loadings",
action='store_true')
parser.add_argument("-wt", "--waiting_time",
help="time between to successive try when watching new outputs (in second)",
type=int,
default=120)
parser.add_argument("-af", "--allowed_failures",
help="number of allowed failures when waiting",
default=30)
parser.add_argument("-i", "--interactive",
help="Interactive mode",
action='store_true')
parser.add_argument("--tag",
help="Add a special tag on output filemanes",
default='')
parser.add_argument("-d", "--disk",
help="compute specific disk radial analysis",
action='store_true')
parser.add_argument("-pd", "--plot_disk",
help="plot specific disk radial analysis",
action='store_true')
parser.add_argument("-m", "--maps",
help="compute generic maps",
action='store_true')
parser.add_argument("-pm", "--plot_maps",
help="plot generic maps",
action='store_true')
parser.add_argument("-c", "--compare",
help="compare different runs",
action='store_true')
parser.add_argument("-ev", "--evolution",
help="plot evolution of quantities over time",
action='store_true')
parser.add_argument("--pdf",
help="plot pdf of fluctuations of column density",
action='store_true')
parser.add_argument("--pdf2",
help="plot patrick's PDF",
action='store_true')
parser.add_argument("--cpdf",
help="compare pdf",
action='store_true')
parser.add_argument("--clump",
help="Do clump study",
action='store_true')
parser.add_argument("--print_outputs",
help="print names of outputs",
action='store_true')
parser.add_argument("--fft",
help="use quick and dirty fft rendering",
action='store_true')
parser.add_argument("--images", nargs='*',
default=['coldens', 'rho', 'speed', 'Q', 'T'],
choices=['coldens', 'rho', 'speed', 'Q', 'T', 'levels', 'cpu', 'jeans', 'jeans_ratio'])
parser.add_argument("--axes", nargs='*',
default=['x', 'y', 'z'],
choices=['x', 'y', 'z'])
parser.add_argument("--zoom",
help="zoom",
type=float,
default=2.)
parser.add_argument("-ms", "--mapsize",
help="size of the maps created in he map mode (in pixel)",
type=int,
default=1024)
parser.add_argument("--nb_bin",
help="Number of bins for azimuthal averages",
type=int,
default=50)
parser.add_argument("--pdf_nb_bin",
help="Number of bins for PDF",
type=int,
default=50)
parser.add_argument("--binning",
help="Kind of binning (logarithmic or linear)",
choices=['log', 'lin'],
default='log')
parser.add_argument("--rad_ext",
help="Value of the highest bin",
type=float,
default=1.)
parser.add_argument("-x",
help="x position of the central point",
type=float,
default=1.)
parser.add_argument("-y",
help="y position of the central point",
type=float,
default=1.)
parser.add_argument("-z",
help="z position of the central point",
type=float,
default=1.)
parser.add_argument(
"-ta", "--time_avg", help="Plot time averaged comparaison", action="store_true"
)
parser.add_argument("--colormap",
help="Colormap used",
choices=dp.P.colormaps(),
default='plasma')
parser.add_argument("--format",
help="Format of the output",
choices=['png', 'jpeg', 'pdf', 'svg', 'ps'],
default='jpeg')
parser.add_argument("--dpi",
help="Resolution of the output",
type=int,
default=400)
parser.add_argument("--beamer",
help="Beamer mode",
action='store_true')
parser.add_argument("--Q_in_name",
help="Whether to use the name to have Q",
action='store_true')
args = parser.parse_args()
project = args.project
runs = args.runs
first = args.begin
last = args.end
step = args.step
rad = 0.5 / args.zoom
# extension for out files
dp.out_ext = '.' + args.format
dp.P.style.use("seaborn-deep")
if args.format == 'pdf':
dp.P.style.use("~/.config/matplotlib/pdf.mplstyle")
if args.beamer:
dp.P.rcParams['font.family'] = 'sans-serif'
dp.P.rcParams['figure.figsize'] = (7, 4.5)
# Plot properties
dp.P.rcParams['image.cmap'] = args.colormap
dp.P.rcParams['savefig.dpi'] = args.dpi
dp.P.rcParams['lines.linewidth'] = 2
dp.P.rcParams['lines.markersize'] = 10
dp.P.rcParams["errorbar.capsize"] = 4
me.P = dp.P
# List of id that were successfully computed
run_succeded = {}
# Care for dependencies
images = args.images
if 'jeans_ratio' in images and not 'jeans' in images :
images = ['jeans'] + images
if 'jeans_ratio' in images and not 'levels' in images :
images.append('levels')
if 'jeans' in images and not 'T' in images :
images.append('T')
if ('T' in images or 'jeans' in images) and not 'rho' in images :
images.append('rho')
if 'Q' in images and not 'coldens' in images :
images.append('coldens')
# Go through all runs
for run in runs:
path_suffix = project + '/' + run
path_in = storage_in + path_suffix
path_out = storage_out + path_suffix
if args.tag == '':
tag_run = run
else:
tag_run = run + '_' + args.tag
if not os.path.exists(path_out):
os.makedirs(path_out)
try:
copy(path_in + '/disk.nml', path_out)
copy(path_in + '/output_00001/compilation.txt', path_out)
except:
pass
run_succeded[run] = []
if args.which_outputs in ['all', 'time'] :
names = glob.glob(path_in + "/output_[0-9][0-9][0-9][0-9][0-9]")
nums_all = [int(n.split('/')[-1].split('_')[1]) for n in names]
nums_all = np.sort(nums_all)
if args.which_outputs == 'all':
nums = nums_all
else:
time = [dp.get_time(path_in, n) for n in nums_all]
nums = [n for i,n in enumerate(nums_all) if time[i] >= args.time_begin and time[i] < args.time_end]
else:
nums = range(first, last + 1, step)
for num in nums:
failures = 0
success = False
while not success:
try:
maps_disk = None
if args.print_outputs:
print("[{}, {}]".format(run, num))
if args.maps:
print("[{}, {}] computing maps".format(run, num))
maps_disk = dp.compute_image_data(path_in, num,
radius=rad,
path_out=path_out,
tag=tag_run,
map_size=args.mapsize,
force=args.force_redo,
axes_los=args.axes,
images=images,
pos_star=np.array([args.x, args.y, args.z]),
fft=args.fft)
print("[{}, {}] maps computed".format(run, num))
if args.plot_maps:
print("[{}, {}] plotting maps".format(run, num))
maps_disk = dp.plot_maps(path_out, num,
maps_disk=maps_disk,
axes_los=args.axes,
images=images,
tag=tag_run,
force=args.force_redo,
interactive=args.interactive,
put_title=(not args.beamer))
print("[{}, {}] maps plotted".format(run, num))
if args.disk:
print("[{}, {}] computing disk props".format(run, num))
dp.disk_prop(path_in, num, path_out=path_out,
nb_bin=args.nb_bin,
binning=args.binning,
rad_ext=args.rad_ext,
force=args.force_redo,
pos_star=np.array([args.x, args.y, args.z]))
print("[{}, {}] disk_props computed".format(run, num))
if args.plot_disk:
print("[{}, {}] plotting disk props".format(run, num))
dp.plot_disk_prop(path_out, num, tag=tag_run,
force=args.force_redo,
interactive=args.interactive,
put_title=(not args.beamer))
print("[{}, {}] disk props plotted".format(run, num))
if args.pdf:
print("[{}, {}] computing pdf #1".format(run, num))
dp.disk_pdf(path_out, num, maps_disk,
pos_star=np.array([args.x, args.y, args.z]),
force=args.force_redo,
tag=tag_run,
nb_bin_hist=args.pdf_nb_bin,
interactive=args.interactive,
put_title=(not args.beamer))
print("[{}, {}] pdf #1 computed".format(run, num))
if args.pdf2:
print("[{}, {}] computing pdf #2".format(run, num))
me.get_pdf(path_in, num, path_out=path_out,
force=args.force_redo,
tag=tag_run)
print("[{}, {}] pdf #2 computed".format(run, num))
if args.clump:
print("[{}, {}] computing clumps".format(run, num))
dp.clump_study(path_in, num, path_out, tag_run)
print("[{}, {}] clumps computed".format(run, num))
if args.interactive:
print("[{}, {}] Interactive session".format(run, num))
maps_disk = dp.interactive_plots(args, path_out, num, tag=tag_run)
# If we are here, success !
success = True
run_succeded[run].append(num)
except (ValueError, IOError, KeyError) as e:
print(e)
if(args.watch and failures < args.allowed_failures):
failures = failures + 1
print("Unable to proceed for run {} output {}. Trying again in {} s ({} tries remaining)".format(run, num, args.waiting_time, args.allowed_failures - failures))
time.sleep(args.waiting_time)
elif args.skip:
break
else:
raise
if args.evolution:
print("[{}] plotting evolution".format(run))
dp.evolution(path_out, run_succeded[run],
force=args.force_redo,
tag=args.tag,
interactive=args.interactive,
pdf=args.pdf or args.cpdf)
print("[{}] evolution plotted".format(run))
if args.compare:
path_suffix = project
path = storage_out + path_suffix
if (args.time_avg):
dp.compare(path, runs, run_succeded,
path_out=path + '/comp',
force=args.force_redo,
interactive=args.interactive,
Q_in_name=args.Q_in_name,
tag=args.tag,
pdf=args.pdf or args.cpdf)
else:
for i in range(first, last + 1, step):
# Select usable runs
runs_ok = [run for run in runs if i in run_succeded[run]]
try:
if len(runs_ok) > 1:
dp.compare(path, runs_ok, i,
path_out=path + '/comp',
force=args.force_redo,
interactive=args.interactive,
Q_in_name=args.Q_in_name,
tag=args.tag,
pdf=args.pdf or args.cpdf)
except (KeyError, IOError) as e:
print(e)
if (args.skip):
pass
else:
raise
plotter.py
+576 -344
View File
File diff suppressed because it is too large Load Diff
postprocessor.py
+918 -478
View File
File diff suppressed because it is too large Load Diff
pp_params.py
+10 -5
View File
@@ -11,15 +11,19 @@ _dir_path = os.path.dirname(os.path.realpath(__file__))
# Add support for '1e3' kind of float # Add support for '1e3' kind of float
_loader = yaml.SafeLoader _loader = yaml.SafeLoader
_loader.add_implicit_resolver( _loader.add_implicit_resolver(
u'tag:yaml.org,2002:float', u"tag:yaml.org,2002:float",
re.compile(u'''^(?: re.compile(
u"""^(?:
[-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)? [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
|[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+) |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
|\\.[0-9_]+(?:[eE][-+][0-9]+)? |\\.[0-9_]+(?:[eE][-+][0-9]+)?
|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]* |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
|[-+]?\\.(?:inf|Inf|INF) |[-+]?\\.(?:inf|Inf|INF)
|\\.(?:nan|NaN|NAN))$''', re.X), |\\.(?:nan|NaN|NAN))$""",
list(u'-+0123456789.')) re.X,
),
list(u"-+0123456789."),
)
def load_params(filename): def load_params(filename):
@@ -27,5 +31,6 @@ def load_params(filename):
para_disk = yaml.load(f, Loader=_loader) para_disk = yaml.load(f, Loader=_loader)
return bunch.bunchify(para_disk) return bunch.bunchify(para_disk)
def default_params(): def default_params():
return load_params(_dir_path + '/pp_params.yml') return load_params(_dir_path + "/pp_params.yml")
run_selector.py
+166
View File
@@ -0,0 +1,166 @@
# coding: utf-8
import os
import glob
from functools import partial
from pp_params import *
import f90nml
class RunSelector:
def __init__(
self,
path_in,
in_runs=None,
in_nums="all",
pp_params=default_params(),
number_run="[0-9]*_",
name_run="*",
namelist_cond={},
sort_run_by=None,
time_min=None,
time_max=None,
):
self.path_in = path_in
self.pp_params = pp_params
self.namelist = {}
self.runs = self.get_runs(
in_runs, number_run, name_run, namelist_cond, sort_run_by
)
self.info = {}
for run in self.runs:
self.info[run] = {}
self.nums = {}
if not type(in_nums) == dict:
nums_temp = in_nums
in_nums = {}
for run in self.runs:
in_nums[run] = nums_temp
for run in self.runs:
self.nums[run] = self.get_nums(run, in_nums[run], time_min, time_max)
def load_namelist(self, run):
path_run = self.path_in + "/" + run
path_nml = path_run + "/" + self.pp_params.input.nml_filename
return f90nml.read(path_nml)
def get_nml_value(self, nml_key, run):
res = self.namelist[run]
for key in nml_key.split("/"):
res = res[key]
return res
def get_runs(
self,
in_runs=None,
number_run="[0-9]*_",
name_run="*",
namelist_cond={},
sort_run_by=None,
):
def try_load_nml(run):
try:
self.namelist[run] = self.load_namelist(run)
success = True
except IOError:
success = False
return success
runs = map(
os.path.basename, glob.glob(self.path_in + "/" + number_run + name_run)
)
if not in_runs is None:
runs = filter(lambda n: n in runs, in_runs)
runs = filter(try_load_nml, runs)
if type(namelist_cond) == tuple:
namelist_cond = [namelist_cond]
for (nml_key, operator, operand) in namelist_cond:
value = {}
for run in runs:
value[run] = self.get_nml_value(nml_key, run)
if operator == "=":
runs = filter(lambda r: value[r] == operand, runs)
if operator == "!=":
runs = filter(lambda r: not value[r] == operand, runs)
elif operator == ">":
runs = filter(lambda r: value[r] > operand, runs)
elif operator == "<":
runs = filter(lambda r: value[r] < operand, runs)
elif operator == "in":
runs = filter(lambda r: value[r] in operand, runs)
# Sort by the value in the namelist of sort_run_by
if not sort_run_by is None:
if type(sort_run_by) == str:
sort_run_by = [sort_run_by]
for nml_key in reversed(sort_run_by):
runs.sort(key=partial(self.get_nml_value, nml_key))
return runs
def load_info(self, run, num):
info_file = open(
self.path_in
+ "/"
+ run
+ "/"
+ "output_"
+ str(num).zfill(5)
+ "/"
+ "info_"
+ str(num).zfill(5)
+ ".txt",
"r",
)
info = {}
for line in info_file.readlines():
parsed = yaml.safe_load(line.replace("=", ":"))
if type(parsed) == dict:
info.update(parsed)
info_file.close()
return info
def get_nums(self, run, in_nums=None, time_min=None, time_max=None):
def try_load_info(num):
try:
self.info[run][num] = self.load_info(run, num)
success = True
except IOError:
success = False
return success
names = glob.glob(
self.path_in + "/" + run + "/output_[0-9][0-9][0-9][0-9][0-9]"
)
nums = map(lambda n: int(n.split("/")[-1].split("_")[1]), names)
if type(in_nums) == list:
nums = filter(lambda n: n in nums, in_nums)
nums = np.sort(nums)
if in_nums == "first":
i = 0
while i < len(nums) and not try_load_info(nums[i]):
i = i + 1
nums = [nums[i]]
elif in_nums == "last":
i = len(nums) - 1
while i >= 0 and not try_load_info(nums[i]):
i = i - 1
nums = [nums[i]]
else:
nums = filter(try_load_info, nums)
if not time_min is None:
nums = filter(lambda n: self.info[run][n]["time"] >= time_min, nums)
if not time_max is None:
nums = filter(lambda n: self.info[run][n]["time"] <= time_max, nums)
return nums
units.py
+71
View File
@@ -0,0 +1,71 @@
# coding: utf-8
import pymses.utils.constants as cst
def parse_exp_unit(u):
splitted = u.split("^")
name_u = cst.Unit.from_name(splitted[0]).latex
exp = ""
if len(splitted) > 1:
exp = "$^{" + str(splitted[1]) + "}$"
return name_u + exp
def convert_exp(number):
splitted = "{:.4g}".format(number).split("e")
if len(splitted) == 1:
return "${}$".format(splitted[0])
else:
coeff = float(splitted[0])
exp = int(splitted[1])
exp_str = "10^{" + str(exp) + "}"
if coeff == 1.0:
return "$" + exp_str + "$"
else:
return "$" + str(coeff) + "\\times" + exp_str + "$"
def unit_str(unit, base=None, prefix=""):
if unit == cst.none:
return ""
elif not base is None:
coeff = unit.express(base)
return unit_str(base, prefix=convert_exp(coeff) + " ")
elif len(unit.latex) > 0:
if ("." in unit.latex or "^" in unit.latex) and not "$" in unit.latex:
base_str = ".".join(map(parse_exp_unit, unit.name.split(".")))
return r" [{}{}]".format(prefix, base_str)
else:
return r" [{}{}]".format(prefix, unit.latex)
elif len(unit.name) > 0:
try:
base_str = ".".join(map(parse_exp_unit, unit.name.split(".")))
u_str = r" [{}{}]".format(prefix, base_str)
except:
u_str = r" [{}{}]".format(prefix, unit.name)
return u_str
else:
base_str = ".".join(
map(parse_exp_unit, unit._decompose_base_units().split("."))
)
return r" [{}{} {}]".format(prefix, unit.coeff, base_str)
cst.coldens = cst.create_unit(
"Msun.pc^-2", base_unit=cst.Msun / cst.pc ** 2, descr="Column density"
)
cst.km_s = cst.create_unit("km.s^-1", base_unit=cst.km / cst.s, descr="Speed")
cst.Msun_pc3 = cst.create_unit(
"Msun.pc^-3", base_unit=cst.Msun / cst.pc ** 3, descr="Density"
)
cst.kg_m3 = cst.create_unit("kg.m^-3", base_unit=cst.kg / cst.m ** 3, descr="Density")
cst.ssfr = cst.create_unit(
"Msun.yr^-1.pc^-2",
base_unit=cst.Msun / cst.year / cst.pc ** 2,
descr="Surfacic SFR",
latex="M$_{\odot}$.yr$^{-1}$.p$c^{-2}$",
)