noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Lot of new things

Browse Source
This commit is contained in:
Noe Brucy
2019-07-18 11:33:15 +02:00
parent 8af0d5574d
commit 1211dbb7b8
3 changed files with 731 additions and 626 deletions
Download Patch File Download Diff File Expand all files Collapse all files
clump.py
-23
View File
@@ -4,27 +4,4 @@ import matplotlib.pyplot as plt
import numpy as np import numpy as np
from matplotlib.collections import EllipseCollection from matplotlib.collections import EllipseCollection
me.make_clump_hop('/home/nbrucy/simus/conv_disk/73_beta3_jeansrefine', 40, 'test_hop', 3, 12,
[0.5, 0.5, 0.5], 1., path_out='/home/nbrucy/visus/conv_disk/73_beta3_jeansrefine/', path_hop=
'./', force=True)
me.clump_properties('test_hop', '/home/nbrucy/simus/conv_disk/73_beta3_jeansrefine', 40, path_out='/home/nbrucy/visus/conv_disk/73_beta3_jeansrefine/', gcomp=False)
f = open('/home/nbrucy/visus/conv_disk/73_beta3_jeansrefine/test_hop_prop_struct.save')
a = pickle.load(f)
f.close()
mask = a['n_max'] > 10**4
p = a['pos_n_max'][mask]
c = a['n_max'][mask]
f = open('/home/nbrucy/visus/conv_disk/73_beta3_jeansrefine/maps_disk_73_beta3_jeansrefine_00040.save')
m = pickle.load(f)
plt.imshow(np.log10(m['coldens_z']), extent=m['im_extent'], cmap='plasma')
plt.scatter(p[:, 0], 2 - p[:, 1], c = c)
fig, ax = plt.subplots(1, 1)
offsets = list(zip(p[:, 0], 2 - p[:, 1]))
size = a['size_iner2'][mask]
ax.add_collection(EllipseCollection(widths=size, heights=size, angles=0, units='xy',
facecolors=plt.cm.hsv(a['n_max'][mask]),
offsets=offsets, transOffset=ax.transData))
disk_postprocess.py
+421 -356
View File
File diff suppressed because it is too large Load Diff
pipeline_disk.py
+218 -155
View File
@@ -1,11 +1,13 @@
# coding: utf-8 # coding: utf-8
import os import os
import glob
from shutil import copy from shutil import copy
import argparse import argparse
import time import time
import numpy as np import numpy as np
import disk_postprocess as dp import disk_postprocess as dp
import module_extract as me
from functools import reduce from functools import reduce
storage_in = "/home/nbrucy/simus/" storage_in = "/home/nbrucy/simus/"
@@ -13,57 +15,99 @@ storage_out = "/home/nbrucy/visus/"
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("runs", help="name of runs", nargs="*", default=["015_iso"]) parser.add_argument("runs", help='name of runs', nargs='*',
parser.add_argument("-b", "--begin", help="id of first output", type=int, default=1) default=[])
parser.add_argument("-e", "--end", help="id of last output", type=int, default=100)
parser.add_argument("-s", "--step", help="step between two output", type=int, default=1)
parser.add_argument("-p", "--project", help="specify project name", default="disk") parser.add_argument("-wo", "--which_outputs",
parser.add_argument( choices=['all', 'id', 'time'],
"-fr", help="Select outputs by time range, id range or all of them",
"--force_redo", default='all')
parser.add_argument("-b", "--begin",
help="id of first output",
type=int,
default=1)
parser.add_argument("-e", "--end",
help="id of last output",
type=int,
default=100)
parser.add_argument("-s", "--step",
help="step between two output",
type=int,
default=1)
parser.add_argument("-tb", "--time_begin",
help="time of first output",
type=float,
default=0.)
parser.add_argument("-te", "--time_end",
help="time of last output",
type=float,
default=6.)
parser.add_argument("-p", "--project",
help="specify project name",
default="disk")
parser.add_argument("-fr", "--force_redo",
help="redo plots even if the files already exist", help="redo plots even if the files already exist",
action="store_true", action='store_true')
) parser.add_argument("-w", "--watch",
parser.add_argument( help="wait and watch for missing outputs",
"-w", "--watch", help="wait and watch for missing outputs", action="store_true" action='store_true')
) parser.add_argument("--skip",
parser.add_argument("--skip", help="skip failed loadings", action="store_true") help="skip failed loadings",
parser.add_argument( action='store_true')
"-wt", parser.add_argument("-wt", "--waiting_time",
"--waiting_time",
help="time between to successive try when watching new outputs (in second)", help="time between to successive try when watching new outputs (in second)",
type=int, type=int,
default=120, default=120)
) parser.add_argument("-af", "--allowed_failures",
parser.add_argument(
"-af",
"--allowed_failures",
help="number of allowed failures when waiting", help="number of allowed failures when waiting",
default=30, default=30)
) parser.add_argument("-i", "--interactive",
parser.add_argument("-i", "--interactive", help="Interactive mode", action="store_true") help="Interactive mode",
action='store_true')
parser.add_argument( parser.add_argument("--tag",
"-d", "--disk", help="compute specific disk radial analysis", action="store_true" help="Add a special tag on output filemanes",
) default='')
parser.add_argument(
"-pd", "--plot_disk", help="plot specific disk radial analysis", action="store_true"
)
parser.add_argument("-m", "--maps", help="compute generic maps", action="store_true") parser.add_argument("-d", "--disk",
parser.add_argument("-pm", "--plot_maps", help="plot generic maps", action="store_true") help="compute specific disk radial analysis",
parser.add_argument( action='store_true')
"-c", "--compare", help="compare different runs", action="store_true" parser.add_argument("-pd", "--plot_disk",
) help="plot specific disk radial analysis",
parser.add_argument( action='store_true')
"-ev", parser.add_argument("-m", "--maps",
"--evolution", help="compute generic maps",
action='store_true')
parser.add_argument("-pm", "--plot_maps",
help="plot generic maps",
action='store_true')
parser.add_argument("-c", "--compare",
help="compare different runs",
action='store_true')
parser.add_argument("-ev", "--evolution",
help="plot evolution of quantities over time", help="plot evolution of quantities over time",
action="store_true", action='store_true')
) parser.add_argument("--pdf",
parser.add_argument( help="plot pdf of fluctuations of column density",
"--pdf", help="plot pdf of fluctuations of column density", action="store_true" action='store_true')
) parser.add_argument("--pdf2",
help="plot patrick's PDF",
action='store_true')
parser.add_argument("--cpdf",
help="compare pdf",
action='store_true')
parser.add_argument("--clump",
help="Do clump study",
action='store_true')
parser.add_argument("--print_outputs",
help="print names of outputs",
action='store_true')
parser.add_argument( parser.add_argument(
"--fft", help="use quick and dirty fft rendering", action="store_true" "--fft", help="use quick and dirty fft rendering", action="store_true"
@@ -87,44 +131,59 @@ parser.add_argument(
) )
parser.add_argument( parser.add_argument("--nb_bin",
"--nb_bin", help="Number of bins for azimuthal averages", type=int, default=50 help="Number of bins for azimuthal averages",
) type=int,
parser.add_argument( default=50)
"--binning", parser.add_argument("--pdf_nb_bin",
help="Number of bins for PDF",
type=int,
default=50)
parser.add_argument("--binning",
help="Kind of binning (logarithmic or linear)", help="Kind of binning (logarithmic or linear)",
choices=["log", "lin"], choices=['log', 'lin'],
default="log", default='log')
) parser.add_argument("--rad_ext",
parser.add_argument( help="Value of the highest bin",
"--rad_ext", help="Value of the highest bin", type=float, default=1.0 type=float,
) default=1.)
parser.add_argument( parser.add_argument("-x",
"-x", help="x position of the central point", type=float, default=1.0 help="x position of the central point",
) type=float,
parser.add_argument( default=1.)
"-y", help="y position of the central point", type=float, default=1.0 parser.add_argument("-y",
) help="y position of the central point",
parser.add_argument( type=float,
"-z", help="z position of the central point", type=float, default=1.0 default=1.)
) parser.add_argument("-z",
help="z position of the central point",
type=float,
default=1.)
parser.add_argument( parser.add_argument(
"-ta", "--time_avg", help="Plot time averaged comparaison", action="store_true" "-ta", "--time_avg", help="Plot time averaged comparaison", action="store_true"
) )
parser.add_argument( parser.add_argument("--colormap",
"--colormap", help="Colormap used", choices=dp.P.colormaps(), default="plasma" help="Colormap used",
) choices=dp.P.colormaps(),
parser.add_argument( default='plasma')
"--format", parser.add_argument("--format",
help="Format of the output", help="Format of the output",
choices=["png", "jpeg", "pdf", "svg", "ps"], choices=['png', 'jpeg', 'pdf', 'svg', 'ps'],
default="jpeg", default='jpeg')
) parser.add_argument("--dpi",
parser.add_argument("--dpi", help="Resolution of the output", type=int, default=400) help="Resolution of the output",
parser.add_argument("--beamer", help="Beamer mode", action="store_true") type=int,
default=400)
parser.add_argument("--beamer",
help="Beamer mode",
action='store_true')
parser.add_argument("--Q_in_name",
help="Whether to use the name to have Q",
action='store_true')
args = parser.parse_args() args = parser.parse_args()
@@ -139,17 +198,25 @@ step = args.step
rad = 0.5 / args.zoom rad = 0.5 / args.zoom
# extension for out files # extension for out files
dp.out_ext = "." + args.format dp.out_ext = '.' + args.format
if format == "pdf": dp.P.style.use("seaborn-deep")
dp.P.style.use("pdf") if args.format == 'pdf':
dp.P.style.use("~/.config/matplotlib/pdf.mplstyle")
if args.beamer: if args.beamer:
dp.P.rcParams["font.family"] = "sans-serif" dp.P.rcParams['font.family'] = 'sans-serif'
dp.P.rcParams["figure.figsize"] = (5, 3.5) dp.P.rcParams['figure.figsize'] = (7, 4.5)
# Plot properties # Plot properties
dp.P.rcParams["image.cmap"] = args.colormap dp.P.rcParams['image.cmap'] = args.colormap
dp.P.rcParams["savefig.dpi"] = args.dpi dp.P.rcParams['savefig.dpi'] = args.dpi
dp.P.rcParams['lines.linewidth'] = 2
dp.P.rcParams['lines.markersize'] = 10
dp.P.rcParams["errorbar.capsize"] = 4
me.P = dp.P
# List of id that were successfully computed # List of id that were successfully computed
run_succeded = {} run_succeded = {}
@@ -159,6 +226,11 @@ for run in runs:
path_in = storage_in + path_suffix path_in = storage_in + path_suffix
path_out = storage_out + path_suffix path_out = storage_out + path_suffix
if args.tag == '':
tag_run = run
else:
tag_run = run + '_' + args.tag
if not os.path.exists(path_out): if not os.path.exists(path_out):
os.makedirs(path_out) os.makedirs(path_out)
try: try:
@@ -169,92 +241,95 @@ for run in runs:
run_succeded[run] = [] run_succeded[run] = []
for i in range(first, last + 1, step): if args.which_outputs in ['all', 'time'] :
names = glob.glob(path_in + "/output_[0-9][0-9][0-9][0-9][0-9]")
nums_all = [int(n.split('/')[-1].split('_')[1]) for n in names]
nums_all = np.sort(nums_all)
if args.which_outputs == 'all':
nums = nums_all
else:
time = [dp.get_time(path_in, n) for n in nums_all]
nums = [n for i,n in enumerate(nums_all) if time[i] >= args.time_begin and time[i] < args.time_end]
else:
nums = range(first, last + 1, step)
for num in nums:
failures = 0 failures = 0
success = False success = False
while not success: while not success:
try: try:
maps_disk = None maps_disk = None
if args.print_outputs:
print("[{}, {}]".format(run, num))
if args.maps: if args.maps:
print("[{}, {}] computing maps".format(run, i)) print("[{}, {}] computing maps".format(run, num))
maps_disk = dp.compute_image_data( maps_disk = dp.compute_image_data(path_in, num,
path_in,
i,
radius=rad, radius=rad,
path_out=path_out, path_out=path_out,
tag=run, tag=tag_run,
map_size=args.mapsize, map_size=args.mapsize,
force=args.force_redo, force=args.force_redo,
axes_los=args.axes, axes_los=args.axes,
images=args.images, images=args.images,
pos_star=np.array([args.x, args.y, args.z]), pos_star=np.array([args.x, args.y, args.z]),
fft=args.fft, fft=args.fft)
) print("[{}, {}] maps computed".format(run, num))
print("[{}, {}] maps computed".format(run, i))
if args.plot_maps: if args.plot_maps:
print("[{}, {}] plotting maps".format(run, i)) print("[{}, {}] plotting maps".format(run, num))
maps_disk = dp.plot_maps( maps_disk = dp.plot_maps(path_out, num,
path_out,
i,
maps_disk=maps_disk, maps_disk=maps_disk,
axes_los=args.axes, axes_los=args.axes,
images=args.images, images=args.images,
tag=run, tag=tag_run,
force=args.force_redo, force=args.force_redo,
interactive=args.interactive, interactive=args.interactive,
put_title=(not args.beamer), put_title=(not args.beamer))
) print("[{}, {}] maps plotted".format(run, num))
print("[{}, {}] maps plotted".format(run, i))
if args.disk: if args.disk:
print("[{}, {}] computing disk props".format(run, i)) print("[{}, {}] computing disk props".format(run, num))
dp.disk_prop( dp.disk_prop(path_in, num, path_out=path_out,
path_in,
i,
path_out=path_out,
nb_bin=args.nb_bin, nb_bin=args.nb_bin,
binning=args.binning, binning=args.binning,
rad_ext=args.rad_ext, rad_ext=args.rad_ext,
force=args.force_redo, force=args.force_redo,
pos_star=np.array([args.x, args.y, args.z]), pos_star=np.array([args.x, args.y, args.z]))
) print("[{}, {}] disk_props computed".format(run, num))
print("[{}, {}] disk_props computed".format(run, i))
if args.plot_disk: if args.plot_disk:
print("[{}, {}] plotting disk props".format(run, i)) print("[{}, {}] plotting disk props".format(run, num))
dp.plot_disk_prop( dp.plot_disk_prop(path_out, num, tag=tag_run,
path_out,
i,
tag=run,
force=args.force_redo, force=args.force_redo,
interactive=args.interactive, interactive=args.interactive,
put_title=(not args.beamer), put_title=(not args.beamer))
) print("[{}, {}] disk props plotted".format(run, num))
print("[{}, {}] disk props plotted".format(run, i))
if args.pdf: if args.pdf:
print("[{}, {}] computing pdf".format(run, i)) print("[{}, {}] computing pdf #1".format(run, num))
dp.disk_pdf( dp.disk_pdf(path_out, num, maps_disk,
path_out,
i,
maps_disk,
pos_star=np.array([args.x, args.y, args.z]), pos_star=np.array([args.x, args.y, args.z]),
force=args.force_redo, force=args.force_redo,
tag=run, tag=tag_run,
nb_bin_hist=args.pdf_nb_bin,
interactive=args.interactive, interactive=args.interactive,
put_title=(not args.beamer), put_title=(not args.beamer))
) print("[{}, {}] pdf #1 computed".format(run, num))
print("[{}, {}] pdf computed".format(run, i)) if args.pdf2:
print("[{}, {}] computing pdf #2".format(run, num))
me.get_pdf(path_in, num, path_out=path_out,
force=args.force_redo,
tag=tag_run)
print("[{}, {}] pdf #2 computed".format(run, num))
if args.clump:
print("[{}, {}] computing clumps".format(run, num))
dp.clump_study(path_in, num, path_out, tag_run)
print("[{}, {}] clumps computed".format(run, num))
# If we are here, success ! # If we are here, success !
success = True success = True
run_succeded[run].append(i) run_succeded[run].append(num)
except (ValueError, IOError, KeyError) as e: except (ValueError, IOError, KeyError) as e:
print(e) print(e)
if args.watch and failures < args.allowed_failures: if args.watch and failures < args.allowed_failures:
failures = failures + 1 failures = failures + 1
print( print("Unable to proceed for run {} output {}. Trying again in {} s ({} tries remaining)".format(run, num, args.waiting_time, args.allowed_failures - failures))
"Unable to proceed for run {} output {}. Trying again in {} s ({} tries remaining)".format(
run, i, args.waiting_time, args.allowed_failures - failures
)
)
time.sleep(args.waiting_time) time.sleep(args.waiting_time)
elif args.skip: elif args.skip:
break break
@@ -262,34 +337,25 @@ for run in runs:
raise raise
if args.evolution: if args.evolution:
print("[{}] plotting evolution".format(run)) print("[{}] plotting evolution".format(run))
dp.evolution( dp.evolution(path_out, run_succeded[run],
path_out,
run_succeded[run],
force=args.force_redo, force=args.force_redo,
tag=args.tag,
interactive=args.interactive, interactive=args.interactive,
pdf=args.pdf, pdf=args.pdf or args.cpdf)
)
print("[{}] evolution plotted".format(run)) print("[{}] evolution plotted".format(run))
if args.compare: if args.compare:
path_suffix = project path_suffix = project
path = storage_out + path_suffix path = storage_out + path_suffix
if args.time_avg: if (args.time_avg):
# Select output availables for at least on run dp.compare(path, runs, run_succeded,
output_ok = reduce(np.union1d, [run_succeded[run] for run in runs]).astype(int) path_out=path + '/comp',
print(output_ok)
if len(output_ok) >= 1:
dp.compare(
path,
runs,
output_ok,
path_out=path + "/comp",
force=args.force_redo, force=args.force_redo,
interactive=args.interactive, interactive=args.interactive,
Q_in_name=(not args.pdf), Q_in_name=args.Q_in_name,
pdf=args.pdf, tag=args.tag,
) pdf=args.pdf or args.cpdf)
else: else:
for i in range(first, last + 1, step): for i in range(first, last + 1, step):
# Select usable runs # Select usable runs
@@ -297,16 +363,13 @@ if args.compare:
try: try:
if len(runs_ok) > 1: if len(runs_ok) > 1:
dp.compare( dp.compare(path, runs_ok, i,
path, path_out=path + '/comp',
runs_ok,
i,
path_out=path + "/comp",
force=args.force_redo, force=args.force_redo,
interactive=args.interactive, interactive=args.interactive,
Q_in_name=(not args.pdf), Q_in_name=args.Q_in_name,
pdf=args.pdf, tag=args.tag,
) pdf=args.pdf or args.cpdf)
except (KeyError, IOError) as e: except (KeyError, IOError) as e:
print(e) print(e)
if args.skip: if args.skip: