noe/pipeline
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

black without mortimer

Browse Source
This commit is contained in:
Noe Brucy
2022-11-28 18:01:18 +01:00
parent 7548ef7e0a
commit fa396178c6
14 changed files with 371 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
aggregator.py
+2
View File
@@ -8,6 +8,7 @@ from utils.mypool import MyPool
try: try:
from mpi4py.futures import MPIPoolExecutor from mpi4py.futures import MPIPoolExecutor
mpi_loaded = True mpi_loaded = True
except ModuleNotFoundError: except ModuleNotFoundError:
mpi_loaded = False mpi_loaded = False
@@ -27,6 +28,7 @@ def _map_aux(fun, path, path_out, params, run_num, **kwargs):
def _map_rule(snap, rule, **kwargs): def _map_rule(snap, rule, **kwargs):
return snap.process(rule, **kwargs) return snap.process(rule, **kwargs)
class Aggregator: class Aggregator:
def get_snap_list(self, select=None): def get_snap_list(self, select=None):
baseprocessor.py
+12 -10
View File
@@ -29,7 +29,6 @@ class Rule:
kind="snapshot", kind="snapshot",
unit=U.none, unit=U.none,
name="", name="",
): ):
self.name = name self.name = name
self.process_fn = process self.process_fn = process
@@ -45,6 +44,7 @@ class Rule:
else: else:
return self.process_fn(**kwargs) return self.process_fn(**kwargs)
class BaseProcessor: class BaseProcessor:
""" """
Base class for processors, should not be instanciated Base class for processors, should not be instanciated
@@ -56,7 +56,6 @@ class BaseProcessor:
rules = {} rules = {}
solve_self_dep = True solve_self_dep = True
def __init__(self, path, path_out=".", params=None, tag=None): def __init__(self, path, path_out=".", params=None, tag=None):
if params is None: if params is None:
self.params = default_params() self.params = default_params()
@@ -77,8 +76,8 @@ class BaseProcessor:
# Initialize logger # Initialize logger
self.logger = logging.getLogger(self.log_id) self.logger = logging.getLogger(self.log_id)
self.logger.propagate = False self.logger.propagate = False
logging_format = '%(levelname)s | %(asctime)s | %(name)s.%(funcName)s:%(lineno)d | %(message)s' logging_format = "%(levelname)s | %(asctime)s | %(name)s.%(funcName)s:%(lineno)d | %(message)s"
formatter = logging.Formatter(logging_format, datefmt = '%H:%M:%S') formatter = logging.Formatter(logging_format, datefmt="%H:%M:%S")
if not self.logger.hasHandlers(): if not self.logger.hasHandlers():
stream = logging.StreamHandler(sys.stdout) stream = logging.StreamHandler(sys.stdout)
@@ -221,9 +220,7 @@ class BaseProcessor:
self.just_done.append(name_full) self.just_done.append(name_full)
return data return data
else: else:
self.logger.info( self.logger.info("Data for {} is already computed.".format(name_full))
"Data for {} is already computed.".format(name_full)
)
def def_rules(self): def def_rules(self):
for rule in self.rules: for rule in self.rules:
@@ -302,7 +299,9 @@ class HDF5Container(BaseProcessor):
if not (unit is None or unit_old is None or unit_old == U.none): if not (unit is None or unit_old is None or unit_old == U.none):
value = value * unit_old.express(unit) value = value * unit_old.express(unit)
except NoSuchNodeError: except NoSuchNodeError:
self.logger.error(f"The value {node_name} is node available", stack_info=True) self.logger.error(
f"The value {node_name} is node available", stack_info=True
)
raise raise
finally: finally:
if not open_before: if not open_before:
@@ -444,8 +443,10 @@ class HDF5Container(BaseProcessor):
group_name = os.path.dirname(name_full) group_name = os.path.dirname(name_full)
if group_name in self.save: if group_name in self.save:
group = self.save.get_node(group_name) group = self.save.get_node(group_name)
if not isinstance(group, class_name_dict['Group']): if not isinstance(group, class_name_dict["Group"]):
self.logger.warning(f"{group_name} already there and no a group, deleting") self.logger.warning(
f"{group_name} already there and no a group, deleting"
)
self.save.remove_node(group) self.save.remove_node(group)
self.save.create_array( self.save.create_array(
group_name, group_name,
@@ -554,5 +555,6 @@ def oct_vect_getter(name, i, dset):
def norm_getter(name, dset): def norm_getter(name, dset):
return np.sqrt(np.sum(dset[name] ** 2, axis=1)) return np.sqrt(np.sum(dset[name] ** 2, axis=1))
def oct_norm_getter(name, dset): def oct_norm_getter(name, dset):
return np.sqrt(np.sum(dset[name] ** 2, axis=2)) return np.sqrt(np.sum(dset[name] ** 2, axis=2))
galturb/galaxy.py
+2 -3
View File
@@ -9,7 +9,7 @@ def get_gas_dm_stars(pp):
# Load arrays # Load arrays
try: try:
pp.load_parts(keys=["pos", "vel", "mass", "epoch"]) pp.load_parts(keys=["pos", "vel", "mass", "epoch"])
except: except KeyError:
pp.load_parts(keys=["pos", "vel", "mass"]) pp.load_parts(keys=["pos", "vel", "mass"])
pp.load_cells(keys=["pos", "vel", "dx", "rho"]) pp.load_cells(keys=["pos", "vel", "dx", "rho"])
@@ -250,9 +250,8 @@ def allinone(pp, redo=False):
def fun(pp): def fun(pp):
return analyse_disk(pp), analyse_rings(pp, [4, 5, 6, 7, 8]) return analyse_disk(pp), analyse_rings(pp, [4, 5, 6, 7, 8])
try: try:
assert(not redo) assert not redo
sectors = pd.read_csv("{pp.run}/disk_{pp.run}_{pp.num}.csv") sectors = pd.read_csv("{pp.run}/disk_{pp.run}_{pp.num}.csv")
disk = pd.read_csv(f"{pp.run}/disk_{pp.run}_{pp.num}.csv") disk = pd.read_csv(f"{pp.run}/disk_{pp.run}_{pp.num}.csv")
galturb/sectors_extraction.py
+2 -3
View File
@@ -9,7 +9,7 @@ def get_gas_dm_stars(pp):
# Load arrays # Load arrays
try: try:
pp.load_parts(keys=["pos", "vel", "mass", "epoch"]) pp.load_parts(keys=["pos", "vel", "mass", "epoch"])
except: except KeyError:
pp.load_parts(keys=["pos", "vel", "mass"]) pp.load_parts(keys=["pos", "vel", "mass"])
pp.load_cells(keys=["pos", "vel", "dx", "rho"]) pp.load_cells(keys=["pos", "vel", "dx", "rho"])
@@ -250,9 +250,8 @@ def allinone(pp, redo=False):
def fun(pp): def fun(pp):
return analyse_disk(pp), analyse_rings(pp, [4, 5, 6, 7, 8]) return analyse_disk(pp), analyse_rings(pp, [4, 5, 6, 7, 8])
try: try:
assert(not redo) assert not redo
sectors = pd.read_csv("{pp.run}/disk_{pp.run}_{pp.num}.csv") sectors = pd.read_csv("{pp.run}/disk_{pp.run}_{pp.num}.csv")
disk = pd.read_csv(f"{pp.run}/disk_{pp.run}_{pp.num}.csv") disk = pd.read_csv(f"{pp.run}/disk_{pp.run}_{pp.num}.csv")
plotter.py
+28 -18
View File
@@ -114,7 +114,6 @@ def quiver(ax, map_h, map_v, extent, key_v=None, lognorm=False, label="", **kwar
map_v *= lognorm_v / norm_v map_v *= lognorm_v / norm_v
key_v = np.log10(key_v) key_v = np.log10(key_v)
# plot vector field # plot vector field
vec_field = ax.quiver(hh, vv, map_h, map_v, units="width", **kwargs) vec_field = ax.quiver(hh, vv, map_h, map_v, units="width", **kwargs)
@@ -241,7 +240,6 @@ class Plotter(Aggregator, BaseProcessor):
# log info # log info
self.log_id = "plotter({})".format(tag) self.log_id = "plotter({})".format(tag)
super(Plotter, self).__init__(path, path_out, params, tag) super(Plotter, self).__init__(path, path_out, params, tag)
# Select runs # Select runs
@@ -831,7 +829,7 @@ class Plotter(Aggregator, BaseProcessor):
s=title, s=title,
color=overtext_color, color=overtext_color,
transform=ax.transAxes, transform=ax.transAxes,
**text_kwargs **text_kwargs,
) )
else: else:
plt.title(title) plt.title(title)
@@ -971,9 +969,7 @@ class Plotter(Aggregator, BaseProcessor):
if sinks: if sinks:
try: try:
self.current_processor.sinks() self.current_processor.sinks()
data = pd.DataFrame( data = pd.DataFrame(self.current_processor.get_value("/datasets/sinks"))
self.current_processor.get_value("/datasets/sinks")
)
part_pos = data[["x", "y", "z"]].values part_pos = data[["x", "y", "z"]].values
unit_length /= self.current_processor.lbox unit_length /= self.current_processor.lbox
except KeyError: except KeyError:
@@ -1038,6 +1034,8 @@ class Plotter(Aggregator, BaseProcessor):
# Scatter plot # Scatter plot
scatter = plt.scatter(part_h, part_v, s=s, c=c, **kwargs) scatter = plt.scatter(part_h, part_v, s=s, c=c, **kwargs)
return scatter
def _overlay_vector( def _overlay_vector(
self, self,
name, name,
@@ -1160,7 +1158,9 @@ class Plotter(Aggregator, BaseProcessor):
else: else:
nml_value = self.study.get_nml(nml_color, run) nml_value = self.study.get_nml(nml_color, run)
if os.path.basename(nml_color) in self.value_convert: if os.path.basename(nml_color) in self.value_convert:
nml_value = self.value_convert[ os.path.basename(nml_color)](nml_value) nml_value = self.value_convert[os.path.basename(nml_color)](
nml_value
)
try: try:
color = colors[nml_value] color = colors[nml_value]
except TypeError: except TypeError:
@@ -1244,7 +1244,6 @@ class Plotter(Aggregator, BaseProcessor):
Generic plot routine, with x, y two numpy arrauys Generic plot routine, with x, y two numpy arrauys
""" """
# Option to smooth data for readability (beware) # Option to smooth data for readability (beware)
if smooth > 0: if smooth > 0:
y = gaussian_filter1d(y, sigma=smooth) y = gaussian_filter1d(y, sigma=smooth)
@@ -1283,7 +1282,9 @@ class Plotter(Aggregator, BaseProcessor):
else: else:
nml_value = self.study.get_nml(nml_color, run) nml_value = self.study.get_nml(nml_color, run)
if os.path.basename(nml_color) in self.value_convert: if os.path.basename(nml_color) in self.value_convert:
nml_value = self.value_convert[os.path.basename(nml_color)](nml_value) nml_value = self.value_convert[os.path.basename(nml_color)](
nml_value
)
try: try:
color = colors[nml_value] color = colors[nml_value]
except TypeError: except TypeError:
@@ -1363,10 +1364,18 @@ class Plotter(Aggregator, BaseProcessor):
# Find proper labels # Find proper labels
xlabel, xunit_old, xunit = self._ax_label_unit( xlabel, xunit_old, xunit = self._ax_label_unit(
name_x, xlabel, xunit, xunit_coeff, put_units=put_units, name_x,
xlabel,
xunit,
xunit_coeff,
put_units=put_units,
) )
ylabel, yunit_old, yunit = self._ax_label_unit( ylabel, yunit_old, yunit = self._ax_label_unit(
name_y, ylabel, yunit, yunit_coeff, put_units=put_units, name_y,
ylabel,
yunit,
yunit_coeff,
put_units=put_units,
) )
# Manage the different forms in which the data may be stored : # Manage the different forms in which the data may be stored :
@@ -1426,8 +1435,7 @@ class Plotter(Aggregator, BaseProcessor):
"Errorbar may be meaningless when ytransform is used" "Errorbar may be meaningless when ytransform is used"
) )
self.plot(x, y, yerr=yerr, xlabel=xlabel, self.plot(x, y, yerr=yerr, xlabel=xlabel, ylabel=ylabel, run=run, **kwargs)
ylabel=ylabel, run=run, **kwargs)
if subname_x: if subname_x:
hdf5_x.close() hdf5_x.close()
@@ -1518,13 +1526,15 @@ class Plotter(Aggregator, BaseProcessor):
This is where rules are defined This is where rules are defined
""" """
self.rules = { self.rules = {
"plot_comp": PlotRule(lambda arg, **kwargs: self._plot(*arg, **kwargs), kind="comp" "plot_comp": PlotRule(
lambda arg, **kwargs: self._plot(*arg, **kwargs), kind="comp"
), ),
"plot_run": PlotRule(lambda arg, **kwargs: self._plot(*arg, **kwargs), kind="run" "plot_run": PlotRule(
lambda arg, **kwargs: self._plot(*arg, **kwargs), kind="run"
), ),
"plot_snapshot": PlotRule(lambda arg, **kwargs: self._plot(*arg, **kwargs) "plot_snapshot": PlotRule(lambda arg, **kwargs: self._plot(*arg, **kwargs)),
), "plot_map": PlotRule(
"plot_map": PlotRule(lambda mapname, **kwargs: self._plot_map(mapname, **kwargs) lambda mapname, **kwargs: self._plot_map(mapname, **kwargs)
), ),
"coldens": PlotRule( "coldens": PlotRule(
partial( partial(
scripts/mngseg.py
-1
View File
@@ -264,7 +264,6 @@ def analyse_sim(im, load=False, scale_image=False):
# make images of the Gaussian and coherent part # make images of the Gaussian and coherent part
make_images(im, wt, M, meanim, label) make_images(im, wt, M, meanim, label)
if scale_image: if scale_image:
# (optional) make the image for each scale # (optional) make the image for each scale
for s in range(fit_min, fit_max + 1): for s in range(fit_min, fit_max + 1):
scripts/pspec.py
+2 -12
View File
@@ -13,7 +13,6 @@ import pymses.utils.misc
import tables as T import tables as T
from numpy.fft import fftn, ifft from numpy.fft import fftn, ifft
from pymses.analysis import Camera, ScalarOperator, cube3d from pymses.analysis import Camera, ScalarOperator, cube3d
import os
__generator__ = "pspec.py" __generator__ = "pspec.py"
__version__ = "0.2" __version__ = "0.2"
@@ -62,8 +61,6 @@ def degrade_cube(cube, lvl, integrate=False):
return cube_new return cube_new
def calc_k(n, nbinsk, nbig, dkbig, dim=3, saxis=2): def calc_k(n, nbinsk, nbig, dkbig, dim=3, saxis=2):
"""Make cubes containing the wave vectors, a list of bins and the """Make cubes containing the wave vectors, a list of bins and the
associated normalization factors associated normalization factors
@@ -433,9 +430,7 @@ parser = argparse.ArgumentParser(
parser.add_argument( parser.add_argument(
"repo", help="RAMSES output repository", type=str, default=".", nargs="?" "repo", help="RAMSES output repository", type=str, default=".", nargs="?"
) )
parser.add_argument( parser.add_argument("iouts", help="output numbers", type=int, default=[1], nargs="+")
"iouts", help="output numbers", type=int, default=[1], nargs="+"
)
parser.add_argument( parser.add_argument(
"outfile", "outfile",
help="output file format (see below for fields)", help="output file format (see below for fields)",
@@ -525,12 +520,7 @@ def main(arg):
read_lvl = None read_lvl = None
if True: if True:
# Load output ------------------------------------------------------------------ # Load output ------------------------------------------------------------------
# If ratarmount was used ro = pymses.RamsesOutput(arg.repo, iout, order=arg.order)
if os.path.exists(f"{self.path}/output_{self.num:05}/output_{self.num:05}"):
path = f"{arg.repo}/output_{self.num:05}"
else:
path = arg.repo
ro = pymses.RamsesOutput(path, iout, order=arg.order)
if arg.magnetic: if arg.magnetic:
amr = ro.amr_source(["rho", "vel", "Bl", "Br"]) amr = ro.amr_source(["rho", "vel", "Bl", "Br"])
else: else:
snapshotprocessor.py
+46 -36
View File
@@ -220,9 +220,9 @@ def pspec2d(map2D):
pmap = pspec.pcube(fmap) pmap = pspec.pcube(fmap)
# Use the power map and the fft to compute the powerspectrum # Use the power map and the fft to compute the powerspectrum
# This is typically an histogram of k weighted by the fourier transform value # This is typically an histogram of k weighted by the fourier transform value
pspec, kbins, pspec2, fbins = pspec.pspectrum(pmap, kmap, kbins, 1, 0) power, kbins, power2, fbins = pspec.pspectrum(pmap, kmap, kbins, 1, 0)
# Return bin center and power spectrum # Return bin center and power spectrum
return 0.5 * (kbins[1:] + kbins[:-1]), pspec return 0.5 * (kbins[1:] + kbins[:-1]), power
def degrade_map(dmap, nnew, integrate=False): def degrade_map(dmap, nnew, integrate=False):
@@ -391,7 +391,8 @@ class SnapshotProcessor(HDF5Container):
unit_time = U.Unit( unit_time = U.Unit(
name=os.path.basename(unit_time), name=os.path.basename(unit_time),
base_unit=self.info["unit_time"], base_unit=self.info["unit_time"],
coeff=factor) coeff=factor,
)
time_in_right_unit = self.time * self.info["unit_time"].express(unit_time) time_in_right_unit = self.time * self.info["unit_time"].express(unit_time)
if self.params.astrophysix.generate: if self.params.astrophysix.generate:
@@ -405,10 +406,9 @@ class SnapshotProcessor(HDF5Container):
try: try:
self.init_pymses() self.init_pymses()
except: except IOError:
self.logger.error("Pymses not initialized", exc_info=1) self.logger.error("Pymses not initialized", exc_info=1)
self.def_rules() self.def_rules()
def init_pymses(self): def init_pymses(self):
@@ -504,9 +504,9 @@ class SnapshotProcessor(HDF5Container):
self.save.root.maps._v_attrs.center = center self.save.root.maps._v_attrs.center = center
self.save.root.maps._v_attrs.radius = self._radius self.save.root.maps._v_attrs.radius = self._radius
self.save.root.maps._v_attrs.im_extent = im_extent self.save.root.maps._v_attrs.im_extent = im_extent
except: except Exception() as e:
self.logger.error("Error in HDF5", exc_info=1) self.logger.error("Error in HDF5", exc_info=1)
raise raise e
finally: finally:
self.close() self.close()
@@ -685,7 +685,6 @@ class SnapshotProcessor(HDF5Container):
if level is None: if level is None:
level = self.get_nml("amr_params/levelmin") level = self.get_nml("amr_params/levelmin")
size = 1.0 size = 1.0
cam = Camera( cam = Camera(
@@ -698,9 +697,7 @@ class SnapshotProcessor(HDF5Container):
map_max_size=2 ** level, map_max_size=2 ** level,
) )
cube = extractor.process( cube = extractor.process(cam, cube_size=1.0, resolution=2 ** level).data
cam, cube_size=1.0, resolution=2 ** level
).data
return cube return cube
def slice(self, getter, ax_los="z", z=0.0, unit=U.none): def slice(self, getter, ax_los="z", z=0.0, unit=U.none):
@@ -731,8 +728,9 @@ class SnapshotProcessor(HDF5Container):
datamap = slicing.SliceMap(self._amr, self._cam[ax_los], op, z=z) datamap = slicing.SliceMap(self._amr, self._cam[ax_los], op, z=z)
return datamap.map.T return datamap.map.T
def ax_avg(
def ax_avg(self, oct_getter, ax_los, unit=U.none, mass_weighted=True, surf_qty=False): self, oct_getter, ax_los, unit=U.none, mass_weighted=True, surf_qty=False
):
""" """
Map of the average of a quantity (given by getter) along an axis (ax_los) Map of the average of a quantity (given by getter) along an axis (ax_los)
Returns 2D array if getter returns a scalar quantity Returns 2D array if getter returns a scalar quantity
@@ -847,7 +845,15 @@ class SnapshotProcessor(HDF5Container):
self.unload_cells() self.unload_cells()
return data return data
def vol_pdf(self, getter, bins=100, old_unit=None, unit=None, logbins=False, weight_func=vol_func): def vol_pdf(
self,
getter,
bins=100,
old_unit=None,
unit=None,
logbins=False,
weight_func=vol_func,
):
self.load_cells() self.load_cells()
data = getter(self.cells) data = getter(self.cells)
if old_unit is not None and unit is not None: if old_unit is not None and unit is not None:
@@ -1057,7 +1063,9 @@ class SnapshotProcessor(HDF5Container):
dmap_omega = rt_omega.process(self._cam[ax_los]).map.T dmap_omega = rt_omega.process(self._cam[ax_los]).map.T
return dmap_omega return dmap_omega
def _toomreQ_disk(self, ax_los, omega_approx=False, G1_units=False, coarsen_factor=1): def _toomreQ_disk(
self, ax_los, omega_approx=False, G1_units=False, coarsen_factor=1
):
""" """
Compute the Toomre Q parameter Compute the Toomre Q parameter
""" """
@@ -1452,8 +1460,6 @@ class SnapshotProcessor(HDF5Container):
pspec.pspec(repo=self.path, iouts=[self.num], outfile=outfile, **kwargs) pspec.pspec(repo=self.path, iouts=[self.num], outfile=outfile, **kwargs)
return np.array([self.pspec_filename]) return np.array([self.pspec_filename])
def _write_particles(self): def _write_particles(self):
"""Ensure particles are written in the hdf5 file""" """Ensure particles are written in the hdf5 file"""
if not os.path.exists(self.parts_filename) and not self.parts_loaded: if not os.path.exists(self.parts_filename) and not self.parts_loaded:
@@ -1604,7 +1610,10 @@ class SnapshotProcessor(HDF5Container):
""" """
# Selection of cells # Selection of cells
mask = self.cells["rho"] * self.info["unit_density"].express(U.H_cc) > threshold_density mask = (
self.cells["rho"] * self.info["unit_density"].express(U.H_cc)
> threshold_density
)
ncells = np.sum(mask) ncells = np.sum(mask)
# fill the matrice with ID, x,y,z and masses of particles # fill the matrice with ID, x,y,z and masses of particles
@@ -1612,29 +1621,31 @@ class SnapshotProcessor(HDF5Container):
cells_group[:, 0] = np.arange(ncells) # index cells_group[:, 0] = np.arange(ncells) # index
position = self.cells["pos"][mask] * self.info["unit_length"].express(U.pc) position = self.cells["pos"][mask] * self.info["unit_length"].express(U.pc)
cells_group[:, 1:4] = position # position cells_group[:, 1:4] = position # position
density = self.cells["rho"][mask] * self.info["unit_density"].express(U.Msun / U.pc**3) density = self.cells["rho"][mask] * self.info["unit_density"].express(
U.Msun / U.pc ** 3
)
size = self.cells["dx"][mask] * self.info["unit_length"].express(U.pc) size = self.cells["dx"][mask] * self.info["unit_length"].express(U.pc)
cells_group[:, 4] = density * size ** 3 # mass cells_group[:, 4] = density * size ** 3 # mass
# save file.txt # save file.txt
head = str(ncells) head = str(ncells)
np.savetxt( np.savetxt(
self.filename[:-3] + '_hop.txt', self.filename[:-3] + "_hop.txt",
cells_group[:, :-1], cells_group[:, :-1],
fmt='%10d %.10e %.10e %.10e %.10e', fmt="%10d %.10e %.10e %.10e %.10e",
header=head, header=head,
delimiter=' ', delimiter=" ",
comments=' ' comments=" ",
) )
# save file.den # save file.den
f = open(self.filename[:-3] + '_hop.den','wb') f = open(self.filename[:-3] + "_hop.den", "wb")
f.write(pack('I', ncells)) f.write(pack("I", ncells))
self.cells["rho"][mask].astype("f").tofile(f) self.cells["rho"][mask].astype("f").tofile(f)
f.close() f.close()
# exec HOP algo # exec HOP algo
h = HOP(self.filename[:-3] + '_hop.txt', os.path.dirname(self.filename)) h = HOP(self.filename[:-3] + "_hop.txt", os.path.dirname(self.filename))
h.process_hop() h.process_hop()
# get the igroup array # get the igroup array
@@ -1647,7 +1658,9 @@ class SnapshotProcessor(HDF5Container):
cells_group[6] = group_ids[ind_sort] cells_group[6] = group_ids[ind_sort]
# Make it a pandas' DataFrame # Make it a pandas' DataFrame
cells_group = pd.DataFrame(cells_group, header=["id", "x", "y", "z", "mass", "group"]) cells_group = pd.DataFrame(
cells_group, header=["id", "x", "y", "z", "mass", "group"]
)
self.clumps = cells_group self.clumps = cells_group
@@ -1658,7 +1671,6 @@ class SnapshotProcessor(HDF5Container):
cells_group = self.make_clump_hop() cells_group = self.make_clump_hop()
cells_group.groupby("group") cells_group.groupby("group")
def def_rules(self): def def_rules(self):
self.rules = { self.rules = {
@@ -1714,8 +1726,7 @@ class SnapshotProcessor(HDF5Container):
dependencies=["slice_rho"], dependencies=["slice_rho"],
unit=self.info["unit_pressure"] / self.info["unit_density"], unit=self.info["unit_pressure"] / self.info["unit_density"],
), ),
"levels": Rule(self._levels, "Max level within line of sight", "/maps" "levels": Rule(self._levels, "Max level within line of sight", "/maps"),
),
"jeans": Rule( "jeans": Rule(
self._jeans, self._jeans,
"Jeans length slice", "Jeans length slice",
@@ -1769,8 +1780,7 @@ class SnapshotProcessor(HDF5Container):
}, },
), ),
"pspec": Rule(self.pspec, "Power spectrum", "/hdf5"), "pspec": Rule(self.pspec, "Power spectrum", "/hdf5"),
"write_particles": Rule(self._write_particles, "Particles file", "/hdf5" "write_particles": Rule(self._write_particles, "Particles file", "/hdf5"),
),
"write_cells": Rule(self._write_cells, "Cells file", "/hdf5"), "write_cells": Rule(self._write_cells, "Cells file", "/hdf5"),
"filaments": Rule( "filaments": Rule(
self._filaments, self._filaments,
@@ -1986,8 +1996,10 @@ class SnapshotProcessor(HDF5Container):
# Norm # Norm
generic_rule( generic_rule(
field + "_norm", partial(norm_getter, field), self.unit_key[field], field + "_norm",
oct_getter=partial(oct_norm_getter, field) partial(norm_getter, field),
self.unit_key[field],
oct_getter=partial(oct_norm_getter, field),
) )
else: else:
@@ -1998,8 +2010,6 @@ class SnapshotProcessor(HDF5Container):
unit = self.rules[name].unit unit = self.rules[name].unit
self.rules["rad_avg_" + name] = Rule( self.rules["rad_avg_" + name] = Rule(
partial(self._rad_avg, name), partial(self._rad_avg, name),
"Azimuthal average of {}".format(name), "Azimuthal average of {}".format(name),
studyprocessor.py
+34 -18
View File
@@ -38,12 +38,9 @@ class StudyProcessor(Aggregator, HDF5Container):
Creates the basic structures needed for the outputs Creates the basic structures needed for the outputs
""" """
# log id # log id
self.log_id = "study({})".format(tag) self.log_id = "study({})".format(tag)
super(StudyProcessor, self).__init__(path, path_out, params, tag) super(StudyProcessor, self).__init__(path, path_out, params, tag)
# Open outfile # Open outfile
@@ -60,7 +57,12 @@ class StudyProcessor(Aggregator, HDF5Container):
# Select runs # Select runs
if selector is None: if selector is None:
selector = RunSelector( selector = RunSelector(
path, runs, nums, self.params.input.nml_filename, unit_time=unit_time, **kwargs path,
runs,
nums,
self.params.input.nml_filename,
unit_time=unit_time,
**kwargs,
) )
# Save infos # Save infos
@@ -68,7 +70,6 @@ class StudyProcessor(Aggregator, HDF5Container):
self.runs = selector.runs self.runs = selector.runs
self.nums = selector.nums self.nums = selector.nums
run0 = self.runs[0] run0 = self.runs[0]
self.info = selector.info[run0][self.nums[run0][0]] self.info = selector.info[run0][self.nums[run0][0]]
self.namelist = selector.namelist self.namelist = selector.namelist
@@ -90,8 +91,6 @@ class StudyProcessor(Aggregator, HDF5Container):
unit_time=unit_time, unit_time=unit_time,
) )
# Save namelist and logs # Save namelist and logs
if self.params.out.copy_info: if self.params.out.copy_info:
for run in self.runs: for run in self.runs:
@@ -158,7 +157,16 @@ class StudyProcessor(Aggregator, HDF5Container):
prop[run] = getter(run) prop[run] = getter(run)
return np.array(list(prop.values())) return np.array(list(prop.values()))
def time_avg(self, name, start=None, end=None, span=None, unit_time=U.Myr, group="/series", select=None): def time_avg(
self,
name,
start=None,
end=None,
span=None,
unit_time=U.Myr,
group="/series",
select=None,
):
"""Do the time average and quantiles of a time series """Do the time average and quantiles of a time series
Parameters Parameters
@@ -284,8 +292,6 @@ class StudyProcessor(Aggregator, HDF5Container):
else: else:
return self.namelist return self.namelist
def get_pdf_slope(self, name, run, num): def get_pdf_slope(self, name, run, num):
snap = self.snaps[run][num] snap = self.snaps[run][num]
snap.process(["fit_pdf_" + name], ["z"], overwrite=self.overwrite_dep) snap.process(["fit_pdf_" + name], ["z"], overwrite=self.overwrite_dep)
@@ -334,7 +340,9 @@ class StudyProcessor(Aggregator, HDF5Container):
for j in range(nb_stellar[i]): for j in range(nb_stellar[i]):
line_stellar = logfile.readline().split() line_stellar = logfile.readline().split()
current_line += 1 current_line += 1
while line_stellar[0] == "random": # random number outputs are ... random while (
line_stellar[0] == "random"
): # random number outputs are ... random
line_stellar = logfile.readline().split() line_stellar = logfile.readline().split()
current_line += 1 current_line += 1
mass = float(line_stellar[3]) mass = float(line_stellar[3])
@@ -476,7 +484,6 @@ class StudyProcessor(Aggregator, HDF5Container):
) )
return series return series
def get_logs(self, run): def get_logs(self, run):
glob_str = f"{self.path}/{run}/{self.params.input.log_prefix}*" glob_str = f"{self.path}/{run}/{self.params.input.log_prefix}*"
logs = glob.glob(glob_str) logs = glob.glob(glob_str)
@@ -508,7 +515,7 @@ class StudyProcessor(Aggregator, HDF5Container):
# Always prefer data from last log, assuming they come in the right order # Always prefer data from last log, assuming they come in the right order
time = series["time"][run] time = series["time"][run]
time_new = time[size] time_new = time[size]
ind_overlap = np.searchsorted(time[:size], time_new, side='right') ind_overlap = np.searchsorted(time[:size], time_new, side="right")
for key in series: for key in series:
del series[key][run][ind_overlap:size] del series[key][run][ind_overlap:size]
@@ -543,7 +550,7 @@ class StudyProcessor(Aggregator, HDF5Container):
mass_sink = self.get_value("/series/sinks_from_log/mass_sink") mass_sink = self.get_value("/series/sinks_from_log/mass_sink")
time_sink = self.get_value("/series/sinks_from_log/time") time_sink = self.get_value("/series/sinks_from_log/time")
total_mass = dict() total_mass = {}
for run in self.runs: for run in self.runs:
if time_sink[run][-1] > time_gas[run][-1]: if time_sink[run][-1] > time_gas[run][-1]:
@@ -556,7 +563,9 @@ class StudyProcessor(Aggregator, HDF5Container):
offset = time_gas[run].size - time_sink[run].size offset = time_gas[run].size - time_sink[run].size
mass_gas[run] = m0 + m0 * mass_gas[run] # convert in Msun mass_gas[run] = m0 + m0 * mass_gas[run] # convert in Msun
total_mass[run] = mass_gas[run].copy() total_mass[run] = mass_gas[run].copy()
total_mass[run][offset:] = mass_gas[run][offset:] + mass_sink[run] # re add sink_mass total_mass[run][offset:] = (
mass_gas[run][offset:] + mass_sink[run]
) # re add sink_mass
return time_gas, total_mass, mass_gas return time_gas, total_mass, mass_gas
def _ssfr_from_mass_sink(self, avg_window=None): def _ssfr_from_mass_sink(self, avg_window=None):
@@ -841,9 +850,16 @@ class StudyProcessor(Aggregator, HDF5Container):
}, },
), ),
"SN_momentum_from_log": Rule( "SN_momentum_from_log": Rule(
partial(self._from_log, ["time", "SN_momentum"], self._extract_SN_Mom_from_log), partial(
self._from_log,
["time", "SN_momentum"],
self._extract_SN_Mom_from_log,
),
group="/series", group="/series",
unit={"time": "unit_time", "SN_momentum" : {"unit_mass" : 1, "unit_velocity" : 1}}, unit={
"time": "unit_time",
"SN_momentum": {"unit_mass": 1, "unit_velocity": 1},
},
description={ description={
"time": "Time", "time": "Time",
"SN_momentum": "Injected momentum", "SN_momentum": "Injected momentum",
@@ -934,7 +950,7 @@ class StudyProcessor(Aggregator, HDF5Container):
"turb_power", "turb_power",
"time_rho_prof", "time_rho_prof",
"time_coldens_pdf", "time_coldens_pdf",
"time_rho_pdf" "time_rho_pdf",
]: ]:
self._gen_rule_avg(name) self._gen_rule_avg(name)
turbox/turbox.py
+44 -21
View File
@@ -41,11 +41,7 @@ def get_pspec(pp, field:str, dim:int=3):
return k, pspec return k, pspec
span_resolution = { span_resolution = {256: (0.8, 1.1), 512: (0.5, 1.7), 1024: (0.4, 1.7)}
256: (0.8, 1.1),
512: (0.5, 1.7),
1024: (0.4, 1.7)
}
def get_pspec_slope(pp, field: str, resol: int, plotdebug: bool = False): def get_pspec_slope(pp, field: str, resol: int, plotdebug: bool = False):
@@ -74,15 +70,27 @@ def get_pspec_slope(pp, field:str, resol:int, plotdebug:bool=False):
if plotdebug: if plotdebug:
plt.figure() plt.figure()
plt.plot(logk, logpower) plt.plot(logk, logpower)
plt.plot(logk[mask], results.slope*logk[mask]+ results.intercept, lw=3, ls=":", color="k") plt.plot(
pp.logger.info(f"Fit results in get_slope({field}, {resol}): slope:{results.slope:.2f}, b:{results.intercept:.2f}, R2:{results.rvalue**2:.2f}") logk[mask],
results.slope * logk[mask] + results.intercept,
lw=3,
ls=":",
color="k",
)
pp.logger.info(
f"Fit results in get_slope({field}, {resol}): slope:{results.slope:.2f}"
+ f", b:{results.intercept:.2f}, R2:{results.rvalue**2:.2f}"
)
if results.rvalue ** 2 < 0.8: if results.rvalue ** 2 < 0.8:
pp.logger.warning(f"Bad fit in get_slope({field}, {resol}) with {logkmin} <= logk < {logkmax}") pp.logger.warning(
f"Bad fit in get_slope({field}, {resol}) with {logkmin} <= logk < {logkmax}"
)
pp.logger.warning(f"log(k) is \n {logk[mask]}") pp.logger.warning(f"log(k) is \n {logk[mask]}")
pp.logger.warning(f"log(power) is \n {logpower[mask]}") pp.logger.warning(f"log(power) is \n {logpower[mask]}")
return results.slope, results.intercept, results.rvalue ** 2 return results.slope, results.intercept, results.rvalue ** 2
def build_suite(pl, redo=False, cs0=0.28834810480560674): def build_suite(pl, redo=False, cs0=0.28834810480560674):
"""Compute an array of parameter for each run in the Plotter pl """Compute an array of parameter for each run in the Plotter pl
@@ -100,7 +108,7 @@ def build_suite(pl, redo=False, cs0=0.28834810480560674):
dataframe with the properties of the simulation dataframe with the properties of the simulation
""" """
df = dict() df = {}
df["snapshots"] = pl.nums.values() df["snapshots"] = pl.nums.values()
df["n0"] = pl.study.get_nml("galbox_params/dens0").values() df["n0"] = pl.study.get_nml("galbox_params/dens0").values()
df["turbinit"] = pl.study.get_nml("galbox_params/turb").values() df["turbinit"] = pl.study.get_nml("galbox_params/turb").values()
@@ -113,8 +121,9 @@ def build_suite(pl, redo=False, cs0=0.28834810480560674):
df["comp"] = pl.study.get_nml("turb_params/comp_frac").values() df["comp"] = pl.study.get_nml("turb_params/comp_frac").values()
df["L"] = pl.study.get_nml("amr_params/boxlen").values() df["L"] = pl.study.get_nml("amr_params/boxlen").values()
df["T_turb"] = (np.array(list(pl.study.get_nml("turb_params/turb_T").values())) df["T_turb"] = np.array(
* pl.study.info["unit_time"].express(U.Myr)) list(pl.study.get_nml("turb_params/turb_T").values())
) * pl.study.info["unit_time"].express(U.Myr)
df = pd.DataFrame(df, index=pl.runs) df = pd.DataFrame(df, index=pl.runs)
if redo: if redo:
@@ -124,21 +133,33 @@ def build_suite(pl, redo=False, cs0=0.28834810480560674):
pl.study.time(overwrite=True) pl.study.time(overwrite=True)
for ax in ["x", "y", "z"]: for ax in ["x", "y", "z"]:
df[f"sigma_{ax}"] = np.array(list(map( df[f"sigma_{ax}"] = np.array(
list(
map(
lambda x: x.T[0], lambda x: x.T[0],
[pl.study.get_value(f"/series/time_sigma_{ax}", [
unit=U.km_s)[run] for run in pl.runs]))) pl.study.get_value(f"/series/time_sigma_{ax}", unit=U.km_s)[run]
for run in pl.runs
],
)
)
)
df["sigma_all"] = df[f"sigma_x"]**2 + df[f"sigma_y"]**2 + df[f"sigma_z"]**2 df["sigma_all"] = df["sigma_x"] ** 2 + df["sigma_y"] ** 2 + df["sigma_z"] ** 2
df["sigma_all"] = list(map(np.sqrt, df["sigma_all"].values)) df["sigma_all"] = list(map(np.sqrt, df["sigma_all"].values))
df["Mach_all"] = list(map(lambda v: v / cs0, df["sigma_all"].values)) df["Mach_all"] = list(map(lambda v: v / cs0, df["sigma_all"].values))
df["time"] = list(map(lambda x : x.T[0], df["time"] = list(
[pl.study.get_value(f"/series/time", unit=U.Myr)[run] map(
for run in pl.runs])) lambda x: x.T[0],
[pl.study.get_value("/series/time", unit=U.Myr)[run] for run in pl.runs],
)
)
df["sigma"] = list(map(lambda l: np.mean(l), df["sigma_all"].values)) df["sigma"] = list(map(lambda l: np.mean(l), df["sigma_all"].values))
df["Mach"] = df["sigma"] / cs0 df["Mach"] = df["sigma"] / cs0
df["turnover"] = (df["L"] * U.pc.express(U.km) / (2 * df["sigma"]))* U.s.express(U.Myr) df["turnover"] = (df["L"] * U.pc.express(U.km) / (2 * df["sigma"])) * U.s.express(
U.Myr
)
return df return df
@@ -149,12 +170,14 @@ def rho_pdf(pp):
rho_0 = np.mean(rho) rho_0 = np.mean(rho)
print(rho_0) print(rho_0)
s = np.log(rho / rho_0) s = np.log(rho / rho_0)
values, edges = np.histogram(s, bins=300, range=(-15, 11), values, edges = np.histogram(s, bins=300, range=(-15, 11), density=True)
density=True)
pp.unload_cells() pp.unload_cells()
centers = 0.5 * (edges[1:] + edges[:-1]) centers = 0.5 * (edges[1:] + edges[:-1])
return (np.stack([values, centers]), {"logbins": True}) return (np.stack([values, centers]), {"logbins": True})
rule_pdf = Rule(rho_pdf, "Density PDF", name="rho_pdf", group="/hist") rule_pdf = Rule(rho_pdf, "Density PDF", name="rho_pdf", group="/hist")
def apply_rule_pdf(pp): def apply_rule_pdf(pp):
return pp.process(rule_pdf, pp, overwrite=True) return pp.process(rule_pdf, pp, overwrite=True)
utils/extract_velcube.py
+13 -8
View File
@@ -1,6 +1,5 @@
from snapshotprocessor import SnapshotProcessor, U from snapshotprocessor import SnapshotProcessor, U
import pandas as pd
import os
def get_velocity_cubes(pp, unit=None): def get_velocity_cubes(pp, unit=None):
velcubes = [None, None, None] velcubes = [None, None, None]
@@ -10,6 +9,7 @@ def get_velocity_cubes(pp, unit=None):
velcubes[i] *= pp.info["unit_velocity"].express(unit) velcubes[i] *= pp.info["unit_velocity"].express(unit)
return velcubes return velcubes
def get_density_cube(pp, unit=None): def get_density_cube(pp, unit=None):
dens_cube = pp.datacube(getter=lambda dset: dset["rho"]) dens_cube = pp.datacube(getter=lambda dset: dset["rho"])
if unit is not None: if unit is not None:
@@ -19,17 +19,24 @@ def get_density_cube(pp, unit=None):
def write_data(filename, vel, dens): def write_data(filename, vel, dens):
# write fields to ramses frig readable ascii file # write fields to ramses frig readable ascii file
f = open(filename, 'w') f = open(filename, "w")
dummy = 1 dummy = 1
size = vel[0].shape[0] size = vel[0].shape[0]
f.write('{:8}{:13.5f}{:13.5f}{:13.5f}{:13.5f}\n'.format(size, dummy, dummy, dummy, dummy)) f.write(
"{:8}{:13.5f}{:13.5f}{:13.5f}{:13.5f}\n".format(
size, dummy, dummy, dummy, dummy
)
)
vx, vy, vz = vel vx, vy, vz = vel
# This strange order matches the one in the galbox condinit # This strange order matches the one in the galbox condinit
for z in range(size): for z in range(size):
for y in range(size): for y in range(size):
for x in range(size): for x in range(size):
f.write('{:13.5f}{:13.5f}{:13.5f}{:13.5f}\n'.format(vx[x,y,z], vy[x,y,z], vz[x,y,z], dens[x, y, z])) f.write(
"{:13.5f}{:13.5f}{:13.5f}{:13.5f}\n".format(
vx[x, y, z], vy[x, y, z], vz[x, y, z], dens[x, y, z]
)
)
def extract_from_pp(pp): def extract_from_pp(pp):
@@ -41,5 +48,3 @@ def extract_from_pp(pp):
def extract(path, snap_number): def extract(path, snap_number):
pp = SnapshotProcessor(path, snap_number, params="../turbox_params.yml") pp = SnapshotProcessor(path, snap_number, params="../turbox_params.yml")
extract_from_pp(pp) extract_from_pp(pp)
utils/runselector.py
+5 -4
View File
@@ -283,7 +283,9 @@ class RunSelector:
def load_info(self, run, num): def load_info(self, run, num):
info_filename_output = f"{self.path_in}/{run}/output_{num:05}/info_{num:05}.txt" info_filename_output = f"{self.path_in}/{run}/output_{num:05}/info_{num:05}.txt"
# Path of the filename if ratarmount was used # Path of the filename if ratarmount was used
info_filename_tarmount_output = f"{self.path_in}/{run}/output_{num:05}/output_{num:05}/info_{num:05}.txt" info_filename_tarmount_output = (
f"{self.path_in}/{run}/output_{num:05}/output_{num:05}/info_{num:05}.txt"
)
info_filename_folder = f"{self.path_in}/{run}/info/info_{num:05}.txt" info_filename_folder = f"{self.path_in}/{run}/info/info_{num:05}.txt"
if os.path.exists(info_filename_output): if os.path.exists(info_filename_output):
@@ -350,11 +352,13 @@ class RunSelector:
elif isinstance(unit_time, str): elif isinstance(unit_time, str):
factor = self.get_nml_value(unit_time, run) factor = self.get_nml_value(unit_time, run)
def get_time(num): def get_time(num):
time_code = self.info[run][num]["time"] time_code = self.info[run][num]["time"]
return time_code / factor return time_code / factor
else: else:
def get_time(num): def get_time(num):
time_code = self.info[run][num]["time"] time_code = self.info[run][num]["time"]
return time_code * self.info[run][num]["unit_time"].express(unit_time) return time_code * self.info[run][num]["unit_time"].express(unit_time)
@@ -496,6 +500,3 @@ class RunSelector:
f = open(os.path.expanduser(filename), "w") f = open(os.path.expanduser(filename), "w")
f.writelines(paths) f.writelines(paths)
f.close() f.close()
utils/snapshotselector.py
+18 -7
View File
@@ -9,6 +9,7 @@ from utils.runselector import RunSelector
from plotter import Plotter, U from plotter import Plotter, U
import os import os
def prep_mcons(study): def prep_mcons(study):
study.coarse_step_from_log() study.coarse_step_from_log()
@@ -33,25 +34,35 @@ def find_nums(study, prep_function, time_function, time_min=0):
prep_function(study) prep_function(study)
for run in study.runs: for run in study.runs:
time_target = max(time_min, time_function(study, run)) time_target = max(time_min, time_function(study, run))
rs = RunSelector(path_in=study.path, in_runs=run, time_min=time_min, time=time_target, unit_time=U.Myr) rs = RunSelector(
path_in=study.path,
in_runs=run,
time_min=time_min,
time=time_target,
unit_time=U.Myr,
)
nums.update(rs.nums) nums.update(rs.nums)
return nums return nums
def write_paths(nums, path_from_home, filename="~/list_file"): def write_paths(nums, path_from_home, filename="~/list_file"):
paths = [] paths = []
for key in nums: for run in nums:
for num in self.nums[run]: for num in nums[run]:
if os.path.exists("{prefix}/{run}/output_{num:05}/output_{num:05}\n"): if os.path.exists(
paths.append(f"{prefix}/{run}/output_{num:05}/output_{num:05}\n") "{path_from_home}/{run}/output_{num:05}/output_{num:05}\n"
):
paths.append(
f"{path_from_home}/{run}/output_{num:05}/output_{num:05}\n"
)
else: else:
paths.append(f"{prefix}/{run}/output_{num:05}\n") paths.append(f"{path_from_home}/{run}/output_{num:05}\n")
f = open(os.path.expanduser(filename), "w") f = open(os.path.expanduser(filename), "w")
f.writelines(paths) f.writelines(paths)
f.close() f.close()
if __name__ == '__main__': if __name__ == "__main__":
path_from_home = "simus/ismfeed/allmode" path_from_home = "simus/ismfeed/allmode"
names = "n6_st_2e5_seed3_T5Myr_nsink1e3_comp*" names = "n6_st_2e5_seed3_T5Myr_nsink1e3_comp*"