Ax-aware plotting, More PDFs, improve format

This commit is contained in:
Noe Brucy
2020-02-19 17:51:23 +01:00
parent a87abeb52d
commit 8d7c5296cc
5 changed files with 281 additions and 108 deletions
+50 -14
View File
@@ -1,9 +1,9 @@
f # coding: utf-8
# coding: utf-8
import sys
import os
import glob as glob
import copy
import tables
import pymses
@@ -87,7 +87,7 @@ class BaseProcessor:
elif type(pp_params) == str:
self.pp_params = load_params(pp_params)
else:
self.pp_params = pp_params
self.pp_params = copy.deepcopy(pp_params)
if tag is not None:
self.pp_params.out.tag = tag
@@ -229,16 +229,24 @@ class HDF5Container(BaseProcessor):
finally:
self.close()
def get_value(self, node_name):
def get_value(self, node_name, unit=None, unit_old=None):
self.open()
try:
node = self.save.get_node(node_name)
if "unit" in node._v_attrs:
unit_old = node._v_attrs.unit
if node._v_attrs.CLASS == "GROUP":
value = {}
for child_name in node._v_children:
value[child_name] = self.get_value(node_name + "/" + child_name)
value[child_name] = self.get_value(
node_name + "/" + child_name, unit, unit_old
)
else:
value = node.read()
if not (unit is None or unit_old is None or unit_old == cst.none):
value = value * unit_old.express(unit)
finally:
self.close()
return value
@@ -303,7 +311,8 @@ class HDF5Container(BaseProcessor):
self.save.get_node(name_full).attrs.unit = unit
if not attrs is None:
self.save.get_node(name_full).attrs.update(attrs)
for key in attrs:
self.save.get_node(name_full)._v_attrs[key] = attrs[key]
def set_value(self, node_name, data, description, unit):
self.open()
@@ -321,6 +330,14 @@ class HDF5Container(BaseProcessor):
self.close()
return attr
def set_attribute(self, node_name, attr_name, attr_value):
self.open()
try:
node = self.save.get_node(node_name)
node._v_attrs[attr_name] = attr_value
finally:
self.close()
def _transform(self, name, transform_fn, group="/maps", **kwargs):
src = self.save.get_node(group + "/" + name).read()
return transform_fn(src, **kwargs)
@@ -362,14 +379,33 @@ class HDF5Container(BaseProcessor):
name = transform_name + "_" + rule_src_name
self.rules[name] = Rule(
self,
fn,
group=group,
unit=unit,
description=description,
dependencies=[rule_src_name],
)
def apply(
self,
fn,
name_array_in,
name_array_out,
unit_out=cst.none,
description="",
recursive=True,
):
array_in = self.get_value(name_array_in)
if recursive and isinstance(array_in, dict):
for key in array_in:
self.apply(
fn,
name_array_in + "/" + key,
name_array_out + "/" + key,
unit_out,
description,
)
self.set_attribute(name_array_out, "unit", unit_out)
else:
try:
array_out = fn(array_in)
except TypeError:
array_out = array_in
self.set_value(name_array_out, array_out, description, unit_out)
def simple_getter(name, dset):
+33 -5
View File
@@ -217,9 +217,21 @@ class Comparator(Aggregator, HDF5Container):
cmd_grep = "sed '/cpu.*/d' {} | grep 'Number of sink' -A 2".format(log_filename)
content = os.popen(cmd_grep).readlines()
for i in range(0, len(content), 4):
series["nb_sink"][run].append(np.int(content[i].split("=")[1]))
series["mass_sink"][run].append(np.float(content[i + 1].split("=")[1]))
series["time"][run].append(np.float(content[i + 2].split("=")[1]))
try:
nb_sink = np.int(content[i].split("=")[1])
mass_sink = np.float(content[i + 1].split("=")[1])
time = np.float(content[i + 2].split("=")[1])
series["nb_sink"][run].append(nb_sink)
series["mass_sink"][run].append(mass_sink)
series["time"][run].append(time)
except (ValueError, IndexError):
self._log(
"Error encountered in parsing {} (grepped block {})".format(
log_filename, i
),
"WARNING",
)
return series
def _extract_sfr_from_log(self, series, log_filename, run):
@@ -299,16 +311,21 @@ class Comparator(Aggregator, HDF5Container):
"/series/sinks_from_log/mass_sink/" + run
).read()
mass_sink = mass_sink * mass_unit.express(cst.Msun)
if avg_window is None:
shift = 1
else:
# We assume that the timestep do not vary a lot ...
shift = np.searchsorted(time, time[0] + avg_window, side="left")
shift = np.searchsorted(time, time[-1] - avg_window, side="right")
shift = len(time) - shift
sfr = (mass_sink[shift:] - mass_sink[:-shift]) / (
time[shift:] - time[:-shift]
)
sfr_beg = (mass_sink[:shift] - mass_sink[0]) / (time[:shift] - time[0])
ssfr[run] = np.zeros(len(mass_sink))
ssfr[run][shift:] = sfr / surface
ssfr[run][:shift] = sfr_beg / surface
return ssfr, {"avg_window": avg_window}
def _turb_power(self):
@@ -376,6 +393,17 @@ class Comparator(Aggregator, HDF5Container):
"q975": "97.5 percentile of {}".format(descr),
}
# units={
# "runs": cst.none,
# "mean": unit,
# "std": unit,
# "median": unit,
# "max": unit,
# "min": unit,
# "q025": unit,
# "q975": unit}
units = unit
if name is None:
name = "avg_" + src_name
@@ -391,7 +419,7 @@ class Comparator(Aggregator, HDF5Container):
self,
fn,
group="/comp",
unit=unit,
unit=units,
description=description,
dependencies=[rule_src_name],
)
+133 -73
View File
@@ -49,7 +49,7 @@ class Plotter(Aggregator, BaseProcessor):
_ax_nb = {"x": 0, "y": 1, "z": 2} # Number of each axes
_axes_h = {"x": "y", "y": "x", "z": "x"} # Associated horizontal axe
_axes_v = {"x": "z", "y": "z", "z": "y"} # Associated vertical axe
_ax_title = {"x": r"$x$ [code]", "y": r"$y$ [code]", "z": r"$z$ [code]"}
_ax_title = {"x": r"$x$", "y": r"$y$", "z": r"$z$"}
G = 1.0 # Gravitational constant
@@ -58,6 +58,7 @@ class Plotter(Aggregator, BaseProcessor):
"beta": "$\\beta$",
"beta_cool": "$\\beta_{c}$",
"dens0": "$n_0$",
"coldens0": "$\Sigma_0$",
"sfr_avg_window": "window",
"comp_frac": "$\\zeta$",
}
@@ -118,7 +119,7 @@ class Plotter(Aggregator, BaseProcessor):
or not os.path.exists(plot_filename)
)
def _process_rule(self, name, rule, arg, overwrite=False, **kwargs):
def _process_rule(self, name, rule, arg, overwrite=False, ax=None, **kwargs):
if not arg is None:
name_full = name + "_" + str(arg)
else:
@@ -126,8 +127,16 @@ class Plotter(Aggregator, BaseProcessor):
if rule.is_valid(arg):
if rule.kind == "classic":
for run in self.runs:
for i, num in enumerate(self.nums[run]):
try:
runs = kwargs.pop("runs")
except KeyError:
runs = self.runs
if ax is None:
ax = [P.subplots(1, 1)[1] for run in runs for num in self.nums[run]]
i = 0
for run in runs:
for num in self.nums[run]:
plot_filename = self._find_filename(name_full, run, num)
save = tables.open_file(self.pp[run][num].filename)
try:
@@ -137,12 +146,27 @@ class Plotter(Aggregator, BaseProcessor):
arg,
plot_filename,
overwrite,
ax=ax[i],
run=run,
**kwargs
)
except TypeError:
self._plot_rule(
rule,
save,
arg,
plot_filename,
overwrite,
ax=ax,
run=run,
**kwargs
)
finally:
save.close()
i = i + 1
else:
if ax is None:
ax = P.gca()
if rule.kind == "series" and len(self.runs) == 1:
run = self.runs[0]
plot_filename = self._find_filename(name_full, run)
@@ -151,25 +175,16 @@ class Plotter(Aggregator, BaseProcessor):
save = tables.open_file(self.comp.filename, "r")
try:
self._plot_rule(
rule,
save,
arg,
plot_filename,
overwrite,
open_figure=not self.pp_params.out.interactive,
**kwargs
rule, save, arg, plot_filename, overwrite, ax, **kwargs
)
finally:
save.close()
else:
self._log("{} is not valid in this context".format(name_full), "ERROR")
def _plot_rule(
self, rule, save, arg, plot_filename, overwrite, open_figure=True, **kwargs
):
def _plot_rule(self, rule, save, arg, plot_filename, overwrite, ax, **kwargs):
P.sca(ax)
if self._needs_computation(overwrite, plot_filename):
if open_figure:
P.figure()
rule.plot(save, arg, **kwargs)
P.tight_layout(pad=1)
if not self.pp_params.out.interactive:
@@ -283,19 +298,26 @@ class Plotter(Aggregator, BaseProcessor):
unit=None,
unit_coeff=1.0,
overlays=[],
overlays_kwargs=[],
title=None,
nml_key=None,
put_time=True,
time_unit=cst.Myr,
unit_space=cst.pc,
cmap="plasma",
norm="log",
put_cbar=True,
autoscale=True,
**kwargs
):
ax_h = self._axes_h[ax_los]
ax_v = self._axes_v[ax_los]
im_extent = self.save.root.maps._v_attrs.im_extent
unit_length = self.save.root._v_attrs["unit_length"]
im_extent = np.array(im_extent) * unit_length.express(unit_space)
node = self.save.get_node("/maps/{}_{}".format(name, ax_los))
dmap = node.read()
@@ -319,10 +341,13 @@ class Plotter(Aggregator, BaseProcessor):
P.locator_params(axis=ax_h, nbins=self.pp_params.plot.ntick)
P.locator_params(axis=ax_v, nbins=self.pp_params.plot.ntick)
P.xlabel(self._ax_title[ax_h])
P.ylabel(self._ax_title[ax_v])
P.xlabel(self._ax_title[ax_h] + unit_str(unit_space))
P.ylabel(self._ax_title[ax_v] + unit_str(unit_space))
cbar = P.colorbar(im)
try:
cbar = P.colorbar(im, cax=P.gca().cax)
except AttributeError:
cbar = P.colorbar()
if not label is None:
cbar.set_label(label)
@@ -331,7 +356,7 @@ class Plotter(Aggregator, BaseProcessor):
if put_time:
time = self.save.root._v_attrs.time * self.comp.info["unit_time"]
time_str = "time = {:.6g} {}".format(
time_str = self.pp_params.plot.time_fmt.format(
time.express(time_unit), time_unit.latex
)
if len(title) > 0:
@@ -341,8 +366,11 @@ class Plotter(Aggregator, BaseProcessor):
P.title(title)
for plot_overlay in overlays:
plot_overlay(ax_los)
for i, plot_overlay in enumerate(overlays):
try:
plot_overlay(ax_los, **overlays_kwargs[i])
except:
plot_overlay(ax_los)
def _overlay_levels(self, ax_los):
map_level = self.save.get_node("/maps/{}_{}".format("levels", ax_los)).read()
@@ -367,7 +395,7 @@ class Plotter(Aggregator, BaseProcessor):
P.clabel(cont, cont.levels[cont.levels < 11], inline=1, fontsize=8.0, fmt="%1d")
def _overlay_speed(self, ax_los, unit=cst.km_s, unit_coeff=1.0):
def _overlay_speed(self, ax_los, unit=cst.km_s, unit_coeff=1.0, key_v=None):
ax_h = self._axes_h[ax_los]
ax_v = self._axes_v[ax_los]
dmap_vh_node = self.save.get_node("/maps/speed_h_{}".format(ax_los))
@@ -397,7 +425,8 @@ class Plotter(Aggregator, BaseProcessor):
Q = P.quiver(hh, vv, map_vh_red, map_vv_red, units="width", color="grey")
label, unit_old, unit = self._ax_label_unit(dmap_vh_node, "", unit, unit_coeff)
key_v = (max_v + min_v) / 2.0
if key_v is None:
key_v = (max_v + min_v) / 2.0
P.quiverkey(
Q,
0.6,
@@ -438,24 +467,35 @@ class Plotter(Aggregator, BaseProcessor):
nml_key=None,
put_time=True,
time_unit=cst.Myr,
xlog=None,
ylog=False,
kind="bar",
colors=None,
nml_color=None,
**kwargs
):
node = self.save.get_node("/hist/" + name)
label, unit_old, unit = self._ax_label_unit(node, label, unit, unit_coeff)
values, centers = node.read() * unit_old.express(unit)
width = centers[1] - centers[0]
if xlog is None:
try:
xlog = node._v_attrs_.logbins
except:
xlog = False
P.bar(centers, values, width, log=ylog, **kwargs)
P.grid()
label, unit_old, unit = self._ax_label_unit(node, label, unit, unit_coeff)
values, centers = node.read()
if xlog:
centers = centers + np.log10(unit_old.express(unit))
else:
centers = centers * unit_old.express(unit)
title = self._label_run(run, node, title, nml_key)
if put_time:
time = self.save.root._v_attrs.time * self.comp.info["unit_time"]
time_str = "time = {:.6g} {}".format(
time_str = self.pp_params.out.time_fmt.format(
time.express(time_unit), time_unit.latex
)
if len(title) > 0:
@@ -465,10 +505,32 @@ class Plotter(Aggregator, BaseProcessor):
P.title(title)
color = None
if not colors is None:
if nml_color is None:
color = colors[run]
else:
nml = self.comp.get_nml(nml_color, run)
try:
color = colors[nml]
except:
color = colors(nml)
if kind == "bar":
width = centers[1] - centers[0]
P.bar(centers, values, width, log=ylog, color=color, label=title, **kwargs)
elif kind == "step":
if ylog:
P.yscale("log")
P.step(centers, values, where="mid", color=color, label=title, **kwargs)
else:
raise ValueError("kind must be 'bar' or 'step'")
P.grid()
if not label is None:
P.xlabel(label)
if "/hist/fit_" + name + "_" + ax_los in self.save:
if not ax_los is None and "/hist/fit_" + name + "_" + ax_los in self.save:
slope = node.attrs.slope
origin = node.attrs.origin
P.plot(
@@ -495,10 +557,10 @@ class Plotter(Aggregator, BaseProcessor):
fit=None,
fitlabel=None,
smooth=0,
sigma_err=2.0,
nml_key=None,
runs=None,
yerr_kind="std",
sigma_err=2.0,
colors=None,
nml_color=None,
**kwargs
@@ -530,43 +592,33 @@ class Plotter(Aggregator, BaseProcessor):
elif "mean" in node_y:
x = node_x.read() * xunit_old.express(xunit)
y = node_y.mean.read() * yunit_old.express(yunit)
if yerr_kind == "std":
yerr = node_y.std.read() * yunit_old.express(yunit) * sigma_err
mask = np.isfinite(x) & np.isfinite(y) & np.isfinite(yerr)
x, y, yerr = x[mask], y[mask], yerr[mask]
if smooth > 0:
y = gaussian_filter1d(y, sigma=smooth)
base_line, _, _ = P.errorbar(x, y, yerr=yerr, label=label, **kwargs)
elif yerr_kind in ["min_max", "95per"]:
if yerr_kind == "min_max":
yerr_min = node_y.min.read() * yunit_old.express(yunit)
yerr_max = node_y.max.read() * yunit_old.express(yunit)
elif yerr_kind == "95per":
yerr_min = node_y.q025.read() * yunit_old.express(yunit)
yerr_max = node_y.q975.read() * yunit_old.express(yunit)
yerr = yerr_max - yerr_min
mask = (
np.isfinite(x)
& np.isfinite(y)
& np.isfinite(yerr_min)
& np.isfinite(yerr_max)
)
x, y, yerr, yerr_min, yerr_max = (
x[mask],
y[mask],
yerr[mask],
yerr_min[mask],
yerr_max[mask],
)
base_line, _, _ = P.errorbar(
x, y, yerr=[y - yerr_min, yerr_max - y], label=label, **kwargs
)
std = node_y.std.read() * yunit_old.express(yunit)
yerr_min = y - sigma_err * std
yerr_max = y + sigma_err * std
elif yerr_kind == "min_max":
yerr_min = node_y.min.read() * yunit_old.express(yunit)
yerr_max = node_y.max.read() * yunit_old.express(yunit)
elif yerr_kind == "95per":
yerr_min = node_y.q025.read() * yunit_old.express(yunit)
yerr_max = node_y.q975.read() * yunit_old.express(yunit)
else:
mask = np.isfinite(y)
x, y = x[mask], y[mask]
if smooth > 0:
y = gaussian_filter1d(y, sigma=smooth)
(base_line,) = P.plot(x, y, "*", **kwargs)
yerr_min = y
yerr_max = y
yerr = yerr_max - yerr_min
mask = np.isfinite(x) & np.isfinite(y) & np.isfinite(yerr)
x, y, yerr, yerr_min, yerr_max = (
x[mask],
y[mask],
yerr[mask],
yerr_min[mask],
yerr_max[mask],
)
base_line, _, _ = P.errorbar(
x, y, yerr=[y - yerr_min, yerr_max - y], label=label, **kwargs
)
else:
if runs is None:
runs = self.runs
@@ -584,10 +636,12 @@ class Plotter(Aggregator, BaseProcessor):
else:
if nml_color is None:
color = colors[i % len(colors)]
(base_line,) = P.plot(x, y, label=label_run, **kwargs)
else:
nml = self.comp.get_nml(nml_color, run)
color = colors[nml]
try:
color = colors[nml]
except:
color = colors(nml)
(base_line,) = P.plot(x, y, label=label_run, color=color, **kwargs)
P.legend()
@@ -632,8 +686,8 @@ class Plotter(Aggregator, BaseProcessor):
if yerr is None:
(a, b, rho, _map_rule, stderr) = linregress(np.log10(x), np.log10(y))
self._log(
"Power law fit y = x^({}) * 10^({}) with R^2 = {} and error is {}".format(
a, b, rho, stderr
"Power law fit y = x^({}) * {} with R^2 = {} and error is {}".format(
a, 10 ** b, rho, stderr
)
)
else:
@@ -651,8 +705,8 @@ class Plotter(Aggregator, BaseProcessor):
b, a = c[0], c[1]
residual = errfunc(c, np.log10(x), np.log10(y), yerr / y)
self._log(
"Power law fit y = x^({}) * 10^({}) with residual {}".format(
a, b, residual
"Power law fit y = x^({}) * {} with residual {}".format(
a, 10 ** b, residual
)
)
if label is None:
@@ -746,6 +800,12 @@ class Plotter(Aggregator, BaseProcessor):
"$\rho$-PDF",
dependencies=["rho_pdf"],
),
"T_pdf": PlotRule(
self, partial(self._plot_hist, "T_pdf"), "T-PDF", dependencies=["T_pdf"]
),
"P_pdf": PlotRule(
self, partial(self._plot_hist, "P_pdf"), "P-PDF", dependencies=["P_pdf"]
),
}
averageables = ["coldens", "rho", "T", "Q"]
+62 -16
View File
@@ -2,8 +2,9 @@
from baseprocessor import *
mass_func = lambda dset: dset["rho"] * dset.get_sizes() ** 3 # Mass function
vol_func = lambda dset: dset.get_sizes() ** 3 # Volume function
mass_func = lambda dset: dset["rho"] * dset["dx"] ** 3 # Mass function
vol_func = lambda dset: dset["dx"] ** 3 # Volume function
getter_T = lambda dset: dset["P"] / dset["rho"] # Temperature
class PostProcessor(HDF5Container):
@@ -35,6 +36,10 @@ class PostProcessor(HDF5Container):
self.filename = path_out + "/postproc_" + tag_name + format(num, "05") + ".h5"
self.cells_filename = (
path_out + "/cells_" + tag_name + format(num, "05") + ".h5"
)
if not os.path.exists(path_out):
os.makedirs(path_out)
@@ -68,10 +73,10 @@ class PostProcessor(HDF5Container):
self._lbox = self.info["boxlen"]
center = pp_params.pymses.center
im_extent = [
(-self._radius + center[0]) * self._lbox,
(self._radius + center[0]) * self._lbox,
(-self._radius + center[1]) * self._lbox,
(self._radius + center[1]) * self._lbox,
(-self._radius + center[0]),
(self._radius + center[0]),
(-self._radius + center[1]),
(self._radius + center[1]),
]
# Get time
@@ -82,13 +87,14 @@ class PostProcessor(HDF5Container):
self.save.root._v_attrs.run = os.path.basename(path)
self.save.root._v_attrs.num = num
self.save.root._v_attrs.lbox = self._lbox
self.save.root._v_attrs.unit_length = self.info["unit_length"]
self.save.root._v_attrs.time = time
if not "/maps" in self.save:
self.save.create_group("/", "maps", "2D maps")
self.save.root.maps._v_attrs.center = center
self.save.root.maps._v_attrs.radius = self._radius
self.save.root.maps._v_attrs.im_extent = im_extent
self.save.root.maps._v_attrs.center = center
self.save.root.maps._v_attrs.radius = self._radius
self.save.root.maps._v_attrs.im_extent = im_extent
# Initialize cameras
self._cam = {}
@@ -119,9 +125,34 @@ class PostProcessor(HDF5Container):
(/!\ Long and memory heavy)
"""
if not self.cells_loaded:
cell_source = CellsToPoints(self._amr)
self.cells = cell_source.flatten()
self.cells_loaded = True
if os.path.exists(self.cells_filename):
cells_hdf5 = tables.open_file(self.cells_filename, mode="r")
try:
node = cells_hdf5.get_node("/cells")
self.cells = {}
for key in node._v_children:
self.cells[key] = cells_hdf5.get_node("/cells/" + key).read()
finally:
cells_hdf5.close()
else:
cell_source = CellsToPoints(self._amr)
cells_pymses = cell_source.flatten()
self.cells = {}
for key in cells_pymses.fields:
self.cells[key] = cells_pymses[key]
self.cells["dx"] = cells_pymses.get_sizes()
if self.pp_params.process.save_cells:
cells_hdf5 = tables.open_file(self.cells_filename, mode="w")
try:
for key in self.cells:
cells_hdf5.create_array(
"/cells", key, self.cells[key], "", createparents=True
)
finally:
cells_hdf5.close()
self.cells_loaded = True
def unload_cells(self):
"""
@@ -195,16 +226,16 @@ class PostProcessor(HDF5Container):
else:
return np.sum(value, axis=0)
def _vol_pdf(self, getter, log=False, weight_func=vol_func):
def _vol_pdf(self, getter, bins=100, logbins=False, weight_func=vol_func):
self.load_cells()
data = getter(self.cells)
if logbins:
data = np.log10(data)
weights = weight_func(self.cells)
values, edges = np.histogram(data, weights=weights)
values, edges = np.histogram(data, bins, weights=weights)
centers = 0.5 * (edges[1:] + edges[:-1])
return np.stack([values, centers])
return (np.stack([values, centers]), {"logbins": logbins})
def _mwa_sigma(self, axes=["x", "y", "z"]):
mw_speed = self.save.get_node("/globals/mwa_speed").read()
@@ -635,9 +666,24 @@ class PostProcessor(HDF5Container):
# PDF
"rho_pdf": Rule(
self,
partial(self._vol_pdf, partial(simple_getter, "rho")),
partial(self._vol_pdf, partial(simple_getter, "rho"), logbins=True),
"Global rho-PDF",
"/hist",
unit=self.info["unit_density"],
),
"T_pdf": Rule(
self,
partial(self._vol_pdf, getter_T, logbins=True),
"Global T-PDF",
"/hist",
unit=self.info["unit_temperature"],
),
"P_pdf": Rule(
self,
partial(self._vol_pdf, getter_T, logbins=True),
"Global P-PDF",
"/hist",
unit=self.info["unit_pressure"],
),
# globals
"time_num": Rule(
+3
View File
@@ -7,6 +7,8 @@ plot : # Plot parameters
# Overlays
vel_red : 40 # Take point each vel_red for velocities
time_fmt : "time = {:.3g} {}" # Time format string, 1st field is time and 2nd is unit
disk: # Disk speficic parameters
enable : False # Enable specific disk analysis
pos_star : [1., 1., 1.] # Position of the central star
@@ -63,6 +65,7 @@ out: # Parameters for post processing
process: # General setting of the post-processor module
verbose : True # Give more infos on what is going on
num_process : 1 # Number of forks
save_cells : True # Save cells structure on disk
rules: # Specific rules parameters
turb_energy_threshold : -1 # Remove invalid data (<0 = no threshold)