prometheus: modernize console + alerts

Signed-off-by: Robert Günzler <r@gnzler.io>

19 files changed, 1898 insertions, 441 deletions

diff --git a/bundles/prometheus/files/alertmanager.yml b/bundles/prometheus/files/alertmanager.yml
new file mode 100644
index 0000000..e725338
--- /dev/null
+++ b/bundles/prometheus/files/alertmanager.yml
@@ -0,0 +1,38 @@
+global:
+
+route:
+  group_by: [instance, alertname, severity]
+  group_wait: 30s
+  group_interval: 5m
+  repeat_interval: 4h
+  mute_time_intervals:
+    - offhours
+  routes:
+    - matchers: [severity=critical]
+      repeat_interval: 1h
+    - matchers: [severity=warning]
+      repeat_interval: 4h
+    - matchers: [severity=info]
+      repeat_interval: 24h
+  receiver: telegram
+
+time_intervals:
+  - name: offhours
+    time_intervals:
+      - location: Europe/Berlin
+        times:
+          - start_time: 23:00
+            end_time: 09:00
+
+receivers:
+  - name: telegram
+    telegram_configs:
+      - bot_token: ${repo.vault.cmd(f"rbw get tgnotify/token")}
+        chat_id: ${repo.vault.cmd(f"rbw get tgnotify/chatid")}
+        message: |
+          {{if eq .GroupLabels.severity "critical"}}🚨{{end -}}
+          {{if eq .GroupLabels.severity "warning"}}⚠️{{end -}}
+          {{if eq .GroupLabels.severity "info"}}📬{{end -}}
+          <b>{{with .GroupLabels.instance}}{{.}}/{{end}}{{.GroupLabels.alertname}}</b>
+          {{.CommonAnnotations.summary}}
+          {{.CommonAnnotations.description}}
diff --git a/bundles/prometheus/files/console_assets/ajax-loader.gif b/bundles/prometheus/files/console_assets/ajax-loader.gif
new file mode 100644
index 0000000..e192ca8
--- /dev/null
+++ b/bundles/prometheus/files/console_assets/ajax-loader.gif
Binary files differdiff --git a/bundles/prometheus/files/console_assets/menu.lib b/bundles/prometheus/files/console_assets/menu.lib
new file mode 100644
index 0000000..e5cf31f
--- /dev/null
+++ b/bundles/prometheus/files/console_assets/menu.lib
@@ -0,0 +1,51 @@
+{{/* vim: set ft=gotmpl: */}}
+
+{{/* Navbar, should be passed . */}}
+{{ define "navbar" }}
+<nav class="navbar sticky-top navbar-expand-sm navbar-dark bg-dark">
+  <!-- Brand and toggle get grouped for better mobile display -->
+  <div class="navbar-header">
+    <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false" aria-controls="navbar-nav" aria-label="toggle navigation">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    <a class="navbar-brand" href="{{ pathPrefix }}/">Prometheus</a>
+  </div>
+  <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+    <ul class="nav navbar-nav">
+      <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/alerts">Alerts</a></li>
+      <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/graph">Graph</a></li>
+      <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/targets">Targets</a></li>
+    </ul>
+  </div>
+</nav>
+{{ end }}
+
+{{/* LHS menu, should be passed . */}}
+{{ define "menu" }}
+<nav aria-label="breadcrumb">
+  <ol class="breadcrumb rounded-0">
+    {{ template "_menuItem" (args . "index.html" "overview") }}
+
+    {{ $ctx := . }}
+
+    {{ template "_menuItem" (args $ctx "node.html?instance=inazuma" "inazuma") }}
+    {{ template "_menuItem" (args $ctx "node.html?instance=shimakaze" "shimakaze") }}
+    {{ template "_menuItem" (args $ctx "prom.html?instance=inazuma" "prometheus") }}
+  </ol>
+</nav>
+{{ end }}
+
+{{/* Helper, pass (args . path name) */}}
+{{ define "_menuItem" }}
+{{ $pagelink := .arg0.Path }}
+{{ if .arg0.Params.instance }}{{ $pagelink = (printf "%v?instance=%v" .arg0.Path .arg0.Params.instance) }}{{ end }}
+<li class="breadcrumb-item{{ if eq $pagelink .arg1 }} active{{ end }} text-lowercase">
+  <a href="{{ .arg1 }}">
+    {{ if eq .arg2 "overview" }}
+    <i class="glyphicon glyphicon-th" title="{{ .arg2 }}"></i>
+    {{ else }}
+    {{ .arg2 }}
+    {{ end }}
+  </a>
+</li>
+{{ end }}
diff --git a/bundles/prometheus/files/console_assets/prom.lib b/bundles/prometheus/files/console_assets/prom.lib
new file mode 100644
index 0000000..fbc39ff
--- /dev/null
+++ b/bundles/prometheus/files/console_assets/prom.lib
@@ -0,0 +1,150 @@
+{{/* vim: set ft=gotmpl: */}}
+{{/* Load Prometheus console library JS/CSS. Should go in head */}}
+
+{{ define "prom_console_head" }}
+<meta name="viewport" content="width=device-width,initial-scale=1.0">
+<meta name="title" content="Prometheus Console">
+<title>Prometheus Console</title>
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.css">
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/css/bootstrap.min.css">
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/css/prom_console.css">
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css">
+<style type="text/css">
+  body {
+    background-color: #191d21 !important;
+    color: #fff !important;
+  }
+  .navbar {
+    margin-bottom: unset !important;
+    padding-left: 15px !important;
+    padding-right: 15px !important;
+  }
+  .prom_console_content {
+    padding-top: unset !important;
+    margin-bottom: unset !important;
+  }
+  .prom_content_div {
+    width: unset !important;
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 15px;
+  }
+  @media (max-width: 768px) {
+    .prom_content_div {
+      grid-template-columns: 1fr;
+    }
+  }
+  .prom_content_div > div {
+    padding: 10px;
+    border: 1px solid var(--gray);
+  }
+  .prom_content_div * {
+    background-color: var(--light) !important;
+    color: var(--dark) !important;
+  }
+  .prom_graph_table {
+    overflow: hidden;
+  }
+</style>
+<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.v3.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.layout.min.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/js/jquery-3.5.1.min.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/js/popper.min.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/js/bootstrap.min.js"></script>
+<script>
+  var PATH_PREFIX = "{{ pathPrefix }}";
+</script>
+<script src="{{ pathPrefix }}/classic/static/js/prom_console.js"></script>
+{{ end }}
+
+{{/* Top of all pages. */}}
+{{ define "head" -}}
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    {{ template "prom_console_head" }}
+  </head>
+  <body>
+    {{ template "navbar" . }}
+    {{ template "menu" . }}
+{{ end }}
+
+{{ define "__prom_query_drilldown_noop" }}{{ . }}{{ end }}
+{{ define "humanize" }}{{ humanize . }}{{ end }}
+{{ define "humanizeNoSmallPrefix" }}{{ if and (lt . 1.0) (gt . -1.0) }}{{ printf "%.3g" . }}{{ else }}{{ humanize . }}{{ end }}{{ end }}
+{{ define "humanize1024" }}{{ humanize1024 . }}{{ end }}
+{{ define "humanizeDuration" }}{{ humanizeDuration . }}{{ end }}
+{{ define "humanizePercentage" }}{{ humanizePercentage . }}{{ end }}
+{{ define "humanizeTimestamp" }}{{ humanizeTimestamp . }}{{ end }}
+{{ define "printf.1f" }}{{ printf "%.1f" . }}{{ end }}
+{{ define "printf.3g" }}{{ printf "%.3g" . }}{{ end }}
+
+{{/* prom_query_drilldown (args expr suffix? renderTemplate?)
+Displays the result of the expression, with a link to /graph for it.
+
+renderTemplate is the name of the template to use to render the value.
+*/}}
+{{ define "prom_query_drilldown" }}
+{{ $expr := .arg0 }}{{ $suffix := (or .arg1 "") }}{{ $renderTemplate := (or .arg2 "__prom_query_drilldown_noop") }}
+<a class="prom_query_drilldown" href="{{ pathPrefix }}{{ graphLink $expr }}"> {{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }}</a>
+{{ end }}
+
+{{ define "prom_path" }}/consoles/{{ .Path }}?{{ range $param, $value := .Params }}{{ $param }}={{ $value }}&amp;{{ end }}{{ end }}"
+
+{{ define "prom_content_head" }}
+    <div class="prom_console_content">
+      {{ template "prom_graph_timecontrol" . }}
+{{ end }}
+{{ define "prom_content_tail" }}
+    </div>
+{{ end }}
+
+{{ define "prom_graph_timecontrol" }}
+    <div class="prom_graph_timecontrol">
+      <div class="prom_graph_timecontrol_inner">
+        <div class="prom_graph_timecontrol_group ">
+          <button class="btn btn-light pull-left" type="button" id="prom_graph_duration_shrink" title="Shrink the time range.">
+            <i class="glyphicon glyphicon-minus"></i>
+          </button><!-- Comments between elements to remove spaces
+   --><input class="input pull-left align-middle" size="3" title="Time range of graph" type="text" id="prom_graph_duration"><!--
+   --><button class="btn btn-light pull-left" type="button" id="prom_graph_duration_grow" title="Grow the time range.">
+            <i class="glyphicon glyphicon-plus"></i>
+          </button>
+        </div>
+        <div class="prom_graph_timecontrol_group ">
+          <button class="btn btn-light pull-left" type="button" id="prom_graph_time_back" title="Rewind the end time.">
+            <i class="glyphicon glyphicon-backward"></i>
+          </button><!--
+   --><input class="input pull-left align-middle" title="End time of graph" placeholder="Until" type="text" id="prom_graph_time_end" size="16" value=""><!--
+    --><button class="btn btn-light pull-left" type="button" id="prom_graph_time_forward" title="Advance the end time.">
+            <i class="glyphicon glyphicon-forward"></i>
+          </button>
+        </div>
+        <div class="prom_graph_timecontrol_group ">
+          <div class="btn-group dropup prom_graph_timecontrol_refresh pull-left">
+            <button type="button" class="btn btn-light pull-left" id="prom_graph_refresh_button" title="Refresh.">
+              <i class="glyphicon glyphicon-repeat"></i>
+              <span class="icon-repeat"></span>
+              (<span id="prom_graph_refresh_button_value">Off</span>)
+            </button>
+            <button type="button" class="btn btn-light pull-left dropdown-toggle" data-toggle="dropdown" title="Set autorefresh." aria-haspopup="true" aria-expanded="false">
+              <span class="caret"></span>&nbsp;
+            </button>
+            <ul class="dropdown-menu" id="prom_graph_refresh_intervals" role="menu">
+            </ul>
+          </div>
+        </div>
+      </div>
+      <script>
+        new PromConsole.TimeControl();
+      </script>
+    </div>
+{{ end }}
+
+{{/* Bottom of all pages. */}}
+
+{{ define "tail" }}
+  </body>
+</html>
+{{ end }}
diff --git a/bundles/prometheus/files/console_assets/prom_console.css b/bundles/prometheus/files/console_assets/prom_console.css
new file mode 100644
index 0000000..e5b8be7
--- /dev/null
+++ b/bundles/prometheus/files/console_assets/prom_console.css
@@ -0,0 +1,190 @@
+.navbar {
+  margin-bottom: 20px;
+}
+
+div.prom_lhs_menu {
+  float: left;
+  margin-right: 2ex;
+  background: #222;
+  min-height: 100%;
+  overflow: auto;
+  width: auto;
+  margin-left: -40px;
+  margin-top: 55px;
+  height: 100%;
+}
+.prom_lhs_menu_nav {
+  padding-right: 0;
+}
+.prom_lhs_menu ul {
+  list-style: none;
+  padding-left: .5ex;
+  margin-left: .5ex;
+}
+.prom_lhs_menu li {
+  padding-right: 1ex;
+  padding-left: 1ex;
+  font-size: 0.9rem;
+}
+.prom_lhs_menu a,
+.prom_lhs_menu li {
+  color: #999999;
+  display: block;
+}
+.prom_lhs_menu a {
+  text-decoration: none;
+  padding-top: 0;
+  padding-bottom: 0;
+}
+.prom_lhs_menu_selected a {
+  pointer-events: none;
+  cursor: default;
+}
+li.prom_lhs_menu_selected {
+  background: #555555;
+  background-clip: padding-box;
+}
+.prom_lhs_menu li:hover {
+  background: #666666;
+}
+
+.prom_console_rhs {
+  float: right;
+  margin-left: 1ex;
+  height: 100%;
+  padding-top: 55px;
+}
+.prom_console_rhs table {
+  margin-top: 1ex;
+  margin-bottom: 32px;  /* Space for time control. */
+}
+.prom_console_rhs th {
+  text-align: center;
+}
+.prom_console_rhs td:nth-child(2) {
+  text-align: right;
+}
+
+.prom_console_content {
+  overflow: visible;
+  margin-bottom: 32px;  /* Space for time control. */
+  padding-top: 70px;   /* Space for navbar */
+}
+
+a.prom_query_drilldown {
+  text-decoration: none;
+  color: black;
+}
+a.prom_query_drilldown:hover, a.prom_query_drilldown:active {
+  text-decoration: underline;
+}
+
+#samplesGraph {
+  width: 100%;
+}
+
+.prom_content_div {
+  width: 80%;
+}
+
+.prom_graph_table {
+}
+.rickshaw_legend {
+ padding: 2px;
+ margin-top: 1px;
+}
+.rickshaw_legend li {
+ min-width: 0;
+}
+.rickshaw_legend ul li {
+ list-style-type: none;
+ display: inline-block;
+}
+.rickshaw_graph {
+  width: 100%;
+  padding: 0;
+}
+.prom_graph_hover_flipped.x_label {
+  right: 0;
+}
+.prom_graph_hover_flipped.item {
+  right: 10px;
+}
+.rickshaw_graph .detail .prom_graph_hover_flipped.item:before {
+  content: "\25b8";
+  left: auto;
+  right: 1px;
+  font-size: 0.8em;
+}
+.prom_graph_ytitle {
+  -webkit-transform: rotate(-90deg);
+  -moz-transform: rotate(-90deg);
+  font-size: 11px;
+  font-family: Arial;
+  max-width: 13px;
+  white-space: nowrap;
+}
+.prom_graph_xtitle {
+  text-align: center;
+  font-size: 11px;
+  font-family: Arial;
+}
+.prom_graph_loading {
+  position: absolute;
+  top: 0px;
+  left: 0px;
+}
+.prom_graph_error {
+  font-family: Arial;
+  text-align: center;
+}
+a.prom_graph_link {
+  text-decoration: none;
+  color: black;
+}
+
+
+.prom_graph_timecontrol {
+  background: #222;
+  position: fixed;
+  padding: 2px;
+  bottom: 0;
+  left: 0;
+  z-index: 2;
+  width: 100%;
+}
+
+.prom_graph_timecontrol_inner {
+  text-align: center;
+  height: 31px;
+}
+
+.prom_graph_timecontrol_group {
+  display: inline-block;
+  margin: 1px 5px;
+}
+.prom_graph_timecontrol_group .btn {
+  font-size: 0.8em;
+  border-radius: 0;
+  height: 29px;
+  border-color: #ccc;
+}
+
+.prom_graph_timecontrol_group .input {
+  height: 29px;
+  padding: 6px 12px;
+  font-size: 14px;
+  line-height: 1.42857143;
+  color: #555;
+  background-color: #fff;
+  background-image: none;
+  border: 1px solid #ccc;
+  text-align: center;
+}
+
+.prom_graph_timecontrol_group .dropdown-menu {
+  min-width: 90px;
+}
+.prom_graph_timecontrol_group .dropdown-menu li {
+  text-align: center;
+}
diff --git a/bundles/prometheus/files/console_assets/prom_console.js b/bundles/prometheus/files/console_assets/prom_console.js
new file mode 100644
index 0000000..cb3501c
--- /dev/null
+++ b/bundles/prometheus/files/console_assets/prom_console.js
@@ -0,0 +1,687 @@
+/*
+ * Functions to make it easier to write prometheus consoles, such
+ * as graphs.
+ *
+ */
+PromConsole = {};
+
+PromConsole.NumberFormatter = {};
+PromConsole.NumberFormatter.prefixesBig = ["k", "M", "G", "T", "P", "E", "Z", "Y"];
+PromConsole.NumberFormatter.prefixesBig1024 = ["ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"];
+PromConsole.NumberFormatter.prefixesSmall = ["m", "u", "n", "p", "f", "a", "z", "y"];
+
+PromConsole._stripTrailingZero = function(x) {
+  if (x.indexOf("e") == -1) {
+    // It's not safe to strip if it's scientific notation.
+    return x.replace(/\.?0*$/, '');
+  }
+  return x;
+};
+
+// Humanize a number.
+PromConsole.NumberFormatter.humanize = function(x) {
+  if (x === null) {
+    return "null";
+  }
+  var ret = PromConsole.NumberFormatter._humanize(
+    x, PromConsole.NumberFormatter.prefixesBig,
+    PromConsole.NumberFormatter.prefixesSmall, 1000);
+  x = ret[0];
+  var prefix = ret[1];
+  if (Math.abs(x) < 1) {
+    return x.toExponential(3) + prefix;
+  }
+  return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix;
+};
+
+// Humanize a number, don't use milli/micro/etc. prefixes.
+PromConsole.NumberFormatter.humanizeNoSmallPrefix = function(x) {
+  if (x === null) {
+    return "null";
+  }
+  if (Math.abs(x) < 1) {
+    return PromConsole._stripTrailingZero(x.toPrecision(3));
+  }
+  var ret = PromConsole.NumberFormatter._humanize(
+    x, PromConsole.NumberFormatter.prefixesBig,
+    [], 1000);
+  x = ret[0];
+  var prefix = ret[1];
+  return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix;
+};
+
+// Humanize a number with 1024 as the base, rather than 1000.
+PromConsole.NumberFormatter.humanize1024 = function(x) {
+  if (x === null) {
+    return "null";
+  }
+  var ret = PromConsole.NumberFormatter._humanize(
+    x, PromConsole.NumberFormatter.prefixesBig1024,
+    [], 1024);
+  x = ret[0];
+  var prefix = ret[1];
+  if (Math.abs(x) < 1) {
+    return x.toExponential(3) + prefix;
+  }
+  return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix;
+};
+
+// Humanize a number, returning an exact representation.
+PromConsole.NumberFormatter.humanizeExact = function(x) {
+  if (x === null) {
+    return "null";
+  }
+  var ret = PromConsole.NumberFormatter._humanize(
+    x, PromConsole.NumberFormatter.prefixesBig,
+    PromConsole.NumberFormatter.prefixesSmall, 1000);
+  return ret[0] + ret[1];
+};
+
+PromConsole.NumberFormatter._humanize = function(x, prefixesBig, prefixesSmall, factor) {
+  var prefix = "";
+  if (x === 0) {
+    /* Do nothing. */
+  } else if (Math.abs(x) >= 1) {
+    for (var i = 0; i < prefixesBig.length && Math.abs(x) >= factor; ++i) {
+      x /= factor;
+      prefix = prefixesBig[i];
+    }
+  } else {
+    for (var i = 0; i < prefixesSmall.length && Math.abs(x) < 1; ++i) {
+      x *= factor;
+      prefix = prefixesSmall[i];
+    }
+  }
+  return [x, prefix];
+};
+
+
+PromConsole.TimeControl = function() {
+  document.getElementById("prom_graph_duration_shrink").onclick = this.decreaseDuration.bind(this);
+  document.getElementById("prom_graph_duration_grow").onclick = this.increaseDuration.bind(this);
+  document.getElementById("prom_graph_time_back").onclick = this.decreaseEnd.bind(this);
+  document.getElementById("prom_graph_time_forward").onclick = this.increaseEnd.bind(this);
+  document.getElementById("prom_graph_refresh_button").onclick = this.refresh.bind(this);
+  this.durationElement = document.getElementById("prom_graph_duration");
+  this.endElement = document.getElementById("prom_graph_time_end");
+  this.durationElement.oninput = this.dispatch.bind(this);
+  this.endElement.oninput = this.dispatch.bind(this);
+  this.endElement.oninput = this.dispatch.bind(this);
+  this.refreshValueElement = document.getElementById("prom_graph_refresh_button_value");
+
+  var refreshList = document.getElementById("prom_graph_refresh_intervals");
+  var refreshIntervals = ["Off", "1m", "5m", "15m", "1h"];
+  for (var i = 0; i < refreshIntervals.length; ++i) {
+    var li = document.createElement("li");
+    li.onclick = this.setRefresh.bind(this, refreshIntervals[i]);
+    li.textContent = refreshIntervals[i];
+    refreshList.appendChild(li);
+  }
+
+  this.durationElement.value = PromConsole.TimeControl.prototype.getHumanDuration(
+    PromConsole.TimeControl._initialValues.duration);
+  if (!PromConsole.TimeControl._initialValues.endTimeNow) {
+    this.endElement.value = PromConsole.TimeControl.prototype.getHumanDate(
+      new Date(PromConsole.TimeControl._initialValues.endTime * 1000));
+  }
+};
+
+PromConsole.TimeControl.timeFactors = {
+  "y": 60 * 60 * 24 * 365,
+  "w": 60 * 60 * 24 * 7,
+  "d": 60 * 60 * 24,
+  "h": 60 * 60,
+  "m": 60,
+  "s": 1
+};
+
+PromConsole.TimeControl.stepValues = [
+  "10s", "1m", "5m", "15m", "30m", "1h", "2h", "6h", "12h", "1d", "2d",
+  "1w", "2w", "4w", "8w", "1y", "2y"
+];
+
+PromConsole.TimeControl.prototype._setHash = function() {
+  var duration = this.parseDuration(this.durationElement.value);
+  var endTime = this.getEndDate() / 1000;
+  window.location.hash = "#pctc" + encodeURIComponent(JSON.stringify(
+    { duration: duration, endTime: endTime }));
+};
+
+PromConsole.TimeControl._initialValues = function() {
+  var hash = window.location.hash;
+  var values;
+  if (hash.indexOf('#pctc') === 0) {
+    values = JSON.parse(decodeURIComponent(hash.substring(5)));
+  } else {
+    values = { duration: 3600, endTime: 0 };
+  }
+  if (values.endTime == 0) {
+    values.endTime = new Date().getTime() / 1000;
+    values.endTimeNow = true;
+  }
+  return values;
+}();
+
+PromConsole.TimeControl.prototype.parseDuration = function(durationText) {
+  var durationRE = new RegExp("^([0-9]+)([ywdhms]?)$");
+  var matches = durationText.match(durationRE);
+  if (!matches) { return 3600; }
+  var value = parseInt(matches[1]);
+  var unit = matches[2] || 's';
+  return value * PromConsole.TimeControl.timeFactors[unit];
+};
+
+PromConsole.TimeControl.prototype.getHumanDuration = function(duration) {
+  var units = [];
+  for (var key in PromConsole.TimeControl.timeFactors) {
+    units.push([PromConsole.TimeControl.timeFactors[key], key]);
+  }
+  units.sort(function(a, b) { return b[0] - a[0]; });
+  for (var i = 0; i < units.length; ++i) {
+    if (duration % units[i][0] === 0) {
+      return (duration / units[i][0]) + units[i][1];
+    }
+  }
+  return duration;
+};
+
+PromConsole.TimeControl.prototype.increaseDuration = function() {
+  var durationSeconds = this.parseDuration(this.durationElement.value);
+  for (var i = 0; i < PromConsole.TimeControl.stepValues.length; i++) {
+    if (durationSeconds < this.parseDuration(PromConsole.TimeControl.stepValues[i])) {
+      this.setDuration(PromConsole.TimeControl.stepValues[i]);
+      this.dispatch();
+      return;
+    }
+  }
+};
+
+PromConsole.TimeControl.prototype.decreaseDuration = function() {
+  var durationSeconds = this.parseDuration(this.durationElement.value);
+  for (var i = PromConsole.TimeControl.stepValues.length - 1; i >= 0; i--) {
+    if (durationSeconds > this.parseDuration(PromConsole.TimeControl.stepValues[i])) {
+      this.setDuration(PromConsole.TimeControl.stepValues[i]);
+      this.dispatch();
+      return;
+    }
+  }
+};
+
+PromConsole.TimeControl.prototype.setDuration = function(duration) {
+  this.durationElement.value = duration;
+  this._setHash();
+};
+
+PromConsole.TimeControl.prototype.getEndDate = function() {
+  if (this.endElement.value === '') {
+    return null;
+  }
+  var dateParts = this.endElement.value.split(/[- :]/)
+  return Date.UTC(dateParts[0], dateParts[1] - 1, dateParts[2], dateParts[3], dateParts[4]);
+};
+
+PromConsole.TimeControl.prototype.getOrSetEndDate = function() {
+  var date = this.getEndDate();
+  if (date) {
+    return date;
+  }
+  date = new Date();
+  this.setEndDate(date);
+  return date.getTime();
+};
+
+PromConsole.TimeControl.prototype.getHumanDate = function(date) {
+  var hours = date.getUTCHours() < 10 ? '0' + date.getUTCHours() : date.getUTCHours();
+  var minutes = date.getUTCMinutes() < 10 ? '0' + date.getUTCMinutes() : date.getUTCMinutes();
+  return date.getUTCFullYear() + "-" + (date.getUTCMonth() + 1) + "-" + date.getUTCDate() + " " +
+    hours + ":" + minutes;
+};
+
+PromConsole.TimeControl.prototype.setEndDate = function(date) {
+  this.setRefresh("Off");
+  this.endElement.value = this.getHumanDate(date);
+  this._setHash();
+};
+
+PromConsole.TimeControl.prototype.increaseEnd = function() {
+  // Increase duration 25% range & convert ms to s.
+  this.setEndDate(new Date(this.getOrSetEndDate() + this.parseDuration(this.durationElement.value) * 1000 / 4));
+  this.dispatch();
+};
+
+PromConsole.TimeControl.prototype.decreaseEnd = function() {
+  this.setEndDate(new Date(this.getOrSetEndDate() - this.parseDuration(this.durationElement.value) * 1000 / 4));
+  this.dispatch();
+};
+
+PromConsole.TimeControl.prototype.refresh = function() {
+  this.endElement.value = '';
+  this._setHash();
+  this.dispatch();
+};
+
+PromConsole.TimeControl.prototype.dispatch = function() {
+  var durationSeconds = this.parseDuration(this.durationElement.value);
+  var end = this.getEndDate();
+  if (end === null) {
+    end = new Date().getTime();
+  }
+  for (var i = 0; i < PromConsole._graph_registry.length; i++) {
+    var graph = PromConsole._graph_registry[i];
+    graph.params.duration = durationSeconds;
+    graph.params.endTime = end / 1000;
+    graph.dispatch();
+  }
+};
+
+PromConsole.TimeControl.prototype._refreshInterval = null;
+
+PromConsole.TimeControl.prototype.setRefresh = function(duration) {
+  if (this._refreshInterval !== null) {
+    window.clearInterval(this._refreshInterval);
+    this._refreshInterval = null;
+  }
+  if (duration != "Off") {
+    if (this.endElement.value !== '') {
+      this.refresh();
+    }
+    var durationSeconds = this.parseDuration(duration);
+    this._refreshInterval = window.setInterval(this.dispatch.bind(this), durationSeconds * 1000);
+  }
+  this.refreshValueElement.textContent = duration;
+};
+
+// List of all graphs, used by time controls.
+PromConsole._graph_registry = [];
+
+PromConsole.graphDefaults = {
+  expr: null,     // Expression to graph. Can be a list of strings.
+  node: null,     // DOM node to place graph under.
+  // How long the graph is over, in seconds.
+  duration: PromConsole.TimeControl._initialValues.duration,
+  // The unixtime the graph ends at.
+  endTime: PromConsole.TimeControl._initialValues.endTime,
+  width: null,    // Height of the graph div, excluding titles and legends.
+  // Defaults to auto-detection.
+  height: 200,    // Height of the graph div, excluding titles and legends.
+  min: "auto",    // Minimum Y-axis value, defaults to lowest data value.
+  max: undefined, // Maximum Y-axis value, defaults to highest data value.
+  renderer: 'line',  // Type of graphs, options are 'line' and 'area'.
+  name: null,     // What to call plots, defaults to trying to do
+  // something reasonable.
+  // If a string, it'll use that. [[ label ]] will be substituted.
+  // If a function it'll be called with a map of keys to values,
+  // and should return the name to use.
+  // Can be a list of strings/functions, each element
+  // will be applied to the plots from the corresponding
+  // element of the expr list.
+  xTitle: "Time",     // The title of the x axis.
+  yUnits: "",     // The units of the y axis.
+  yTitle: "",     // The title of the y axis.
+  // Number formatter for y axis.
+  yAxisFormatter: PromConsole.NumberFormatter.humanize,
+  // Number of ticks (horizontal lines) for the y axis
+  yAxisTicks: null,
+  // Number formatter for y values hover detail.
+  yHoverFormatter: PromConsole.NumberFormatter.humanizeExact,
+  // Color scheme to be used by the plots. Can be either a list of hex color
+  // codes or one of the color scheme names supported by Rickshaw.
+  colorScheme: null,
+};
+
+PromConsole.Graph = function(params) {
+  for (var k in PromConsole.graphDefaults) {
+    if (!(k in params)) {
+      params[k] = PromConsole.graphDefaults[k];
+    }
+  }
+  if (typeof params.expr == "string") {
+    params.expr = [params.expr];
+  }
+  if (typeof params.name == "string" || typeof params.name == "function") {
+    var name = [];
+    for (var i = 0; i < params.expr.length; i++) {
+      name.push(params.name);
+    }
+    params.name = name;
+  }
+
+  this.params = params;
+  this.rendered_data = null;
+  // Keep a reference so that further updates (e.g. annotations) can be made
+  // by the user in their templates.
+  this.rickshawGraph = null;
+  PromConsole._graph_registry.push(this);
+
+  /*
+   * Table layout:
+   * | yTitle | Graph  |
+   * |        | xTitle |
+   * | /graph | Legend |
+   */
+  var table = document.createElement("table");
+  table.className = "prom_graph_table";
+  params.node.appendChild(table);
+  var tr = document.createElement("tr");
+  table.appendChild(tr);
+  var yTitleTd = document.createElement("td");
+  tr.appendChild(yTitleTd);
+  var yTitleDiv = document.createElement("td");
+  yTitleTd.appendChild(yTitleDiv);
+  yTitleDiv.className = "prom_graph_ytitle";
+  yTitleDiv.textContent = params.yTitle + (params.yUnits ? " (" + params.yUnits.trim() + ")" : "");
+
+  this.graphTd = document.createElement("td");
+  tr.appendChild(this.graphTd);
+  this.graphTd.className = "rickshaw_graph";
+  this.graphTd.width = params.width;
+  this.graphTd.height = params.height;
+
+  tr = document.createElement("tr");
+  table.appendChild(tr);
+  tr.appendChild(document.createElement("td"));
+  var xTitleTd = document.createElement("td");
+  tr.appendChild(xTitleTd);
+  xTitleTd.className = "prom_graph_xtitle";
+  xTitleTd.textContent = params.xTitle;
+
+  tr = document.createElement("tr");
+  table.appendChild(tr);
+  var graphLinkTd = document.createElement("td");
+  tr.appendChild(graphLinkTd);
+  var graphLinkA = document.createElement("a");
+  graphLinkTd.appendChild(graphLinkA);
+  graphLinkA.className = "prom_graph_link";
+  graphLinkA.textContent = "+";
+  graphLinkA.href = PromConsole._graphsToSlashGraphURL(params.expr);
+  var legendTd = document.createElement("td");
+  tr.appendChild(legendTd);
+  this.legendDiv = document.createElement("div");
+  legendTd.width = params.width;
+  legendTd.appendChild(this.legendDiv);
+
+  window.addEventListener('resize', function() {
+    if (this.rendered_data !== null) {
+      this._render(this.rendered_data);
+    }
+  }.bind(this));
+
+  this.dispatch();
+};
+
+PromConsole.Graph.prototype._parseValue = function(value) {
+  var val = parseFloat(value);
+  if (isNaN(val)) {
+    // "+Inf", "-Inf", "+Inf" will be parsed into NaN by parseFloat(). The
+    // can't be graphed, so show them as gaps (null).
+    return null;
+  }
+  return val;
+};
+
+PromConsole.Graph.prototype._escapeHTML = function(string) {
+  var entityMap = {
+    "&": "&amp;",
+    "<": "&lt;",
+    ">": "&gt;",
+    '"': '&quot;',
+    "'": '&#39;',
+    "/": '&#x2F;'
+  };
+
+  return string.replace(/[&<>"'\/]/g, function(s) {
+    return entityMap[s];
+  });
+};
+
+PromConsole.Graph.prototype._render = function(data) {
+  var self = this;
+  var palette = new Rickshaw.Color.Palette({ scheme: this.params.colorScheme });
+  var series = [];
+
+  // This will be used on resize.
+  this.rendered_data = data;
+
+
+  var nameFuncs = [];
+  if (this.params.name === null) {
+    var chooser = PromConsole._chooseNameFunction(data);
+    for (var i = 0; i < this.params.expr.length; i++) {
+      nameFuncs.push(chooser);
+    }
+  } else {
+    for (var i = 0; i < this.params.name.length; i++) {
+      if (typeof this.params.name[i] == "string") {
+        nameFuncs.push(function(i, metric) {
+          return PromConsole._interpolateName(this.params.name[i], metric);
+        }.bind(this, i));
+      } else {
+        nameFuncs.push(this.params.name[i]);
+      }
+    }
+  }
+
+  // Get the data into the right format.
+  var seriesLen = 0;
+
+  for (var e = 0; e < data.length; e++) {
+    for (var i = 0; i < data[e].data.result.length; i++) {
+      series[seriesLen] = {
+        data: data[e].data.result[i].values.map(function(s) { return { x: s[0], y: self._parseValue(s[1]) }; }),
+        color: palette.color(),
+        name: self._escapeHTML(nameFuncs[e](data[e].data.result[i].metric)),
+      };
+      // Insert nulls for all missing steps.
+      var newSeries = [];
+      var pos = 0;
+      var start = self.params.endTime - self.params.duration;
+      var step = Math.floor(self.params.duration / this.graphTd.offsetWidth * 1000) / 1000;
+      for (var t = start; t <= self.params.endTime; t += step) {
+        // Allow for floating point inaccuracy.
+        if (series[seriesLen].data.length > pos && series[seriesLen].data[pos].x < t + step / 100) {
+          newSeries.push(series[seriesLen].data[pos]);
+          pos++;
+        } else {
+          newSeries.push({ x: t, y: null });
+        }
+      }
+      series[seriesLen].data = newSeries;
+      seriesLen++;
+    }
+  }
+  this._clearGraph();
+  if (!series.length) {
+    var errorText = document.createElement("div");
+    errorText.className = 'prom_graph_error';
+    errorText.textContent = 'No timeseries returned';
+    this.graphTd.appendChild(errorText);
+    return;
+  }
+  // Render.
+  var graph = new Rickshaw.Graph({
+    interpolation: "linear",
+    width: this.graphTd.offsetWidth,
+    height: this.params.height,
+    element: this.graphTd,
+    renderer: this.params.renderer,
+    max: this.params.max,
+    min: this.params.min,
+    series: series
+  });
+  var hoverDetail = new Rickshaw.Graph.HoverDetail({
+    graph: graph,
+    onRender: function() {
+      var xLabel = this.element.getElementsByClassName("x_label")[0];
+      var item = this.element.getElementsByClassName("item")[0];
+      if (xLabel.offsetWidth + xLabel.offsetLeft + this.element.offsetLeft > graph.element.offsetWidth ||
+        item.offsetWidth + item.offsetLeft + this.element.offsetLeft > graph.element.offsetWidth) {
+        xLabel.classList.add("prom_graph_hover_flipped");
+        item.classList.add("prom_graph_hover_flipped");
+      } else {
+        xLabel.classList.remove("prom_graph_hover_flipped");
+        item.classList.remove("prom_graph_hover_flipped");
+      }
+    },
+    yFormatter: function(y) {
+      if (y === null) {
+        return "";
+      }
+      return this.params.yHoverFormatter(y) + this.params.yUnits;
+    }.bind(this)
+  });
+  var yAxis = new Rickshaw.Graph.Axis.Y({
+    graph: graph,
+    tickFormat: this.params.yAxisFormatter,
+    ticks: this.params.yAxisTicks
+  });
+  var xAxis = new Rickshaw.Graph.Axis.Time({
+    graph: graph,
+  });
+  var legend = new Rickshaw.Graph.Legend({
+    graph: graph,
+    element: this.legendDiv
+  });
+  xAxis.render();
+  yAxis.render();
+  graph.render();
+
+  this.rickshawGraph = graph;
+};
+
+PromConsole.Graph.prototype._clearGraph = function() {
+  while (this.graphTd.lastChild) {
+    this.graphTd.removeChild(this.graphTd.lastChild);
+  }
+  while (this.legendDiv.lastChild) {
+    this.legendDiv.removeChild(this.legendDiv.lastChild);
+  }
+  this.rickshawGraph = null;
+};
+
+PromConsole.Graph.prototype._xhrs = [];
+
+PromConsole.Graph.prototype.buildQueryUrl = function(expr) {
+  var p = this.params;
+  return PATH_PREFIX + "/api/v1/query_range?query=" +
+    encodeURIComponent(expr) +
+    "&step=" + Math.floor(p.duration / this.graphTd.offsetWidth * 1000) / 1000 +
+    "&start=" + (p.endTime - p.duration) + "&end=" + p.endTime;
+};
+
+PromConsole.Graph.prototype.dispatch = function() {
+  for (var j = 0; j < this._xhrs.length; j++) {
+    this._xhrs[j].abort();
+  }
+  var all_data = new Array(this.params.expr.length);
+  this._xhrs = new Array(this.params.expr.length);
+  var pending_requests = this.params.expr.length;
+  for (var i = 0; i < this.params.expr.length; ++i) {
+    var endTime = this.params.endTime;
+    var url = this.buildQueryUrl(this.params.expr[i]);
+    var xhr = new XMLHttpRequest();
+    xhr.open('get', url, true);
+    xhr.responseType = 'json';
+    xhr.onerror = function(xhr, i) {
+      this._clearGraph();
+      var errorText = document.createElement("div");
+      errorText.className = 'prom_graph_error';
+      errorText.textContent = 'Error loading data';
+      this.graphTd.appendChild(errorText);
+      console.log('Error loading data for ' + this.params.expr[i]);
+      pending_requests = 0;
+      // onabort gets any aborts.
+      for (var j = 0; j < pending_requests; j++) {
+        this._xhrs[j].abort();
+      }
+    }.bind(this, xhr, i);
+    xhr.onload = function(xhr, i) {
+      if (pending_requests === 0) {
+        // Got an error before this success.
+        return;
+      }
+      var data = xhr.response;
+      if (typeof data !== "object") {
+        data = JSON.parse(xhr.responseText);
+      }
+      pending_requests -= 1;
+      all_data[i] = data;
+      if (pending_requests === 0) {
+        this._xhrs = [];
+        this._render(all_data);
+      }
+    }.bind(this, xhr, i);
+    xhr.send();
+    this._xhrs[i] = xhr;
+  }
+
+  var loadingImg = document.createElement("img");
+  loadingImg.src = PATH_PREFIX + '/user/ajax-loader.gif';
+  loadingImg.alt = 'Loading...';
+  loadingImg.className = 'prom_graph_loading';
+  this.graphTd.appendChild(loadingImg);
+};
+
+// Substitute the value of 'label' for [[ label ]].
+PromConsole._interpolateName = function(name, metric) {
+  var re = /(.*?)\[\[\s*(\w+)+\s*\]\](.*?)/g;
+  var result = '';
+  while (match = re.exec(name)) {
+    result = result + match[1] + metric[match[2]] + match[3];
+  }
+  if (!result) {
+    return name;
+  }
+  return result;
+};
+
+// Given the data returned by the API, return an appropriate function
+// to return plot names.
+PromConsole._chooseNameFunction = function(data) {
+  // By default, use the full metric name.
+  var nameFunc = function(metric) {
+    var name = metric.__name__ + "{";
+    for (var label in metric) {
+      if (label.substring(0, 2) == "__") {
+        continue;
+      }
+      name += label + "='" + metric[label] + "',";
+    }
+    return name + "}";
+  };
+
+  // If only one label varies, use that value.
+  var labelValues = {};
+  for (var e = 0; e < data.length; e++) {
+    for (var i = 0; i < data[e].data.result.length; i++) {
+      for (var label in data[e].data.result[i].metric) {
+        if (!(label in labelValues)) {
+          labelValues[label] = {};
+        }
+        labelValues[label][data[e].data.result[i].metric[label]] = 1;
+      }
+    }
+  }
+
+  var multiValueLabels = [];
+  for (var label in labelValues) {
+    if (Object.keys(labelValues[label]).length > 1) {
+      multiValueLabels.push(label);
+    }
+  }
+  if (multiValueLabels.length == 1) {
+    nameFunc = function(metric) {
+      return metric[multiValueLabels[0]];
+    };
+  }
+  return nameFunc;
+};
+
+// Given a list of expressions, produce the /graph url for them.
+PromConsole._graphsToSlashGraphURL = function(exprs) {
+  var data = [];
+  for (var i = 0; i < exprs.length; ++i) {
+    data.push({ 'expr': exprs[i], 'tab': 0 });
+  }
+  return PATH_PREFIX + '/graph#' + encodeURIComponent(JSON.stringify(data));
+
+};
diff --git a/bundles/prometheus/files/console_libraries/menu.lib b/bundles/prometheus/files/console_libraries/menu.lib
index dc87f9f..d9e4baf 100644
--- a/bundles/prometheus/files/console_libraries/menu.lib
+++ b/bundles/prometheus/files/console_libraries/menu.lib
@@ -1,64 +1,51 @@
-{{/* vim: set ft=html: */}}
+{{/* vim: set ft=gotmpl: */}}
 
 {{/* Navbar, should be passed . */}}
 {{ define "navbar" }}
-<nav class="navbar fixed-top navbar-expand-sm navbar-dark bg-dark">
-  <div class="container-fluid">
-    <!-- Brand and toggle get grouped for better mobile display -->
-    <div class="navbar-header">
-      <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"
-        aria-expanded="false" aria-controls="navbar-nav" aria-label="toggle navigation">
-        <span class="navbar-toggler-icon"></span>
-      </button>
-      <a class="navbar-brand" href="{{ pathPrefix }}/">Prometheus</a>
-    </div>
-
-    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
-      <ul class="nav navbar-nav">
-        <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/alerts">Alerts</a></li>
-      </ul>
-    </div>
+<nav class="navbar sticky-top navbar-expand-sm navbar-dark bg-dark">
+  <!-- Brand and toggle get grouped for better mobile display -->
+  <div class="navbar-header">
+    <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false" aria-controls="navbar-nav" aria-label="toggle navigation">
+      <span class="navbar-toggler-icon"></span>
+    </button>
+    <a class="navbar-brand" href="{{ pathPrefix }}/">Prometheus</a>
+  </div>
+  <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+    <ul class="nav navbar-nav">
+      <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/alerts">Alerts</a></li>
+      <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/graph">Graph</a></li>
+      <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/targets">Targets</a></li>
+    </ul>
   </div>
 </nav>
 {{ end }}
 
 {{/* LHS menu, should be passed . */}}
 {{ define "menu" }}
-<div class="prom_lhs_menu row">
-  <nav class="col-md-2 md-block bg-dark sidebar prom_lhs_menu_nav">
-    <div class="sidebar-sticky">
-      <ul class="nav flex-column">
-
-        {{ template "_menuItem" (args . "index.html" "Overview") }}
+<nav aria-label="breadcrumb">
+  <ol class="breadcrumb rounded-0">
+    {{ template "_menuItem" (args . "index.html" "overview") }}
 
-        {{ $ctx := . }}
+    {{ $ctx := . }}
 
-        {{ range query "up{job='node-exporter'}" | sortByLabel "instance" }}
-        {{ $instance := .Labels.instance }}
-        <ul>
-          {{ template "_menuItem" (args $ctx (printf "node-overview.html?instance=%v" $instance) $instance) }}
-          <ul>
-            {{ template "_menuItem" (args $ctx (printf "node-cpu.html?instance=%v" $instance) "CPU") }}
-            {{ template "_menuItem" (args $ctx (printf "node-disk.html?instance=%v" $instance) "Disk") }}
-          </ul>
-        </ul>
-        {{ end }}
-
-        {{/* XXX: there is only one prometheus instance in my setup */}}
-        {{ with query "up{job='prometheus'}" | first }}
-        {{ template "_menuItem" (args $ctx (printf "prometheus-overview.html?instance=%v" .Labels.instance) "Prometheus") }}
-        {{ end }}
-
-      </ul>
-    </div>
-  </nav>
-</div>
+    {{ template "_menuItem" (args $ctx "node.html?instance=inazuma" "inazuma") }}
+    {{ template "_menuItem" (args $ctx "node.html?instance=shimakaze" "shimakaze") }}
+    {{ template "_menuItem" (args $ctx "prom.html?instance=inazuma" "prometheus") }}
+  </ol>
+</nav>
 {{ end }}
 
 {{/* Helper, pass (args . path name) */}}
 {{ define "_menuItem" }}
 {{ $pagelink := .arg0.Path }}
 {{ if .arg0.Params.instance }}{{ $pagelink = (printf "%v?instance=%v" .arg0.Path .arg0.Params.instance) }}{{ end }}
-<li class="{{ if eq $pagelink .arg1 }}prom_lhs_menu_selected {{ end }}nav-item"><a class="nav-link"
-    href="{{ .arg1 }}">{{ .arg2 }}</a></li>
+<li class="breadcrumb-item{{ if eq $pagelink .arg1 }} active{{ end }} text-lowercase">
+  <a href="{{ .arg1 }}">
+    {{ if eq .arg2 "overview" }}
+    <i class="bi bi-grid-3x3-gap-fill" title="{{ .arg2 }}"></i>
+    {{ else }}
+    {{ .arg2 }}
+    {{ end }}
+  </a>
+</li>
 {{ end }}
diff --git a/bundles/prometheus/files/console_libraries/prom.lib b/bundles/prometheus/files/console_libraries/prom.lib
index f934d43..c00bb6b 100644
--- a/bundles/prometheus/files/console_libraries/prom.lib
+++ b/bundles/prometheus/files/console_libraries/prom.lib
@@ -1,35 +1,76 @@
-{{/* vim: set ft=html: */}}
+{{/* vim: set ft=gotmpl: */}}
 {{/* Load Prometheus console library JS/CSS. Should go in head */}}
 
 {{ define "prom_console_head" }}
-<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.css">
-<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/css/bootstrap.min.css">
-<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/css/prom_console.css">
-<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css">
-<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.v3.js"></script>
-<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.layout.min.js"></script>
-<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.js"></script>
-<script src="{{ pathPrefix }}/classic/static/vendor/js/jquery-3.5.1.min.js"></script>
-<script src="{{ pathPrefix }}/classic/static/vendor/js/popper.min.js"></script>
-<script src="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/js/bootstrap.min.js"></script>
-
+<meta name="viewport" content="width=device-width,initial-scale=1.0">
+<meta name="title" content="Prometheus &mdash; Console">
+<title>Prometheus &mdash; Console</title>
+<!-- rickshaw -->
+<script src="https://unpkg.com/rickshaw@1.7.1/vendor/d3.v3.js"></script>
+<script src="https://unpkg.com/rickshaw@1.7.1/vendor/d3.layout.min.js"></script>
+<link type="text/css" rel="stylesheet" href="https://unpkg.com/rickshaw@1.7.1/rickshaw.min.css">
+<script src="https://unpkg.com/rickshaw@1.7.1/rickshaw.min.js"></script>
+<!-- bootstrap -->
+<script src="https://unpkg.com/jquery@3.5.1/dist/jquery.min.js"></script>
+<link type="text/css" rel="stylesheet" href="https://unpkg.com/bootstrap@4.5.2/dist/css/bootstrap.min.css">
+<script src="https://unpkg.com/bootstrap@4.5.2/dist/js/bootstrap.bundle.min.js"></script>
+<!-- icons -->
+<link type="text/css" rel="stylesheet" href="https://unpkg.com/bootstrap-icons@1.13.1/font/bootstrap-icons.min.css">
+<!-- user assets -->
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/user/prom_console.css">
+<script src="{{ pathPrefix }}/user/prom_console.js"></script>
+<style type="text/css">
+  body {
+    background-color: #191d21 !important;
+    color: #fff !important;
+  }
+  .navbar {
+    margin-bottom: unset !important;
+    padding-left: 15px !important;
+    padding-right: 15px !important;
+  }
+  .prom_console_content {
+    padding-top: unset !important;
+    margin-bottom: unset !important;
+  }
+  .prom_content_div {
+    width: unset !important;
+    display: grid;
+    grid-template-columns: 1fr 1fr;
+    gap: 15px;
+  }
+  @media (max-width: 768px) {
+    .prom_content_div {
+      grid-template-columns: 1fr;
+    }
+  }
+  .prom_content_div > div {
+    padding: 10px;
+    border: 1px solid var(--gray);
+  }
+  .prom_content_div * {
+    background-color: var(--light) !important;
+    color: var(--dark) !important;
+  }
+  .prom_graph_table {
+    overflow: hidden;
+  }
+</style>
 <script>
-var PATH_PREFIX = "{{ pathPrefix }}";
+  var PATH_PREFIX = "{{ pathPrefix }}";
 </script>
-<script src="{{ pathPrefix }}/classic/static/js/prom_console.js"></script>
 {{ end }}
 
 {{/* Top of all pages. */}}
 {{ define "head" -}}
-<!doctype html>
+<!DOCTYPE html>
 <html lang="en">
-<head>
-{{ template "prom_console_head" }}
-</head>
-<body>
-{{ template "navbar" . }}
-
-{{ template "menu" . }}
+  <head>
+    {{ template "prom_console_head" }}
+  </head>
+  <body>
+    {{ template "navbar" . }}
+    {{ template "menu" . }}
 {{ end }}
 
 {{ define "__prom_query_drilldown_noop" }}{{ . }}{{ end }}
@@ -49,91 +90,64 @@ renderTemplate is the name of the template to use to render the value.
 */}}
 {{ define "prom_query_drilldown" }}
 {{ $expr := .arg0 }}{{ $suffix := (or .arg1 "") }}{{ $renderTemplate := (or .arg2 "__prom_query_drilldown_noop") }}
-<a class="prom_query_drilldown" href="{{ pathPrefix }}{{ graphLink $expr }}">{{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }}</a>
+<a class="prom_query_drilldown" href="{{ pathPrefix }}{{ graphLink $expr }}"> {{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }}</a>
 {{ end }}
 
 {{ define "prom_path" }}/consoles/{{ .Path }}?{{ range $param, $value := .Params }}{{ $param }}={{ $value }}&amp;{{ end }}{{ end }}"
 
-{{ define "prom_right_table_head" }}
-<div class="prom_console_rhs">
-<table class="table table-bordered table-hover table-sm">
-{{ end }}
-{{ define "prom_right_table_tail" }}
-</table>
-</div>
-{{ end }}
-
-{{/* RHS table head, pass job name. Should be used after prom_right_table_head. */}}
-{{ define "prom_right_table_job_head" }}
-<tr>
-  <th>{{  .  }}</th>
-  <th>{{  template "prom_query_drilldown" (args (printf "sum(up{job='%s'})" .))  }} / {{ template "prom_query_drilldown" (args (printf "count(up{job='%s'})" .)) }}</th>
-</tr>
-<tr>
-  <td>CPU</td>
-  <td>{{  template "prom_query_drilldown" (args (printf "avg by(job)(irate(process_cpu_seconds_total{job='%s'}[5m]))" .) "s/s" "humanizeNoSmallPrefix") }}</td>
-</tr>
-<tr>
-  <td>Memory</td>
-  <td>{{  template "prom_query_drilldown" (args (printf "avg by(job)(process_resident_memory_bytes{job='%s'})" .) "B" "humanize1024") }}</td>
-</tr>
-{{ end }}
-
-
 {{ define "prom_content_head" }}
-<div class="prom_console_content">
-  <div class="container-fluid">
-{{ template "prom_graph_timecontrol" . }}
+    <div class="prom_console_content">
+      {{ template "prom_graph_timecontrol" . }}
 {{ end }}
 {{ define "prom_content_tail" }}
-  </div>
-</div>
+    </div>
 {{ end }}
 
 {{ define "prom_graph_timecontrol" }}
-<div class="prom_graph_timecontrol">
-  <div class="prom_graph_timecontrol_inner">
-    <div class="prom_graph_timecontrol_group ">
-      <button class="btn btn-light pull-left" type="button" id="prom_graph_duration_shrink" title="Shrink the time range.">
-        <i class="glyphicon glyphicon-minus"></i>
-      </button><!-- Comments between elements to remove spaces
+    <div class="prom_graph_timecontrol">
+      <div class="prom_graph_timecontrol_inner">
+        <div class="prom_graph_timecontrol_group ">
+          <button class="btn btn-light pull-left" type="button" id="prom_graph_duration_shrink" title="Shrink the time range.">
+            <i class="bi bi-dash-lg"></i>
+          </button><!-- Comments between elements to remove spaces
    --><input class="input pull-left align-middle" size="3" title="Time range of graph" type="text" id="prom_graph_duration"><!--
    --><button class="btn btn-light pull-left" type="button" id="prom_graph_duration_grow" title="Grow the time range.">
-        <i class="glyphicon glyphicon-plus"></i>
-      </button>
-    </div>
-    <div class="prom_graph_timecontrol_group ">
-      <button class="btn btn-light pull-left" type="button" id="prom_graph_time_back" title="Rewind the end time.">
-        <i class="glyphicon glyphicon-backward"></i>
-      </button><!--
+            <i class="bi bi-plus-lg"></i>
+          </button>
+        </div>
+        <div class="prom_graph_timecontrol_group ">
+          <button class="btn btn-light pull-left" type="button" id="prom_graph_time_back" title="Rewind the end time.">
+            <i class="bi bi-arrow-left"></i>
+          </button><!--
    --><input class="input pull-left align-middle" title="End time of graph" placeholder="Until" type="text" id="prom_graph_time_end" size="16" value=""><!--
     --><button class="btn btn-light pull-left" type="button" id="prom_graph_time_forward" title="Advance the end time.">
-        <i class="glyphicon glyphicon-forward"></i>
-      </button>
-    </div>
-    <div class="prom_graph_timecontrol_group ">
-      <div class="btn-group dropup prom_graph_timecontrol_refresh pull-left">
-        <button type="button" class="btn btn-light pull-left" id="prom_graph_refresh_button" title="Refresh.">
-          <i class="glyphicon glyphicon-repeat"></i>
-          <span class="icon-repeat"></span>
-          (<span id="prom_graph_refresh_button_value">Off</span>)
-        </button>
-        <button type="button" class="btn btn-light pull-left dropdown-toggle" data-toggle="dropdown" title="Set autorefresh." aria-haspopup="true" aria-expanded="false">
-          <span class="caret"></span>&nbsp;
-        </button>
-        <ul class="dropdown-menu" id="prom_graph_refresh_intervals" role="menu">
-        </ul>
+            <i class="bi bi-arrow-right"></i>
+          </button>
+        </div>
+        <div class="prom_graph_timecontrol_group ">
+          <div class="btn-group dropup prom_graph_timecontrol_refresh pull-left">
+            <button type="button" class="btn btn-light pull-left" id="prom_graph_refresh_button" title="Refresh.">
+              <i class="bi bi-arrow-clockwise"></i>
+              <span class="icon-repeat"></span>
+              (<span id="prom_graph_refresh_button_value">Off</span>)
+            </button>
+            <button type="button" class="btn btn-light pull-left" data-toggle="dropdown" title="Set autorefresh." aria-haspopup="true" aria-expanded="false">
+              <i class="bi bi-caret-up-fill"></i>
+            </button>
+            <ul class="dropdown-menu" id="prom_graph_refresh_intervals" role="menu">
+            </ul>
+          </div>
+        </div>
       </div>
+      <script>
+        new PromConsole.TimeControl();
+      </script>
     </div>
-  </div>
-  <script>
-  new PromConsole.TimeControl();
-  </script>
-</div>
 {{ end }}
 
 {{/* Bottom of all pages. */}}
+
 {{ define "tail" }}
-</body>
+  </body>
 </html>
 {{ end }}
diff --git a/bundles/prometheus/files/consoles/node-cpu.html b/bundles/prometheus/files/consoles/cpu.html
index 64a2cad..64a2cad 100644
--- a/bundles/prometheus/files/consoles/node-cpu.html
+++ b/bundles/prometheus/files/consoles/cpu.html
diff --git a/bundles/prometheus/files/consoles/node-disk.html b/bundles/prometheus/files/consoles/disk.html
index 52afac7..52afac7 100644
--- a/bundles/prometheus/files/consoles/node-disk.html
+++ b/bundles/prometheus/files/consoles/disk.html
diff --git a/bundles/prometheus/files/consoles/index.html b/bundles/prometheus/files/consoles/index.html
index 29c2884..f0b102d 100644
--- a/bundles/prometheus/files/consoles/index.html
+++ b/bundles/prometheus/files/consoles/index.html
@@ -1,81 +1,102 @@
-{{/* vim: set ft=html: */}}
+{{/* vim: set ft=gotmpl: */}}
 
-{{ define "menu" }}
-foobar
+{{ define "s6_rc_services_count" }}
+{{ $instance := .arg0 }}{{ $bundle := .arg1 }}
+{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{instance='%s',bundle='%s',service=~'.*-srv'}" $instance $bundle)) }}
 {{ end }}
 
 {{ template "head" . }}
 
-{{ template "prom_right_table_head" . }}
-<tr>
-	<th colspan="2">S6 - inazuma</th>
-</tr>
-<tr>
-	<td>default</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='default'}")) }}</td>
-</tr>
-<tr>
-	<td>containers</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='containers'}")) }}</td>
-</tr>
-<tr>
-	<td>www</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='www'}")) }}</td>
-</tr>
-<tr>
-	<td>services</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='services'}")) }}</td>
-</tr>
-<tr>
-	<th colspan="2">S6 - shimakaze</th>
-</tr>
-<tr>
-	<td>default</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='default'}")) }}</td>
-</tr>
-<tr>
-	<td>containers</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='containers'}")) }}</td>
-</tr>
-<tr>
-	<td>www</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='www'}")) }}</td>
-</tr>
-<tr>
-	<td>services</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='services'}")) }}</td>
-</tr>
-{{ template "prom_right_table_tail" . }}
-
 {{ template "prom_content_head" . }}
 <div class="prom_content_div">
-	<h1>Overview</h1>
-	<h3>Traefik Requests OK - inazuma</h3>
-	<div id="traefikGraph"></div>
-	<script>
-		new PromConsole.Graph({
-			node: document.querySelector("#traefikGraph"),
-			expr: "irate(traefik_service_request_duration_seconds_count{job='traefik',instance='inazuma',code='200'}[5m])",
-			name: '[[service]]',
-			yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-			yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-			yTitle: "Requests",
-			yUnits: "/s",
-		})
-	</script>
-	<h3>Jellyfin HTTP Requests - shimakaze</h3>
-	<div id="jellyfinGraph"></div>
-	<script>
-		new PromConsole.Graph({
-			node: document.querySelector("#jellyfinGraph"),
-			expr: "irate(http_request_duration_seconds_count{job='jellyfin',instance='shimakaze'}[5m])",
-			name: '[[action]]',
-			yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-			yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-			yTitle: "Requests",
-			yUnits: "/s",
-		})
-	</script>
+  <div>
+    <h2>Overview</h2>
+    <code>IZ: {{ template "s6_rc_services_count" (args "inazuma" "default") }}</code>
+    <code>SK: {{ template "s6_rc_services_count" (args "shimakaze" "default") }}</code>
+  </div>
+  <div>
+    <h4>Traefik Latency</h4>
+    <div id="traefikLatencyGraph"></div>
+    <script>
+      new PromConsole.Graph({
+        node: document.querySelector("#traefikLatencyGraph"),
+        expr: [
+          `histogram_quantile(0.99, sum(rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~"https?"}[10m])) by (le, instance))`,
+        ],
+        name: '[[instance]]',
+        yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+        yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+        yTitle: "Latency (p99)",
+        yUnits: "s",
+      })
+    </script>
+  </div>
+  <div>
+    <h4>Node CPU Utilisation</h4>
+    <div id="cpuUtilGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#cpuUtilGraph"),
+      expr: [
+        `(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[1m]))) * 100.0`,
+      ],
+      name: '[[instance]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yUnits: "%",
+      min: 0,
+      max: 100,
+    })
+    </script>
+  </div>
+  <div>
+    <h4>Node CPU Saturation</h4>
+    <div id="cpuSatGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#cpuSatGraph"),
+      expr: [
+        `sum by (instance) (node_load15) / count by (instance) (sum by (instance, cpu) (node_cpu_seconds_total))`,
+      ],
+      name: '[[instance]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    })
+    </script>
+  </div>
+  <div>
+    <h4>Node Memory Utilisation</h4>
+    <div id="memUtilGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#memUtilGraph"),
+      expr: [
+        `(1 - sum by (instance) (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / sum by (instance) (node_memory_MemTotal_bytes)) * 100.0`,
+      ],
+      name: '[[instance]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yUnits: "%",
+      min: 0,
+      max: 100,
+    })
+    </script>
+  </div>
+  <div>
+    <h4>Node Memory Saturation</h4>
+    <div id="memSatGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#memSatGraph"),
+      expr: [
+        `1e3 * sum by (instance) (rate(node_vmstat_pgpgin[2m]) + rate(node_vmstat_pgpgout[2m]))`,
+      ],
+      name: '[[instance]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    })
+    </script>
+  </div>
 </div>
 {{ template "prom_content_tail" . }}
 
diff --git a/bundles/prometheus/files/consoles/node-overview.html b/bundles/prometheus/files/consoles/node-overview.html
deleted file mode 100644
index edeca2e..0000000
--- a/bundles/prometheus/files/consoles/node-overview.html
+++ /dev/null
@@ -1,121 +0,0 @@
-{{ template "head" . }}
-
-{{ template "prom_right_table_head" }}
-  <tr><th colspan="2">Overview</th></tr>
-  <tr>
-    <td>User CPU</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
-  </tr>
-  <tr>
-    <td>System CPU</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
-  </tr>
-  <tr>
-    <td>Memory Total</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
-  </tr>
-  <tr>
-    <td>Memory Free</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
-  </tr>
-<tr>
-  <th colspan="2">Network</th>
-</tr>
-{{ range printf "node_network_receive_bytes_total{job='node-exporter',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }}
-  <tr>
-    <td>{{ .Labels.device }} Received</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
-  </tr>
-  <tr>
-    <td>{{ .Labels.device }} Transmitted</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
-  </tr>
-{{ end }}
-<tr>
-  <th colspan="2">Disks</th>
-</tr>
-{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }}
-  <tr>
-    <td>{{ .Labels.device }} Utilization</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
-  </tr>
-{{ end }}
-{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
-  <tr>
-    <td>{{ .Labels.device }} Throughput</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
-  </tr>
-{{ end }}
-<tr>
-  <th colspan="2">Filesystem Fullness</th>
-</tr>
-{{ define "roughlyNearZero" }}
-{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }}
-{{ end }}
-{{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }}
-  <tr>
-    <td>{{ .Labels.mountpoint }}</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td>
-  </tr>
-{{ end }}
-
-{{ template "prom_right_table_tail" }}
-
-{{ template "prom_content_head" . }}
-  <h1>Node Overview - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1>
-
-  <h3>CPU Usage</h3>
-  <div id="cpuGraph"></div>
-  <script>
-  new PromConsole.Graph({
-    node: document.querySelector("#cpuGraph"),
-    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
-    renderer: 'area',
-    max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
-    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-    yTitle: 'Cores'
-  })
-  </script>
-
-  <h3>Disk I/O Utilization</h3>
-  <div id="diskioGraph"></div>
-  <script>
-  new PromConsole.Graph({
-    node: document.querySelector("#diskioGraph"),
-    expr: [
-      "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',device!~'^(md\\\\d+$|dm-)'}[5m]) * 100",
-    ],
-    min: 0,
-    name: '[[ device ]]',
-    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-    yUnits: "%",
-    yTitle: 'Disk I/O Utilization'
-  })
-  </script>
-
-  <h3>Memory</h3>
-  <div id="memoryGraph"></div>
-  <script>
-  new PromConsole.Graph({
-    node: document.querySelector("#memoryGraph"),
-    renderer: 'area',
-    expr: [
-      "node_memory_Cached_bytes{job='node-exporter',instance='{{ .Params.instance }}'}",
-      "node_memory_Buffers_bytes{job='node-exporter',instance='{{ .Params.instance }}'}",
-      "node_memory_MemTotal_bytes{job='node-exporter',instance='{{ .Params.instance }}'} - node_memory_MemFree_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Buffers_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Cached_bytes{job='node-exporter',instance='{{.Params.instance}}'}",
-      "node_memory_MemFree{job='node-exporter',instance='{{ .Params.instance }}'}",
-    ],
-    name: ["Cached", "Buffers", "Used", "Free"],
-    min: 0,
-    yUnits: "B",
-    yAxisFormatter: PromConsole.NumberFormatter.humanize1024,
-    yHoverFormatter: PromConsole.NumberFormatter.humanize1024,
-    yTitle: 'Memory'
-  })
-  </script>
-
-{{ template "prom_content_tail" . }}
-
-{{ template "tail" }}
diff --git a/bundles/prometheus/files/consoles/node.html b/bundles/prometheus/files/consoles/node.html
new file mode 100644
index 0000000..9ef79ce
--- /dev/null
+++ b/bundles/prometheus/files/consoles/node.html
@@ -0,0 +1,133 @@
+{{/* vim: set ft=gotmpl: */}}
+
+{{ template "head" . }}
+
+{{ template "prom_content_head" . }}
+<div class="prom_content_div">
+  <div style="columns:2;">
+    <h1 style="margin-bottom:10px;">{{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1>
+    <table class="table">
+      <thead class="thead-dark">
+        <tr><th colspan="2">Overview</th></tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td>User CPU</td>
+          <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
+        </tr>
+        <tr>
+          <td>System CPU</td>
+          <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
+        </tr>
+        <tr>
+          <td>Memory Total</td>
+          <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
+        </tr>
+        <tr>
+          <td>Memory Free</td>
+          <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
+        </tr>
+      </tbody>
+    </table>
+    <table class="table">
+      <tr><th colspan="2">Network</th></tr>
+      {{ range printf "node_network_receive_bytes_total{job='node-exporter',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }}
+      <tr>
+        <td>{{ .Labels.device }} Received</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+      </tr>
+      <tr>
+        <td>{{ .Labels.device }} Transmitted</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+      </tr>
+      {{ end }}
+    </table>
+    <table class="table">
+      <tr><th colspan="2">Disks</th></tr>
+      {{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }}
+      <tr>
+        <td>{{ .Labels.device }} Utilization</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
+      </tr>
+      {{ end }}
+      {{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
+      <tr>
+        <td>{{ .Labels.device }} Throughput</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+      </tr>
+      {{ end }}
+    </table>
+    <table class="table">
+      <tr><th colspan="2">Filesystem Fullness</th></tr>
+      {{ define "roughlyNearZero" }}
+      {{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }}
+      {{ end }}
+      {{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }}
+      <tr>
+        <td>{{ .Labels.mountpoint }}</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td>
+      </tr>
+      {{ end }}
+    </table>
+  </div>
+  <div>
+    <h3>CPU Usage</h3>
+    <div id="cpuGraph"></div>
+    <script>
+      new PromConsole.Graph({
+        node: document.querySelector("#cpuGraph"),
+        expr: [
+          `sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))`,
+        ],
+        renderer: 'area',
+        max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
+        yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+        yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+        yTitle: 'Cores'
+      })
+    </script>
+  </div>
+  <div>
+    <h3>Disk I/O Utilization</h3>
+    <div id="diskioGraph"></div>
+    <script>
+      new PromConsole.Graph({
+        node: document.querySelector("#diskioGraph"),
+        expr: [
+          `irate(node_disk_io_time_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',device!~'^(md\\\\d+$|dm-)'}[5m]) * 100`,
+        ],
+        min: 0,
+        name: '[[ device ]]',
+        yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+        yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+        yUnits: "%",
+        yTitle: 'Disk I/O Utilization'
+      })
+    </script>
+  </div>
+  <div>
+    <h3>Memory</h3>
+    <div id="memoryGraph"></div>
+    <script>
+      new PromConsole.Graph({
+        node: document.querySelector("#memoryGraph"),
+        renderer: 'area',
+        expr: [
+          `node_memory_Cached_bytes{job='node-exporter',instance='{{ .Params.instance }}'}`,
+          `node_memory_Buffers_bytes{job='node-exporter',instance='{{ .Params.instance }}'}`,
+          `node_memory_MemTotal_bytes{job='node-exporter',instance='{{ .Params.instance }}'} - node_memory_MemFree_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Buffers_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Cached_bytes{job='node-exporter',instance='{{.Params.instance}}'}`,
+          `node_memory_MemFree{job='node-exporter',instance='{{ .Params.instance }}'}`,
+        ],
+        name: ["Cached", "Buffers", "Used", "Free"],
+        min: 0,
+        yUnits: "B",
+        yAxisFormatter: PromConsole.NumberFormatter.humanize1024,
+        yHoverFormatter: PromConsole.NumberFormatter.humanize1024,
+        yTitle: 'Memory'
+      })
+    </script>
+  </div>
+</div>
+{{ template "prom_content_tail" . }}
+
+{{ template "tail" . }}
diff --git a/bundles/prometheus/files/consoles/prom.html b/bundles/prometheus/files/consoles/prom.html
new file mode 100644
index 0000000..4bc5eb7
--- /dev/null
+++ b/bundles/prometheus/files/consoles/prom.html
@@ -0,0 +1,88 @@
+{{/* vim: set ft=gotmpl: */}}
+
+{{ template "head" . }}
+
+{{ template "prom_content_head" . }}
+<div class="prom_content_div">
+  <div style="columns:2;">
+    <h1 style="margin-bottom:10px;">Prometheus</h1>
+    <table>
+      <tr><th colspan="2">Overview</th></tr>
+      <tr>
+        <td>CPU</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }}</td>
+      </tr>
+      <tr>
+        <td>Memory</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
+      </tr>
+      <tr>
+        <td>Version</td>
+        <td>{{ with query (printf "prometheus_build_info{job='prometheus',instance='%s'}" .Params.instance) }}{{. | first | label "version"}}{{end}}</td>
+      </tr>
+    </table>
+    <table>
+      <tr><th colspan="2">Storage</th></tr>
+      <tr>
+        <td>Ingested Samples</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }}</td>
+      </tr>
+      <tr>
+        <td>Head Series</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_head_series{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
+      </tr>
+      <tr>
+        <td>Blocks Loaded</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_blocks_loaded{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
+      </tr>
+    </table>
+    <table>
+      <tr><th colspan="2">Rules</th></tr>
+      <tr>
+        <td>Evaluation Duration</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_evaluator_duration_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
+      </tr>
+      <tr>
+        <td>Notification Latency</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_notifications_latency_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
+      </tr>
+      <tr>
+        <td>Notification Queue</td>
+        <td>{{ template "prom_query_drilldown" (args (printf "prometheus_notifications_queue_length{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
+      </tr>
+    </table>
+  </div>
+  <div>
+    <h3>Ingested Samples</h3>
+    <div id="samplesGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#samplesGraph"),
+      expr: "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='{{ .Params.instance }}'}[5m])",
+      name: 'Ingested Samples',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yTitle: "Samples",
+      yUnits: "/s",
+    })
+    </script>
+  </div>
+  <div>
+    <h3>HTTP Server</h3>
+    <div id="serverGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#serverGraph"),
+      expr: "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='{{ .Params.instance }}'}[5m])",
+      name: '[[handler]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yTitle: "Requests",
+      yUnits: "/s",
+    })
+    </script>
+  </div>
+</div>
+{{ template "prom_content_tail" . }}
+
+{{ template "tail" }}
diff --git a/bundles/prometheus/files/consoles/prometheus-overview.html b/bundles/prometheus/files/consoles/prometheus-overview.html
deleted file mode 100644
index 08e027d..0000000
--- a/bundles/prometheus/files/consoles/prometheus-overview.html
+++ /dev/null
@@ -1,96 +0,0 @@
-{{ template "head" . }}
-
-{{ template "prom_right_table_head" }}
-  <tr>
-    <th colspan="2">Overview</th>
-  </tr>
-  <tr>
-    <td>CPU</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }}</td>
-  </tr>
-  <tr>
-    <td>Memory</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
-  </tr>
-  <tr>
-    <td>Version</td>
-    <td>{{ with query (printf "prometheus_build_info{job='prometheus',instance='%s'}" .Params.instance) }}{{. | first | label "version"}}{{end}}</td>
-  </tr>
-
-  <tr>
-    <th colspan="2">Storage</th>
-  </tr>
-  <tr>
-    <td>Ingested Samples</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }}</td>
-  </tr>
-  <tr>
-    <td>Head Series</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_head_series{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
-  </tr>
-  <tr>
-    <td>Blocks Loaded</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_blocks_loaded{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
-  </tr>
-  <tr>
-    <th colspan="2">Rules</th>
-  </tr>
-  <tr>
-    <td>Evaluation Duration</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_evaluator_duration_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
-  </tr>
-  <tr>
-    <td>Notification Latency</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_notifications_latency_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
-  </tr>
-  <tr>
-    <td>Notification Queue</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "prometheus_notifications_queue_length{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
-  </tr>
-  <tr>
-    <th colspan="2">HTTP Server</th>
-  </tr>
-{{ range printf "prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s'}" .Params.instance | query | sortByLabel "handler" }}
-  <tr>
-    <td>{{ .Labels.handler }}</td>
-    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s',handler='%s'}[5m])" .Labels.instance .Labels.handler) "/s" "humanizeNoSmallPrefix") }}</td>
-  </tr>
-{{ end }}
-
-{{ template "prom_right_table_tail" }}
-
-{{ template "prom_content_head" . }}
- <div class="prom_content_div">
-  <h1>Prometheus Overview - {{ .Params.instance }}</h1>
-
-  <h3>Ingested Samples</h3>
-  <div id="samplesGraph"></div>
-  <script>
-  new PromConsole.Graph({
-    node: document.querySelector("#samplesGraph"),
-    expr: "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='{{ .Params.instance }}'}[5m])",
-    name: 'Ingested Samples',
-    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-    yTitle: "Samples",
-    yUnits: "/s",
-  })
-  </script>
-
-  <h3>HTTP Server</h3>
-  <div id="serverGraph"></div>
-  <script>
-  new PromConsole.Graph({
-    node: document.querySelector("#serverGraph"),
-    expr: "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='{{ .Params.instance }}'}[5m])",
-    name: '[[handler]]',
-    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-    yTitle: "Requests",
-    yUnits: "/s",
-  })
-  </script>
- </div>
-{{ template "prom_content_tail" . }}
-
-{{ template "tail" }}
diff --git a/bundles/prometheus/files/kube.yaml b/bundles/prometheus/files/kube.yaml
index 952f485..7a7f961 100644
--- a/bundles/prometheus/files/kube.yaml
+++ b/bundles/prometheus/files/kube.yaml
@@ -9,40 +9,97 @@ metadata:
     traefik.http.routers.prometheus.entrypoints: http,https
     traefik.http.routers.prometheus.rule: Host(`prom.gzr.im`)
     traefik.http.routers.prometheus.service: prometheus
+    traefik.http.routers.prometheus.middlewares: prometheus-consoles-redirect
     traefik.http.services.prometheus.loadbalancer.server.port: 9090
+    traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.regex: ^https://prom.gzr.im/consoles(|/)$
+    traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.replacement: https://prom.gzr.im/consoles/index.html
+    traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.permanent: true
 spec:
-  restartPolicy: Never
   dnsPolicy: Default
   dnsConfig:
     searches:
       - tail2d6ce.ts.net
+  securityContext:
+    runAsUser: 0
   containers:
   - name: prom
-    image: quay.io/prometheus/prometheus:v2.52.0
+    image: quay.io/prometheus/prometheus:v3.7.3
+    args:
+      - --config.file=/etc/prometheus/prometheus.yml
+      - --storage.tsdb.path=/prometheus
+      - --storage.tsdb.retention.time=30d
+      - --storage.tsdb.retention.size=6GB
+      - --web.page-title=Prometheus
+      - --web.console.templates=/usr/share/prometheus/consoles
+      - --web.console.libraries=/usr/share/prometheus/console_libraries
+      - --web.user-assets=/usr/share/prometheus/console_assets
     volumeMounts:
-    - mountPath: /etc/prometheus:z
+    - mountPath: /etc/prometheus
       name: config
       readOnly: true
-    - mountPath: /usr/share/prometheus/consoles:z
+    - mountPath: /usr/share/prometheus/consoles
       name: config
       subPath: consoles
       readOnly: true
-    - mountPath: /usr/share/prometheus/console_libraries:z
+    - mountPath: /usr/share/prometheus/console_libraries
       name: config
       subPath: console_libraries
       readOnly: true
+    - mountPath: /usr/share/prometheus/console_assets
+      name: config
+      subPath: console_assets
+      readOnly: true
+    - mountPath: /prometheus
+      name: prom-data
     ports:
     - containerPort: 9090
       protocol: TCP
+    livenessProbe:
+      exec:
+        command:
+        - sh
+        - -c
+        - wget -qO - http://localhost:9090/-/healthy | grep -q 'is Healthy'
     resources:
       limits:
         cpu: 100m
-        memory: 200Mi
+        memory: 256Mi
       requests:
         cpu: 25m
-        memory: 100Mi
+        memory: 128Mi
+
+  - name: alert
+    image: quay.io/prometheus/alertmanager:v0.29.0
+    volumeMounts:
+    - mountPath: /etc/alertmanager
+      name: config
+      readOnly: true
+    - mountPath: /alertmanager
+      name: alert-data
+    ports:
+    - containerPort: 9093
+      protocol: TCP
+    livenessProbe:
+      exec:
+        command:
+        - sh
+        - -c
+        - wget -qO - http://localhost:9093/-/healthy | grep -q 'OK'
+    resources:
+      limits:
+        cpu: 50m
+        memory: 64Mi
+
   volumes:
   - name: config
     hostPath:
       path: /etc/prometheus
       type: Directory
+  - name: prom-data
+    hostPath:
+      path: /var/lib/prometheus/prom
+      type: DirectoryOrCreate
+  - name: alert-data
+    hostPath:
+      path: /var/lib/prometheus/alert
+      type: DirectoryOrCreate
diff --git a/bundles/prometheus/files/prometheus.yml b/bundles/prometheus/files/prometheus.yml
index 0614c8c..38841e5 100644
--- a/bundles/prometheus/files/prometheus.yml
+++ b/bundles/prometheus/files/prometheus.yml
@@ -9,16 +9,24 @@ global:
   # label_name_length_limit: 200
   # label_value_length_limit: 200
 
+# TODO: pending release
+# storage:
+#   tsdb:
+#     retention:
+#       time: 30d
+#       size: 6GB
+
 rule_files:
-  - rule_*.yml
+  - /etc/prometheus/rules/*.json
+  - /etc/prometheus/rules/*.yaml
+  - /etc/prometheus/rules/*.yml
 
 scrape_configs:
-
   - job_name: prometheus
     static_configs:
-    - targets: [localhost:9090]
-      labels:
-        instance: inazuma
+      - targets: [localhost:9090]
+        labels:
+          instance: inazuma
 
   - job_name: service-discovery
     file_sd_configs:
@@ -26,3 +34,9 @@ scrape_configs:
           - /etc/prometheus/services/*.json
           - /etc/prometheus/services/*.yaml
           - /etc/prometheus/services/*.yml
+
+alerting:
+  alertmanagers:
+    - scheme: http
+      static_configs:
+        - targets: [localhost:9093]
diff --git a/bundles/prometheus/files/rules.yml b/bundles/prometheus/files/rules.yml
new file mode 100644
index 0000000..45b130a
--- /dev/null
+++ b/bundles/prometheus/files/rules.yml
@@ -0,0 +1,218 @@
+groups:
+  - name: Prometheus
+    rules:
+      - alert: PrometheusTargetMissing
+        expr: up == 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            Prometheus target missing (instance {{ $labels.instance }})
+          description: |-
+            A Prometheus target has disappeared. An exporter might be crashed.
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+  - name: Host
+    rules:
+      - alert: HostOutOfMemory
+        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < .10)
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host out of memory (instance {{ $labels.instance }})
+          description: |-
+            Node memory is filling up (< 10% left)
+              VALUE = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostMemoryUnderMemoryPressure
+        expr: (rate(node_vmstat_pgmajfault[5m]) > 1000)
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host memory under memory pressure (instance {{ $labels.instance }})
+          description: |-
+            The node is under heavy memory pressure. High rate of loading memory pages from disk.
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostUnusualDiskReadRate
+        expr: (rate(node_disk_io_time_seconds_total[5m]) > .80)
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host unusual disk read rate (instance {{ $labels.instance }})
+          description: |-
+            Disk is too busy (IO wait > 80%)
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostOutOfDiskSpace
+        expr: (node_filesystem_avail_bytes{fstype!~"^(fuse.*|-tmpfs|cifs|nfs)",mountpoint!="/media/usb"} / node_filesystem_size_bytes < .10 and on (instance, device, mountpoint) node_filesystem_readonly == 0)
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            Host out of disk space (instance {{ $labels.instance }})
+          description: |-
+            Disk is almost full (< 10% left)
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostHighCpuLoad
+        expr: 1 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > .80
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host high CPU load (instance {{ $labels.instance }})
+          description: |-
+            CPU load is > 80%
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostUnusualNetworkThroughputOut
+        expr: ((rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host unusual outbound network traffic (instance {{ $labels.instance }})
+          description: |-
+            Host transmit bandwidth is high (>80%)
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostUnusualNetworkThroughputIn
+        expr: ((rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host unusual inbound network traffic (instance {{ $labels.instance }})
+          description: |-
+            Host receive bandwidth is high (>80%)
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostServiceDown
+        expr: count(s6_rc_service_up{bundle="default", service=~".*-srv"}) by (instance, service) == 0
+        for: 0m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            s6-rc service down (instance {{ $labels.instance }})
+          description: |-
+            s6-rc service {{ $labels.service }} on {{ $labels.instance }} is down!
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+  - name: Traefik
+    rules:
+      - alert: TraefikHighHttp4xxErrorRateService
+        expr: sum(rate(traefik_service_requests_total{service!~".*go-away@.*",code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            Traefik high HTTP 4xx error rate service (instance {{ $labels.instance }})
+          description: |-
+            Traefik service 4xx error rate is above 5%
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: TraefikHighHttp5xxErrorRateService
+        expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            Traefik high HTTP 5xx error rate service (instance {{ $labels.instance }})
+          description: |-
+            Traefik service 5xx error rate is above 5%
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: TraefikLatencyHigh
+        expr: histogram_quantile(0.99, sum(rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~"https?"}[3m])) by (le, instance)) > 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Traefik latency high (instance {{ $labels.instance }})
+          description: |-
+            Traefik p99 latency is higher than 10 seconds
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+  - name: Podman
+    rules:
+      - alert: ContainerHighMemoryUsage
+        expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) > 80
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Container High Memory usage (instance {{ $labels.instance }})
+          description: |-
+            Container Memory usage is above 80%
+              VALUE = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: ContainerLowMemoryUsage
+        expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) < 20
+        for: 7d
+        labels:
+          severity: info
+        annotations:
+          summary: |-
+            Container Low Memory usage (instance {{ $labels.instance }})
+          description: |-
+            Container Memory usage is under 20% for 1 week. Consider reducing the allocated memory.
+              VALUE = {{ $value }}
+              LABELS = {{ $labels }}
+
+  - name: BadBots
+    rules:
+      - alert: BadBotsFail2BanRate
+        expr: rate(f2b_jail_banned_current[15m]) > 10
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Increased rate of bans
+          description: |-
+            Fail2Ban has banned more than 10 IPs in the last 15 minutes.
+              VALUE = {{ $value }}
+              LABELS = {{ $labels }}
+
+      # - alert: BadBotsGoAway...
+      #   expr: rate(f2b_jail_banned_current[15m]) > 10
+      #   for: 0m
+      #   labels:
+      #     severity: warning
+      #   annotations:
+      #     summary: |-
+      #       Increased rate of bans
+      #     description: |-
+      #       Fail2Ban has banned more than 10 IPs in the last 15 minutes.
+      #         VALUE = {{ $value }}
+      #         LABELS = {{ $labels }}
diff --git a/bundles/prometheus/items.py b/bundles/prometheus/items.py
index 234aca5..99aecbc 100644
--- a/bundles/prometheus/items.py
+++ b/bundles/prometheus/items.py
@@ -1,16 +1,34 @@
 from sys import exit
 from os.path import join
 
+actions = {
+    "reload_prometheus": {
+        "command": "podman kill -s HUP prometheus-prom",
+        "triggered": True,
+    },
+    "reload_alertmanager": {
+        "command": "podman kill -s HUP prometheus-alert",
+        "triggered": True,
+    },
+}
+
 files = {
     "/etc/deployments/prometheus/kube.yaml": {
-        "content_type": "text",
-        "source": "kube.yaml",
+        "content_type": "mako",
+        "context": node.metadata.get("containers")["prometheus"],
         "triggers": {"svc_s6rc:prometheus:restart"},
     },
     "/etc/prometheus/prometheus.yml": {
         "content_type": "text",
-        "source": "prometheus.yml",
-        "triggers": {"svc_s6rc:prometheus:restart"},
+        "triggers": {"action:reload_prometheus"},
+    },
+    "/etc/prometheus/rules/generic.yml": {
+        "source": "rules.yml",
+        "triggers": {"action:reload_prometheus"},
+    },
+    "/etc/prometheus/alertmanager.yml": {
+        "content_type": "mako",
+        "triggers": {"action:reload_alertmanager"},
     },
 }
 
@@ -26,11 +44,19 @@ repo.libs.gen.add_files_recursive(
     target_root="/etc/prometheus",
     files_root=join(repo.path, "bundles", "prometheus", "files"),
 )
+repo.libs.gen.add_files_recursive(
+    files,
+    join(repo.path, "bundles", "prometheus", "files", "console_assets"),
+    target_root="/etc/prometheus",
+    files_root=join(repo.path, "bundles", "prometheus", "files"),
+)
 
 directories = {
-        "/etc/prometheus/services": {},
-        "/etc/prometheus/consoles": {},
-        "/etc/prometheus/console_libraries": {},
+    "/etc/prometheus/services": {"purge": True},
+    "/etc/prometheus/rules": {"purge": True},
+    "/etc/prometheus/consoles": {"purge": True},
+    "/etc/prometheus/console_libraries": {"purge": True},
+    "/etc/prometheus/console_assets": {"purge": True},
 }
 
 # NOTE: services that want to have their metrics scraped need to expose them