initial commit

Signed-off-by: Robert Günzler <r@gnzler.io>

12 files changed, 780 insertions, 0 deletions

diff --git a/bundles/prometheus/files/console_libraries/menu.lib b/bundles/prometheus/files/console_libraries/menu.lib
new file mode 100644
index 0000000..dc87f9f
--- /dev/null
+++ b/bundles/prometheus/files/console_libraries/menu.lib
@@ -0,0 +1,64 @@
+{{/* vim: set ft=html: */}}
+
+{{/* Navbar, should be passed . */}}
+{{ define "navbar" }}
+<nav class="navbar fixed-top navbar-expand-sm navbar-dark bg-dark">
+  <div class="container-fluid">
+    <!-- Brand and toggle get grouped for better mobile display -->
+    <div class="navbar-header">
+      <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1"
+        aria-expanded="false" aria-controls="navbar-nav" aria-label="toggle navigation">
+        <span class="navbar-toggler-icon"></span>
+      </button>
+      <a class="navbar-brand" href="{{ pathPrefix }}/">Prometheus</a>
+    </div>
+
+    <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+      <ul class="nav navbar-nav">
+        <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/alerts">Alerts</a></li>
+      </ul>
+    </div>
+  </div>
+</nav>
+{{ end }}
+
+{{/* LHS menu, should be passed . */}}
+{{ define "menu" }}
+<div class="prom_lhs_menu row">
+  <nav class="col-md-2 md-block bg-dark sidebar prom_lhs_menu_nav">
+    <div class="sidebar-sticky">
+      <ul class="nav flex-column">
+
+        {{ template "_menuItem" (args . "index.html" "Overview") }}
+
+        {{ $ctx := . }}
+
+        {{ range query "up{job='node-exporter'}" | sortByLabel "instance" }}
+        {{ $instance := .Labels.instance }}
+        <ul>
+          {{ template "_menuItem" (args $ctx (printf "node-overview.html?instance=%v" $instance) $instance) }}
+          <ul>
+            {{ template "_menuItem" (args $ctx (printf "node-cpu.html?instance=%v" $instance) "CPU") }}
+            {{ template "_menuItem" (args $ctx (printf "node-disk.html?instance=%v" $instance) "Disk") }}
+          </ul>
+        </ul>
+        {{ end }}
+
+        {{/* XXX: there is only one prometheus instance in my setup */}}
+        {{ with query "up{job='prometheus'}" | first }}
+        {{ template "_menuItem" (args $ctx (printf "prometheus-overview.html?instance=%v" .Labels.instance) "Prometheus") }}
+        {{ end }}
+
+      </ul>
+    </div>
+  </nav>
+</div>
+{{ end }}
+
+{{/* Helper, pass (args . path name) */}}
+{{ define "_menuItem" }}
+{{ $pagelink := .arg0.Path }}
+{{ if .arg0.Params.instance }}{{ $pagelink = (printf "%v?instance=%v" .arg0.Path .arg0.Params.instance) }}{{ end }}
+<li class="{{ if eq $pagelink .arg1 }}prom_lhs_menu_selected {{ end }}nav-item"><a class="nav-link"
+    href="{{ .arg1 }}">{{ .arg2 }}</a></li>
+{{ end }}
diff --git a/bundles/prometheus/files/console_libraries/prom.lib b/bundles/prometheus/files/console_libraries/prom.lib
new file mode 100644
index 0000000..f934d43
--- /dev/null
+++ b/bundles/prometheus/files/console_libraries/prom.lib
@@ -0,0 +1,139 @@
+{{/* vim: set ft=html: */}}
+{{/* Load Prometheus console library JS/CSS. Should go in head */}}
+
+{{ define "prom_console_head" }}
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.css">
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/css/bootstrap.min.css">
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/css/prom_console.css">
+<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css">
+<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.v3.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.layout.min.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/js/jquery-3.5.1.min.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/js/popper.min.js"></script>
+<script src="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/js/bootstrap.min.js"></script>
+
+<script>
+var PATH_PREFIX = "{{ pathPrefix }}";
+</script>
+<script src="{{ pathPrefix }}/classic/static/js/prom_console.js"></script>
+{{ end }}
+
+{{/* Top of all pages. */}}
+{{ define "head" -}}
+<!doctype html>
+<html lang="en">
+<head>
+{{ template "prom_console_head" }}
+</head>
+<body>
+{{ template "navbar" . }}
+
+{{ template "menu" . }}
+{{ end }}
+
+{{ define "__prom_query_drilldown_noop" }}{{ . }}{{ end }}
+{{ define "humanize" }}{{ humanize . }}{{ end }}
+{{ define "humanizeNoSmallPrefix" }}{{ if and (lt . 1.0) (gt . -1.0) }}{{ printf "%.3g" . }}{{ else }}{{ humanize . }}{{ end }}{{ end }}
+{{ define "humanize1024" }}{{ humanize1024 . }}{{ end }}
+{{ define "humanizeDuration" }}{{ humanizeDuration . }}{{ end }}
+{{ define "humanizePercentage" }}{{ humanizePercentage . }}{{ end }}
+{{ define "humanizeTimestamp" }}{{ humanizeTimestamp . }}{{ end }}
+{{ define "printf.1f" }}{{ printf "%.1f" . }}{{ end }}
+{{ define "printf.3g" }}{{ printf "%.3g" . }}{{ end }}
+
+{{/* prom_query_drilldown (args expr suffix? renderTemplate?)
+Displays the result of the expression, with a link to /graph for it.
+
+renderTemplate is the name of the template to use to render the value.
+*/}}
+{{ define "prom_query_drilldown" }}
+{{ $expr := .arg0 }}{{ $suffix := (or .arg1 "") }}{{ $renderTemplate := (or .arg2 "__prom_query_drilldown_noop") }}
+<a class="prom_query_drilldown" href="{{ pathPrefix }}{{ graphLink $expr }}">{{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }}</a>
+{{ end }}
+
+{{ define "prom_path" }}/consoles/{{ .Path }}?{{ range $param, $value := .Params }}{{ $param }}={{ $value }}&amp;{{ end }}{{ end }}"
+
+{{ define "prom_right_table_head" }}
+<div class="prom_console_rhs">
+<table class="table table-bordered table-hover table-sm">
+{{ end }}
+{{ define "prom_right_table_tail" }}
+</table>
+</div>
+{{ end }}
+
+{{/* RHS table head, pass job name. Should be used after prom_right_table_head. */}}
+{{ define "prom_right_table_job_head" }}
+<tr>
+  <th>{{  .  }}</th>
+  <th>{{  template "prom_query_drilldown" (args (printf "sum(up{job='%s'})" .))  }} / {{ template "prom_query_drilldown" (args (printf "count(up{job='%s'})" .)) }}</th>
+</tr>
+<tr>
+  <td>CPU</td>
+  <td>{{  template "prom_query_drilldown" (args (printf "avg by(job)(irate(process_cpu_seconds_total{job='%s'}[5m]))" .) "s/s" "humanizeNoSmallPrefix") }}</td>
+</tr>
+<tr>
+  <td>Memory</td>
+  <td>{{  template "prom_query_drilldown" (args (printf "avg by(job)(process_resident_memory_bytes{job='%s'})" .) "B" "humanize1024") }}</td>
+</tr>
+{{ end }}
+
+
+{{ define "prom_content_head" }}
+<div class="prom_console_content">
+  <div class="container-fluid">
+{{ template "prom_graph_timecontrol" . }}
+{{ end }}
+{{ define "prom_content_tail" }}
+  </div>
+</div>
+{{ end }}
+
+{{ define "prom_graph_timecontrol" }}
+<div class="prom_graph_timecontrol">
+  <div class="prom_graph_timecontrol_inner">
+    <div class="prom_graph_timecontrol_group ">
+      <button class="btn btn-light pull-left" type="button" id="prom_graph_duration_shrink" title="Shrink the time range.">
+        <i class="glyphicon glyphicon-minus"></i>
+      </button><!-- Comments between elements to remove spaces
+   --><input class="input pull-left align-middle" size="3" title="Time range of graph" type="text" id="prom_graph_duration"><!--
+   --><button class="btn btn-light pull-left" type="button" id="prom_graph_duration_grow" title="Grow the time range.">
+        <i class="glyphicon glyphicon-plus"></i>
+      </button>
+    </div>
+    <div class="prom_graph_timecontrol_group ">
+      <button class="btn btn-light pull-left" type="button" id="prom_graph_time_back" title="Rewind the end time.">
+        <i class="glyphicon glyphicon-backward"></i>
+      </button><!--
+   --><input class="input pull-left align-middle" title="End time of graph" placeholder="Until" type="text" id="prom_graph_time_end" size="16" value=""><!--
+    --><button class="btn btn-light pull-left" type="button" id="prom_graph_time_forward" title="Advance the end time.">
+        <i class="glyphicon glyphicon-forward"></i>
+      </button>
+    </div>
+    <div class="prom_graph_timecontrol_group ">
+      <div class="btn-group dropup prom_graph_timecontrol_refresh pull-left">
+        <button type="button" class="btn btn-light pull-left" id="prom_graph_refresh_button" title="Refresh.">
+          <i class="glyphicon glyphicon-repeat"></i>
+          <span class="icon-repeat"></span>
+          (<span id="prom_graph_refresh_button_value">Off</span>)
+        </button>
+        <button type="button" class="btn btn-light pull-left dropdown-toggle" data-toggle="dropdown" title="Set autorefresh." aria-haspopup="true" aria-expanded="false">
+          <span class="caret"></span>&nbsp;
+        </button>
+        <ul class="dropdown-menu" id="prom_graph_refresh_intervals" role="menu">
+        </ul>
+      </div>
+    </div>
+  </div>
+  <script>
+  new PromConsole.TimeControl();
+  </script>
+</div>
+{{ end }}
+
+{{/* Bottom of all pages. */}}
+{{ define "tail" }}
+</body>
+</html>
+{{ end }}
diff --git a/bundles/prometheus/files/consoles/index.html b/bundles/prometheus/files/consoles/index.html
new file mode 100644
index 0000000..29c2884
--- /dev/null
+++ b/bundles/prometheus/files/consoles/index.html
@@ -0,0 +1,82 @@
+{{/* vim: set ft=html: */}}
+
+{{ define "menu" }}
+foobar
+{{ end }}
+
+{{ template "head" . }}
+
+{{ template "prom_right_table_head" . }}
+<tr>
+	<th colspan="2">S6 - inazuma</th>
+</tr>
+<tr>
+	<td>default</td>
+	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='default'}")) }}</td>
+</tr>
+<tr>
+	<td>containers</td>
+	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='containers'}")) }}</td>
+</tr>
+<tr>
+	<td>www</td>
+	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='www'}")) }}</td>
+</tr>
+<tr>
+	<td>services</td>
+	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='services'}")) }}</td>
+</tr>
+<tr>
+	<th colspan="2">S6 - shimakaze</th>
+</tr>
+<tr>
+	<td>default</td>
+	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='default'}")) }}</td>
+</tr>
+<tr>
+	<td>containers</td>
+	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='containers'}")) }}</td>
+</tr>
+<tr>
+	<td>www</td>
+	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='www'}")) }}</td>
+</tr>
+<tr>
+	<td>services</td>
+	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='services'}")) }}</td>
+</tr>
+{{ template "prom_right_table_tail" . }}
+
+{{ template "prom_content_head" . }}
+<div class="prom_content_div">
+	<h1>Overview</h1>
+	<h3>Traefik Requests OK - inazuma</h3>
+	<div id="traefikGraph"></div>
+	<script>
+		new PromConsole.Graph({
+			node: document.querySelector("#traefikGraph"),
+			expr: "irate(traefik_service_request_duration_seconds_count{job='traefik',instance='inazuma',code='200'}[5m])",
+			name: '[[service]]',
+			yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+			yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+			yTitle: "Requests",
+			yUnits: "/s",
+		})
+	</script>
+	<h3>Jellyfin HTTP Requests - shimakaze</h3>
+	<div id="jellyfinGraph"></div>
+	<script>
+		new PromConsole.Graph({
+			node: document.querySelector("#jellyfinGraph"),
+			expr: "irate(http_request_duration_seconds_count{job='jellyfin',instance='shimakaze'}[5m])",
+			name: '[[action]]',
+			yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+			yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+			yTitle: "Requests",
+			yUnits: "/s",
+		})
+	</script>
+</div>
+{{ template "prom_content_tail" . }}
+
+{{ template "tail" . }}
diff --git a/bundles/prometheus/files/consoles/node-cpu.html b/bundles/prometheus/files/consoles/node-cpu.html
new file mode 100644
index 0000000..64a2cad
--- /dev/null
+++ b/bundles/prometheus/files/consoles/node-cpu.html
@@ -0,0 +1,60 @@
+{{ template "head" . }}
+
+{{ template "prom_right_table_head" }}
+  <tr>
+    <th colspan="2">CPU(s): {{ template "prom_query_drilldown" (args (printf "scalar(count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'})))" .Params.instance)) }}</th>
+  </tr>
+{{ range printf "sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='%s'}[5m])) * 100 / scalar(count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'})))" .Params.instance .Params.instance | query | sortByLabel "mode" }}
+  <tr>
+    <td>{{ .Labels.mode | title }} CPU</td>
+    <td>{{ .Value | printf "%.1f" }}%</td>
+  </tr>
+{{ end }}
+  <tr><th colspan="2">Misc</th></tr>
+  <tr>
+    <td>Processes Running</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "node_procs_running{job='node-exporter',instance='%s'}" .Params.instance) "" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Processes Blocked</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "node_procs_blocked{job='node-exporter',instance='%s'}" .Params.instance) "" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Forks</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_forks_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Context Switches</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_context_switches_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Interrupts</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_intr_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>1m Loadavg</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "node_load1{job='node-exporter',instance='%s'}" .Params.instance)) }}</td>
+  </tr>
+<tr>
+</tr>
+{{ template "prom_right_table_tail" }}
+
+{{ template "prom_content_head" . }}
+  <h1>Node CPU - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1>
+
+  <h3>CPU Usage</h3>
+  <div id="cpuGraph"></div>
+  <script>
+  new PromConsole.Graph({
+    node: document.querySelector("#cpuGraph"),
+    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
+    renderer: 'area',
+    max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
+    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yTitle: 'Cores'
+  })
+  </script>
+{{ template "prom_content_tail" . }}
+
+{{ template "tail" }}
diff --git a/bundles/prometheus/files/consoles/node-disk.html b/bundles/prometheus/files/consoles/node-disk.html
new file mode 100644
index 0000000..52afac7
--- /dev/null
+++ b/bundles/prometheus/files/consoles/node-disk.html
@@ -0,0 +1,78 @@
+{{ template "head" . }}
+
+{{ template "prom_right_table_head" }}
+<tr>
+  <th colspan="2">Disks</th>
+</tr>
+{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
+  <th colspan="2">{{ .Labels.device }}</th>
+  <tr>
+    <td>Utilization</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
+  </tr>
+  <tr>
+    <td>Throughput</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Avg Read Time</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) / irate(node_disk_reads_completed_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Avg Write Time</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) / irate(node_disk_writes_completed_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }}</td>
+  </tr>
+{{ end }}
+<tr>
+  <th colspan="2">Filesystem Fullness</th>
+</tr>
+{{ define "roughlyNearZero" }}
+{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }}
+{{ end }}
+{{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }}
+  <tr>
+    <td>{{ .Labels.mountpoint }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td>
+  </tr>
+{{ end }}
+<tr>
+</tr>
+{{ template "prom_right_table_tail" }}
+
+{{ template "prom_content_head" . }}
+  <h1>Node Disk - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1>
+
+  <h3>Disk I/O Utilization</h3>
+  <div id="diskioGraph"></div>
+  <script>
+  new PromConsole.Graph({
+    node: document.querySelector("#diskioGraph"),
+    expr: [
+      "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',device!~'^(md\\\\d+$|dm-)'}[5m]) * 100",
+    ],
+    min: 0,
+    name: '[[ device ]]',
+    yUnits: "%",
+    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yTitle: 'Disk I/O Utilization'
+  })
+  </script>
+  <h3>Filesystem Usage</h3>
+  <div id="fsGraph"></div>
+  <script>
+  new PromConsole.Graph({
+    node: document.querySelector("#fsGraph"),
+    expr: "100 - node_filesystem_avail_bytes{job='node-exporter',instance='{{ .Params.instance }}'} / node_filesystem_size_bytes{job='node-exporter'} * 100",
+    min: 0,
+    max: 100,
+    name: '[[ mountpoint ]]',
+    yUnits: "%",
+    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yTitle: 'Filesystem Fullness'
+  })
+  </script>
+{{ template "prom_content_tail" . }}
+
+{{ template "tail" }}
diff --git a/bundles/prometheus/files/consoles/node-overview.html b/bundles/prometheus/files/consoles/node-overview.html
new file mode 100644
index 0000000..edeca2e
--- /dev/null
+++ b/bundles/prometheus/files/consoles/node-overview.html
@@ -0,0 +1,121 @@
+{{ template "head" . }}
+
+{{ template "prom_right_table_head" }}
+  <tr><th colspan="2">Overview</th></tr>
+  <tr>
+    <td>User CPU</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
+  </tr>
+  <tr>
+    <td>System CPU</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td>
+  </tr>
+  <tr>
+    <td>Memory Total</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
+  </tr>
+  <tr>
+    <td>Memory Free</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
+  </tr>
+<tr>
+  <th colspan="2">Network</th>
+</tr>
+{{ range printf "node_network_receive_bytes_total{job='node-exporter',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }}
+  <tr>
+    <td>{{ .Labels.device }} Received</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>{{ .Labels.device }} Transmitted</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+  </tr>
+{{ end }}
+<tr>
+  <th colspan="2">Disks</th>
+</tr>
+{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }}
+  <tr>
+    <td>{{ .Labels.device }} Utilization</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td>
+  </tr>
+{{ end }}
+{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }}
+  <tr>
+    <td>{{ .Labels.device }} Throughput</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td>
+  </tr>
+{{ end }}
+<tr>
+  <th colspan="2">Filesystem Fullness</th>
+</tr>
+{{ define "roughlyNearZero" }}
+{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }}
+{{ end }}
+{{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }}
+  <tr>
+    <td>{{ .Labels.mountpoint }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td>
+  </tr>
+{{ end }}
+
+{{ template "prom_right_table_tail" }}
+
+{{ template "prom_content_head" . }}
+  <h1>Node Overview - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1>
+
+  <h3>CPU Usage</h3>
+  <div id="cpuGraph"></div>
+  <script>
+  new PromConsole.Graph({
+    node: document.querySelector("#cpuGraph"),
+    expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))",
+    renderer: 'area',
+    max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}},
+    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yTitle: 'Cores'
+  })
+  </script>
+
+  <h3>Disk I/O Utilization</h3>
+  <div id="diskioGraph"></div>
+  <script>
+  new PromConsole.Graph({
+    node: document.querySelector("#diskioGraph"),
+    expr: [
+      "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',device!~'^(md\\\\d+$|dm-)'}[5m]) * 100",
+    ],
+    min: 0,
+    name: '[[ device ]]',
+    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yUnits: "%",
+    yTitle: 'Disk I/O Utilization'
+  })
+  </script>
+
+  <h3>Memory</h3>
+  <div id="memoryGraph"></div>
+  <script>
+  new PromConsole.Graph({
+    node: document.querySelector("#memoryGraph"),
+    renderer: 'area',
+    expr: [
+      "node_memory_Cached_bytes{job='node-exporter',instance='{{ .Params.instance }}'}",
+      "node_memory_Buffers_bytes{job='node-exporter',instance='{{ .Params.instance }}'}",
+      "node_memory_MemTotal_bytes{job='node-exporter',instance='{{ .Params.instance }}'} - node_memory_MemFree_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Buffers_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Cached_bytes{job='node-exporter',instance='{{.Params.instance}}'}",
+      "node_memory_MemFree{job='node-exporter',instance='{{ .Params.instance }}'}",
+    ],
+    name: ["Cached", "Buffers", "Used", "Free"],
+    min: 0,
+    yUnits: "B",
+    yAxisFormatter: PromConsole.NumberFormatter.humanize1024,
+    yHoverFormatter: PromConsole.NumberFormatter.humanize1024,
+    yTitle: 'Memory'
+  })
+  </script>
+
+{{ template "prom_content_tail" . }}
+
+{{ template "tail" }}
diff --git a/bundles/prometheus/files/consoles/prometheus-overview.html b/bundles/prometheus/files/consoles/prometheus-overview.html
new file mode 100644
index 0000000..08e027d
--- /dev/null
+++ b/bundles/prometheus/files/consoles/prometheus-overview.html
@@ -0,0 +1,96 @@
+{{ template "head" . }}
+
+{{ template "prom_right_table_head" }}
+  <tr>
+    <th colspan="2">Overview</th>
+  </tr>
+  <tr>
+    <td>CPU</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }}</td>
+  </tr>
+  <tr>
+    <td>Memory</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td>
+  </tr>
+  <tr>
+    <td>Version</td>
+    <td>{{ with query (printf "prometheus_build_info{job='prometheus',instance='%s'}" .Params.instance) }}{{. | first | label "version"}}{{end}}</td>
+  </tr>
+
+  <tr>
+    <th colspan="2">Storage</th>
+  </tr>
+  <tr>
+    <td>Ingested Samples</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }}</td>
+  </tr>
+  <tr>
+    <td>Head Series</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_head_series{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
+  </tr>
+  <tr>
+    <td>Blocks Loaded</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_blocks_loaded{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
+  </tr>
+  <tr>
+    <th colspan="2">Rules</th>
+  </tr>
+  <tr>
+    <td>Evaluation Duration</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_evaluator_duration_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
+  </tr>
+  <tr>
+    <td>Notification Latency</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_notifications_latency_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td>
+  </tr>
+  <tr>
+    <td>Notification Queue</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "prometheus_notifications_queue_length{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td>
+  </tr>
+  <tr>
+    <th colspan="2">HTTP Server</th>
+  </tr>
+{{ range printf "prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s'}" .Params.instance | query | sortByLabel "handler" }}
+  <tr>
+    <td>{{ .Labels.handler }}</td>
+    <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s',handler='%s'}[5m])" .Labels.instance .Labels.handler) "/s" "humanizeNoSmallPrefix") }}</td>
+  </tr>
+{{ end }}
+
+{{ template "prom_right_table_tail" }}
+
+{{ template "prom_content_head" . }}
+ <div class="prom_content_div">
+  <h1>Prometheus Overview - {{ .Params.instance }}</h1>
+
+  <h3>Ingested Samples</h3>
+  <div id="samplesGraph"></div>
+  <script>
+  new PromConsole.Graph({
+    node: document.querySelector("#samplesGraph"),
+    expr: "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='{{ .Params.instance }}'}[5m])",
+    name: 'Ingested Samples',
+    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yTitle: "Samples",
+    yUnits: "/s",
+  })
+  </script>
+
+  <h3>HTTP Server</h3>
+  <div id="serverGraph"></div>
+  <script>
+  new PromConsole.Graph({
+    node: document.querySelector("#serverGraph"),
+    expr: "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='{{ .Params.instance }}'}[5m])",
+    name: '[[handler]]',
+    yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    yTitle: "Requests",
+    yUnits: "/s",
+  })
+  </script>
+ </div>
+{{ template "prom_content_tail" . }}
+
+{{ template "tail" }}
diff --git a/bundles/prometheus/files/kube.yaml b/bundles/prometheus/files/kube.yaml
new file mode 100644
index 0000000..952f485
--- /dev/null
+++ b/bundles/prometheus/files/kube.yaml
@@ -0,0 +1,48 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: prometheus
+  labels:
+    traefik.enable: true
+    traefik.http.routers.prometheus.tls: true
+    traefik.http.routers.prometheus.tls.certresolver: le
+    traefik.http.routers.prometheus.entrypoints: http,https
+    traefik.http.routers.prometheus.rule: Host(`prom.gzr.im`)
+    traefik.http.routers.prometheus.service: prometheus
+    traefik.http.services.prometheus.loadbalancer.server.port: 9090
+spec:
+  restartPolicy: Never
+  dnsPolicy: Default
+  dnsConfig:
+    searches:
+      - tail2d6ce.ts.net
+  containers:
+  - name: prom
+    image: quay.io/prometheus/prometheus:v2.52.0
+    volumeMounts:
+    - mountPath: /etc/prometheus:z
+      name: config
+      readOnly: true
+    - mountPath: /usr/share/prometheus/consoles:z
+      name: config
+      subPath: consoles
+      readOnly: true
+    - mountPath: /usr/share/prometheus/console_libraries:z
+      name: config
+      subPath: console_libraries
+      readOnly: true
+    ports:
+    - containerPort: 9090
+      protocol: TCP
+    resources:
+      limits:
+        cpu: 100m
+        memory: 200Mi
+      requests:
+        cpu: 25m
+        memory: 100Mi
+  volumes:
+  - name: config
+    hostPath:
+      path: /etc/prometheus
+      type: Directory
diff --git a/bundles/prometheus/files/prometheus.yml b/bundles/prometheus/files/prometheus.yml
new file mode 100644
index 0000000..0614c8c
--- /dev/null
+++ b/bundles/prometheus/files/prometheus.yml
@@ -0,0 +1,28 @@
+global:
+  scrape_interval: 60s
+  scrape_timeout: 30s
+  # evaluation_interval: 30s
+  # body_size_limit: 15MB
+  # sample_limit: 1500
+  # target_limit: 30
+  # label_limit: 30
+  # label_name_length_limit: 200
+  # label_value_length_limit: 200
+
+rule_files:
+  - rule_*.yml
+
+scrape_configs:
+
+  - job_name: prometheus
+    static_configs:
+    - targets: [localhost:9090]
+      labels:
+        instance: inazuma
+
+  - job_name: service-discovery
+    file_sd_configs:
+      - files:
+          - /etc/prometheus/services/*.json
+          - /etc/prometheus/services/*.yaml
+          - /etc/prometheus/services/*.yml
diff --git a/bundles/prometheus/files/service.json b/bundles/prometheus/files/service.json
new file mode 100644
index 0000000..1c124ad
--- /dev/null
+++ b/bundles/prometheus/files/service.json
@@ -0,0 +1,12 @@
+[
+	{
+		"targets": [
+			"${hostname}:9010"
+		],
+		"labels": {
+			"job": "${job_name}",
+			"instance": "${hostname}",
+			"__metrics_path__": "/metrics/${job_name}"
+		}
+	}
+]
diff --git a/bundles/prometheus/items.py b/bundles/prometheus/items.py
new file mode 100644
index 0000000..234aca5
--- /dev/null
+++ b/bundles/prometheus/items.py
@@ -0,0 +1,47 @@
+from sys import exit
+from os.path import join
+
+files = {
+    "/etc/deployments/prometheus/kube.yaml": {
+        "content_type": "text",
+        "source": "kube.yaml",
+        "triggers": {"svc_s6rc:prometheus:restart"},
+    },
+    "/etc/prometheus/prometheus.yml": {
+        "content_type": "text",
+        "source": "prometheus.yml",
+        "triggers": {"svc_s6rc:prometheus:restart"},
+    },
+}
+
+repo.libs.gen.add_files_recursive(
+    files,
+    join(repo.path, "bundles", "prometheus", "files", "consoles"),
+    target_root="/etc/prometheus",
+    files_root=join(repo.path, "bundles", "prometheus", "files"),
+)
+repo.libs.gen.add_files_recursive(
+    files,
+    join(repo.path, "bundles", "prometheus", "files", "console_libraries"),
+    target_root="/etc/prometheus",
+    files_root=join(repo.path, "bundles", "prometheus", "files"),
+)
+
+directories = {
+        "/etc/prometheus/services": {},
+        "/etc/prometheus/consoles": {},
+        "/etc/prometheus/console_libraries": {},
+}
+
+# NOTE: services that want to have their metrics scraped need to expose them
+#       under traefik:4000/metrics/<svc> and set metadata.metrics.<svc>
+for node in repo.nodes:
+    for job_name, _ in node.metadata.get("metrics", {}).items():
+        files[f"/etc/prometheus/services/{node.hostname}.{job_name}.json"] = {
+            "content_type": "mako",
+            "source": "service.json",
+            "context": {
+                "job_name": job_name,
+                "hostname": node.hostname,
+            },
+        }
diff --git a/bundles/prometheus/metadata.py b/bundles/prometheus/metadata.py
new file mode 100644
index 0000000..a431a4c
--- /dev/null
+++ b/bundles/prometheus/metadata.py
@@ -0,0 +1,5 @@
+defaults = {
+    "containers": {
+        "prometheus": {},
+    },
+}