From c9197b9bfd5cd3ab49ee7dfdc81d207715ccd5d8 Mon Sep 17 00:00:00 2001
From: Robert Günzler <r@gnzler.io>
Date: Mon, 23 Feb 2026 14:19:55 +0100
Subject: prometheus: modernize console + alerts
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Robert Günzler <r@gnzler.io>
---
 bundles/prometheus/files/consoles/index.html | 163 +++++++++++++++------------
 1 file changed, 92 insertions(+), 71 deletions(-)

(limited to 'bundles/prometheus/files/consoles/index.html')

diff --git a/bundles/prometheus/files/consoles/index.html b/bundles/prometheus/files/consoles/index.html
index 29c2884..f0b102d 100644
--- a/bundles/prometheus/files/consoles/index.html
+++ b/bundles/prometheus/files/consoles/index.html
@@ -1,81 +1,102 @@
-{{/* vim: set ft=html: */}}
+{{/* vim: set ft=gotmpl: */}}
 
-{{ define "menu" }}
-foobar
+{{ define "s6_rc_services_count" }}
+{{ $instance := .arg0 }}{{ $bundle := .arg1 }}
+{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{instance='%s',bundle='%s',service=~'.*-srv'}" $instance $bundle)) }}
 {{ end }}
 
 {{ template "head" . }}
 
-{{ template "prom_right_table_head" . }}
-<tr>
-	<th colspan="2">S6 - inazuma</th>
-</tr>
-<tr>
-	<td>default</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='default'}")) }}</td>
-</tr>
-<tr>
-	<td>containers</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='containers'}")) }}</td>
-</tr>
-<tr>
-	<td>www</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='www'}")) }}</td>
-</tr>
-<tr>
-	<td>services</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='services'}")) }}</td>
-</tr>
-<tr>
-	<th colspan="2">S6 - shimakaze</th>
-</tr>
-<tr>
-	<td>default</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='default'}")) }}</td>
-</tr>
-<tr>
-	<td>containers</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='containers'}")) }}</td>
-</tr>
-<tr>
-	<td>www</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='www'}")) }}</td>
-</tr>
-<tr>
-	<td>services</td>
-	<td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='services'}")) }}</td>
-</tr>
-{{ template "prom_right_table_tail" . }}
-
 {{ template "prom_content_head" . }}
 <div class="prom_content_div">
-	<h1>Overview</h1>
-	<h3>Traefik Requests OK - inazuma</h3>
-	<div id="traefikGraph"></div>
-	<script>
-		new PromConsole.Graph({
-			node: document.querySelector("#traefikGraph"),
-			expr: "irate(traefik_service_request_duration_seconds_count{job='traefik',instance='inazuma',code='200'}[5m])",
-			name: '[[service]]',
-			yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-			yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-			yTitle: "Requests",
-			yUnits: "/s",
-		})
-	</script>
-	<h3>Jellyfin HTTP Requests - shimakaze</h3>
-	<div id="jellyfinGraph"></div>
-	<script>
-		new PromConsole.Graph({
-			node: document.querySelector("#jellyfinGraph"),
-			expr: "irate(http_request_duration_seconds_count{job='jellyfin',instance='shimakaze'}[5m])",
-			name: '[[action]]',
-			yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-			yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
-			yTitle: "Requests",
-			yUnits: "/s",
-		})
-	</script>
+  <div>
+    <h2>Overview</h2>
+    <code>IZ: {{ template "s6_rc_services_count" (args "inazuma" "default") }}</code>
+    <code>SK: {{ template "s6_rc_services_count" (args "shimakaze" "default") }}</code>
+  </div>
+  <div>
+    <h4>Traefik Latency</h4>
+    <div id="traefikLatencyGraph"></div>
+    <script>
+      new PromConsole.Graph({
+        node: document.querySelector("#traefikLatencyGraph"),
+        expr: [
+          `histogram_quantile(0.99, sum(rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~"https?"}[10m])) by (le, instance))`,
+        ],
+        name: '[[instance]]',
+        yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+        yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+        yTitle: "Latency (p99)",
+        yUnits: "s",
+      })
+    </script>
+  </div>
+  <div>
+    <h4>Node CPU Utilisation</h4>
+    <div id="cpuUtilGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#cpuUtilGraph"),
+      expr: [
+        `(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[1m]))) * 100.0`,
+      ],
+      name: '[[instance]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yUnits: "%",
+      min: 0,
+      max: 100,
+    })
+    </script>
+  </div>
+  <div>
+    <h4>Node CPU Saturation</h4>
+    <div id="cpuSatGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#cpuSatGraph"),
+      expr: [
+        `sum by (instance) (node_load15) / count by (instance) (sum by (instance, cpu) (node_cpu_seconds_total))`,
+      ],
+      name: '[[instance]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    })
+    </script>
+  </div>
+  <div>
+    <h4>Node Memory Utilisation</h4>
+    <div id="memUtilGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#memUtilGraph"),
+      expr: [
+        `(1 - sum by (instance) (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / sum by (instance) (node_memory_MemTotal_bytes)) * 100.0`,
+      ],
+      name: '[[instance]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yUnits: "%",
+      min: 0,
+      max: 100,
+    })
+    </script>
+  </div>
+  <div>
+    <h4>Node Memory Saturation</h4>
+    <div id="memSatGraph"></div>
+    <script>
+      new PromConsole.Graph({
+      node: document.querySelector("#memSatGraph"),
+      expr: [
+        `1e3 * sum by (instance) (rate(node_vmstat_pgpgin[2m]) + rate(node_vmstat_pgpgout[2m]))`,
+      ],
+      name: '[[instance]]',
+      yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+      yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix,
+    })
+    </script>
+  </div>
 </div>
 {{ template "prom_content_tail" . }}
 
-- 
cgit 1.4.1