prometheus: modernize console + alerts

Signed-off-by: Robert Günzler <r@gnzler.io>

1 files changed, 218 insertions, 0 deletions

diff --git a/bundles/prometheus/files/rules.yml b/bundles/prometheus/files/rules.yml
new file mode 100644
index 0000000..45b130a
--- /dev/null
+++ b/bundles/prometheus/files/rules.yml
@@ -0,0 +1,218 @@
+groups:
+  - name: Prometheus
+    rules:
+      - alert: PrometheusTargetMissing
+        expr: up == 0
+        for: 10m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            Prometheus target missing (instance {{ $labels.instance }})
+          description: |-
+            A Prometheus target has disappeared. An exporter might be crashed.
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+  - name: Host
+    rules:
+      - alert: HostOutOfMemory
+        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < .10)
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host out of memory (instance {{ $labels.instance }})
+          description: |-
+            Node memory is filling up (< 10% left)
+              VALUE = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostMemoryUnderMemoryPressure
+        expr: (rate(node_vmstat_pgmajfault[5m]) > 1000)
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host memory under memory pressure (instance {{ $labels.instance }})
+          description: |-
+            The node is under heavy memory pressure. High rate of loading memory pages from disk.
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostUnusualDiskReadRate
+        expr: (rate(node_disk_io_time_seconds_total[5m]) > .80)
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host unusual disk read rate (instance {{ $labels.instance }})
+          description: |-
+            Disk is too busy (IO wait > 80%)
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostOutOfDiskSpace
+        expr: (node_filesystem_avail_bytes{fstype!~"^(fuse.*|-tmpfs|cifs|nfs)",mountpoint!="/media/usb"} / node_filesystem_size_bytes < .10 and on (instance, device, mountpoint) node_filesystem_readonly == 0)
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            Host out of disk space (instance {{ $labels.instance }})
+          description: |-
+            Disk is almost full (< 10% left)
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostHighCpuLoad
+        expr: 1 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > .80
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host high CPU load (instance {{ $labels.instance }})
+          description: |-
+            CPU load is > 80%
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostUnusualNetworkThroughputOut
+        expr: ((rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host unusual outbound network traffic (instance {{ $labels.instance }})
+          description: |-
+            Host transmit bandwidth is high (>80%)
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostUnusualNetworkThroughputIn
+        expr: ((rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Host unusual inbound network traffic (instance {{ $labels.instance }})
+          description: |-
+            Host receive bandwidth is high (>80%)
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: HostServiceDown
+        expr: count(s6_rc_service_up{bundle="default", service=~".*-srv"}) by (instance, service) == 0
+        for: 0m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            s6-rc service down (instance {{ $labels.instance }})
+          description: |-
+            s6-rc service {{ $labels.service }} on {{ $labels.instance }} is down!
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+  - name: Traefik
+    rules:
+      - alert: TraefikHighHttp4xxErrorRateService
+        expr: sum(rate(traefik_service_requests_total{service!~".*go-away@.*",code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            Traefik high HTTP 4xx error rate service (instance {{ $labels.instance }})
+          description: |-
+            Traefik service 4xx error rate is above 5%
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: TraefikHighHttp5xxErrorRateService
+        expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
+        for: 5m
+        labels:
+          severity: critical
+        annotations:
+          summary: |-
+            Traefik high HTTP 5xx error rate service (instance {{ $labels.instance }})
+          description: |-
+            Traefik service 5xx error rate is above 5%
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: TraefikLatencyHigh
+        expr: histogram_quantile(0.99, sum(rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~"https?"}[3m])) by (le, instance)) > 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Traefik latency high (instance {{ $labels.instance }})
+          description: |-
+            Traefik p99 latency is higher than 10 seconds
+              VALUE  = {{ $value }}
+              LABELS = {{ $labels }}
+
+  - name: Podman
+    rules:
+      - alert: ContainerHighMemoryUsage
+        expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) > 80
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Container High Memory usage (instance {{ $labels.instance }})
+          description: |-
+            Container Memory usage is above 80%
+              VALUE = {{ $value }}
+              LABELS = {{ $labels }}
+
+      - alert: ContainerLowMemoryUsage
+        expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) < 20
+        for: 7d
+        labels:
+          severity: info
+        annotations:
+          summary: |-
+            Container Low Memory usage (instance {{ $labels.instance }})
+          description: |-
+            Container Memory usage is under 20% for 1 week. Consider reducing the allocated memory.
+              VALUE = {{ $value }}
+              LABELS = {{ $labels }}
+
+  - name: BadBots
+    rules:
+      - alert: BadBotsFail2BanRate
+        expr: rate(f2b_jail_banned_current[15m]) > 10
+        for: 0m
+        labels:
+          severity: warning
+        annotations:
+          summary: |-
+            Increased rate of bans
+          description: |-
+            Fail2Ban has banned more than 10 IPs in the last 15 minutes.
+              VALUE = {{ $value }}
+              LABELS = {{ $labels }}
+
+      # - alert: BadBotsGoAway...
+      #   expr: rate(f2b_jail_banned_current[15m]) > 10
+      #   for: 0m
+      #   labels:
+      #     severity: warning
+      #   annotations:
+      #     summary: |-
+      #       Increased rate of bans
+      #     description: |-
+      #       Fail2Ban has banned more than 10 IPs in the last 15 minutes.
+      #         VALUE = {{ $value }}
+      #         LABELS = {{ $labels }}