groups:
  - name: Prometheus
    rules:
      - alert: PrometheusTargetMissing
        expr: up == 0
        for: 10m
        labels:
          severity: critical
        annotations:
          summary: |-
            Prometheus target missing (instance {{ $labels.instance }})
          description: |-
            A Prometheus target has disappeared. An exporter might be crashed.
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

  - name: Host
    rules:
      - alert: HostOutOfMemory
        expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < .10)
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: |-
            Host out of memory (instance {{ $labels.instance }})
          description: |-
            Node memory is filling up (< 10% left)
              VALUE = {{ $value }}
              LABELS = {{ $labels }}

      - alert: HostMemoryUnderMemoryPressure
        expr: (rate(node_vmstat_pgmajfault[5m]) > 1000)
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: |-
            Host memory under memory pressure (instance {{ $labels.instance }})
          description: |-
            The node is under heavy memory pressure. High rate of loading memory pages from disk.
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

      - alert: HostUnusualDiskReadRate
        expr: (rate(node_disk_io_time_seconds_total[5m]) > .80)
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: |-
            Host unusual disk read rate (instance {{ $labels.instance }})
          description: |-
            Disk is too busy (IO wait > 80%)
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

      - alert: HostOutOfDiskSpace
        expr: (node_filesystem_avail_bytes{fstype!~"^(fuse.*|-tmpfs|cifs|nfs)",mountpoint!="/media/usb"} / node_filesystem_size_bytes < .10 and on (instance, device, mountpoint) node_filesystem_readonly == 0)
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: |-
            Host out of disk space (instance {{ $labels.instance }})
          description: |-
            Disk is almost full (< 10% left)
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

      - alert: HostHighCpuLoad
        expr: 1 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > .80
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: |-
            Host high CPU load (instance {{ $labels.instance }})
          description: |-
            CPU load is > 80%
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

      - alert: HostUnusualNetworkThroughputOut
        expr: ((rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: |-
            Host unusual outbound network traffic (instance {{ $labels.instance }})
          description: |-
            Host transmit bandwidth is high (>80%)
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

      - alert: HostUnusualNetworkThroughputIn
        expr: ((rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80)
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: |-
            Host unusual inbound network traffic (instance {{ $labels.instance }})
          description: |-
            Host receive bandwidth is high (>80%)
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

      - alert: HostServiceDown
        expr: count(s6_rc_service_up{bundle="default", service=~".*-srv"}) by (instance, service) == 0
        for: 0m
        labels:
          severity: critical
        annotations:
          summary: |-
            s6-rc service down (instance {{ $labels.instance }})
          description: |-
            s6-rc service {{ $labels.service }} on {{ $labels.instance }} is down!
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

  - name: Traefik
    rules:
      - alert: TraefikHighHttp4xxErrorRateService
        expr: sum(rate(traefik_service_requests_total{service!~".*go-away@.*",code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: |-
            Traefik high HTTP 4xx error rate service (instance {{ $labels.instance }})
          description: |-
            Traefik service 4xx error rate is above 5%
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

      - alert: TraefikHighHttp5xxErrorRateService
        expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: |-
            Traefik high HTTP 5xx error rate service (instance {{ $labels.instance }})
          description: |-
            Traefik service 5xx error rate is above 5%
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

      - alert: TraefikLatencyHigh
        expr: histogram_quantile(0.99, sum(rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~"https?"}[3m])) by (le, instance)) > 10
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: |-
            Traefik latency high (instance {{ $labels.instance }})
          description: |-
            Traefik p99 latency is higher than 10 seconds
              VALUE  = {{ $value }}
              LABELS = {{ $labels }}

  - name: Podman
    rules:
      - alert: ContainerHighMemoryUsage
        expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) > 80
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: |-
            Container High Memory usage (instance {{ $labels.instance }})
          description: |-
            Container Memory usage is above 80%
              VALUE = {{ $value }}
              LABELS = {{ $labels }}

      - alert: ContainerLowMemoryUsage
        expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) < 20
        for: 7d
        labels:
          severity: info
        annotations:
          summary: |-
            Container Low Memory usage (instance {{ $labels.instance }})
          description: |-
            Container Memory usage is under 20% for 1 week. Consider reducing the allocated memory.
              VALUE = {{ $value }}
              LABELS = {{ $labels }}

  - name: BadBots
    rules:
      - alert: BadBotsFail2BanRate
        expr: rate(f2b_jail_banned_current[15m]) > 10
        for: 0m
        labels:
          severity: warning
        annotations:
          summary: |-
            Increased rate of bans
          description: |-
            Fail2Ban has banned more than 10 IPs in the last 15 minutes.
              VALUE = {{ $value }}
              LABELS = {{ $labels }}

      # - alert: BadBotsGoAway...
      #   expr: rate(f2b_jail_banned_current[15m]) > 10
      #   for: 0m
      #   labels:
      #     severity: warning
      #   annotations:
      #     summary: |-
      #       Increased rate of bans
      #     description: |-
      #       Fail2Ban has banned more than 10 IPs in the last 15 minutes.
      #         VALUE = {{ $value }}
      #         LABELS = {{ $labels }}