From c9197b9bfd5cd3ab49ee7dfdc81d207715ccd5d8 Mon Sep 17 00:00:00 2001 From: Robert Günzler Date: Mon, 23 Feb 2026 14:19:55 +0100 Subject: prometheus: modernize console + alerts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Robert Günzler --- bundles/prometheus/files/alertmanager.yml | 38 ++ .../files/console_assets/ajax-loader.gif | Bin 0 -> 847 bytes bundles/prometheus/files/console_assets/menu.lib | 51 ++ bundles/prometheus/files/console_assets/prom.lib | 150 +++++ .../files/console_assets/prom_console.css | 190 ++++++ .../files/console_assets/prom_console.js | 687 +++++++++++++++++++++ .../prometheus/files/console_libraries/menu.lib | 79 +-- .../prometheus/files/console_libraries/prom.lib | 190 +++--- bundles/prometheus/files/consoles/cpu.html | 60 ++ bundles/prometheus/files/consoles/disk.html | 78 +++ bundles/prometheus/files/consoles/index.html | 163 ++--- bundles/prometheus/files/consoles/node-cpu.html | 60 -- bundles/prometheus/files/consoles/node-disk.html | 78 --- .../prometheus/files/consoles/node-overview.html | 121 ---- bundles/prometheus/files/consoles/node.html | 133 ++++ bundles/prometheus/files/consoles/prom.html | 88 +++ .../files/consoles/prometheus-overview.html | 96 --- bundles/prometheus/files/kube.yaml | 71 ++- bundles/prometheus/files/prometheus.yml | 24 +- bundles/prometheus/files/rules.yml | 218 +++++++ bundles/prometheus/items.py | 40 +- 21 files changed, 2036 insertions(+), 579 deletions(-) create mode 100644 bundles/prometheus/files/alertmanager.yml create mode 100644 bundles/prometheus/files/console_assets/ajax-loader.gif create mode 100644 bundles/prometheus/files/console_assets/menu.lib create mode 100644 bundles/prometheus/files/console_assets/prom.lib create mode 100644 bundles/prometheus/files/console_assets/prom_console.css create mode 100644 bundles/prometheus/files/console_assets/prom_console.js create mode 100644 bundles/prometheus/files/consoles/cpu.html create mode 100644 bundles/prometheus/files/consoles/disk.html delete mode 100644 bundles/prometheus/files/consoles/node-cpu.html delete mode 100644 bundles/prometheus/files/consoles/node-disk.html delete mode 100644 bundles/prometheus/files/consoles/node-overview.html create mode 100644 bundles/prometheus/files/consoles/node.html create mode 100644 bundles/prometheus/files/consoles/prom.html delete mode 100644 bundles/prometheus/files/consoles/prometheus-overview.html create mode 100644 bundles/prometheus/files/rules.yml diff --git a/bundles/prometheus/files/alertmanager.yml b/bundles/prometheus/files/alertmanager.yml new file mode 100644 index 0000000..e725338 --- /dev/null +++ b/bundles/prometheus/files/alertmanager.yml @@ -0,0 +1,38 @@ +global: + +route: + group_by: [instance, alertname, severity] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + mute_time_intervals: + - offhours + routes: + - matchers: [severity=critical] + repeat_interval: 1h + - matchers: [severity=warning] + repeat_interval: 4h + - matchers: [severity=info] + repeat_interval: 24h + receiver: telegram + +time_intervals: + - name: offhours + time_intervals: + - location: Europe/Berlin + times: + - start_time: 23:00 + end_time: 09:00 + +receivers: + - name: telegram + telegram_configs: + - bot_token: ${repo.vault.cmd(f"rbw get tgnotify/token")} + chat_id: ${repo.vault.cmd(f"rbw get tgnotify/chatid")} + message: | + {{if eq .GroupLabels.severity "critical"}}🚨{{end -}} + {{if eq .GroupLabels.severity "warning"}}⚠️{{end -}} + {{if eq .GroupLabels.severity "info"}}📬{{end -}} + {{with .GroupLabels.instance}}{{.}}/{{end}}{{.GroupLabels.alertname}} + {{.CommonAnnotations.summary}} + {{.CommonAnnotations.description}} diff --git a/bundles/prometheus/files/console_assets/ajax-loader.gif b/bundles/prometheus/files/console_assets/ajax-loader.gif new file mode 100644 index 0000000..e192ca8 Binary files /dev/null and b/bundles/prometheus/files/console_assets/ajax-loader.gif differ diff --git a/bundles/prometheus/files/console_assets/menu.lib b/bundles/prometheus/files/console_assets/menu.lib new file mode 100644 index 0000000..e5cf31f --- /dev/null +++ b/bundles/prometheus/files/console_assets/menu.lib @@ -0,0 +1,51 @@ +{{/* vim: set ft=gotmpl: */}} + +{{/* Navbar, should be passed . */}} +{{ define "navbar" }} + +{{ end }} + +{{/* LHS menu, should be passed . */}} +{{ define "menu" }} + +{{ end }} + +{{/* Helper, pass (args . path name) */}} +{{ define "_menuItem" }} +{{ $pagelink := .arg0.Path }} +{{ if .arg0.Params.instance }}{{ $pagelink = (printf "%v?instance=%v" .arg0.Path .arg0.Params.instance) }}{{ end }} +
  • + + {{ if eq .arg2 "overview" }} + + {{ else }} + {{ .arg2 }} + {{ end }} + +
    • +{{ end }} diff --git a/bundles/prometheus/files/console_assets/prom.lib b/bundles/prometheus/files/console_assets/prom.lib new file mode 100644 index 0000000..fbc39ff --- /dev/null +++ b/bundles/prometheus/files/console_assets/prom.lib @@ -0,0 +1,150 @@ +{{/* vim: set ft=gotmpl: */}} +{{/* Load Prometheus console library JS/CSS. Should go in head */}} + +{{ define "prom_console_head" }} + + +Prometheus Console + + + + + + + + + + + + + +{{ end }} + +{{/* Top of all pages. */}} +{{ define "head" -}} + + + + {{ template "prom_console_head" }} + + + {{ template "navbar" . }} + {{ template "menu" . }} +{{ end }} + +{{ define "__prom_query_drilldown_noop" }}{{ . }}{{ end }} +{{ define "humanize" }}{{ humanize . }}{{ end }} +{{ define "humanizeNoSmallPrefix" }}{{ if and (lt . 1.0) (gt . -1.0) }}{{ printf "%.3g" . }}{{ else }}{{ humanize . }}{{ end }}{{ end }} +{{ define "humanize1024" }}{{ humanize1024 . }}{{ end }} +{{ define "humanizeDuration" }}{{ humanizeDuration . }}{{ end }} +{{ define "humanizePercentage" }}{{ humanizePercentage . }}{{ end }} +{{ define "humanizeTimestamp" }}{{ humanizeTimestamp . }}{{ end }} +{{ define "printf.1f" }}{{ printf "%.1f" . }}{{ end }} +{{ define "printf.3g" }}{{ printf "%.3g" . }}{{ end }} + +{{/* prom_query_drilldown (args expr suffix? renderTemplate?) +Displays the result of the expression, with a link to /graph for it. + +renderTemplate is the name of the template to use to render the value. +*/}} +{{ define "prom_query_drilldown" }} +{{ $expr := .arg0 }}{{ $suffix := (or .arg1 "") }}{{ $renderTemplate := (or .arg2 "__prom_query_drilldown_noop") }} + {{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }} +{{ end }} + +{{ define "prom_path" }}/consoles/{{ .Path }}?{{ range $param, $value := .Params }}{{ $param }}={{ $value }}&{{ end }}{{ end }}" + +{{ define "prom_content_head" }} +
    + {{ template "prom_graph_timecontrol" . }} +{{ end }} +{{ define "prom_content_tail" }} +
    +{{ end }} + +{{ define "prom_graph_timecontrol" }} +
    +
    +
    + +
    +
    + +
    +
    +
    + + + +
    +
    +
    + +
    +{{ end }} + +{{/* Bottom of all pages. */}} + +{{ define "tail" }} + + +{{ end }} diff --git a/bundles/prometheus/files/console_assets/prom_console.css b/bundles/prometheus/files/console_assets/prom_console.css new file mode 100644 index 0000000..e5b8be7 --- /dev/null +++ b/bundles/prometheus/files/console_assets/prom_console.css @@ -0,0 +1,190 @@ +.navbar { + margin-bottom: 20px; +} + +div.prom_lhs_menu { + float: left; + margin-right: 2ex; + background: #222; + min-height: 100%; + overflow: auto; + width: auto; + margin-left: -40px; + margin-top: 55px; + height: 100%; +} +.prom_lhs_menu_nav { + padding-right: 0; +} +.prom_lhs_menu ul { + list-style: none; + padding-left: .5ex; + margin-left: .5ex; +} +.prom_lhs_menu li { + padding-right: 1ex; + padding-left: 1ex; + font-size: 0.9rem; +} +.prom_lhs_menu a, +.prom_lhs_menu li { + color: #999999; + display: block; +} +.prom_lhs_menu a { + text-decoration: none; + padding-top: 0; + padding-bottom: 0; +} +.prom_lhs_menu_selected a { + pointer-events: none; + cursor: default; +} +li.prom_lhs_menu_selected { + background: #555555; + background-clip: padding-box; +} +.prom_lhs_menu li:hover { + background: #666666; +} + +.prom_console_rhs { + float: right; + margin-left: 1ex; + height: 100%; + padding-top: 55px; +} +.prom_console_rhs table { + margin-top: 1ex; + margin-bottom: 32px; /* Space for time control. */ +} +.prom_console_rhs th { + text-align: center; +} +.prom_console_rhs td:nth-child(2) { + text-align: right; +} + +.prom_console_content { + overflow: visible; + margin-bottom: 32px; /* Space for time control. */ + padding-top: 70px; /* Space for navbar */ +} + +a.prom_query_drilldown { + text-decoration: none; + color: black; +} +a.prom_query_drilldown:hover, a.prom_query_drilldown:active { + text-decoration: underline; +} + +#samplesGraph { + width: 100%; +} + +.prom_content_div { + width: 80%; +} + +.prom_graph_table { +} +.rickshaw_legend { + padding: 2px; + margin-top: 1px; +} +.rickshaw_legend li { + min-width: 0; +} +.rickshaw_legend ul li { + list-style-type: none; + display: inline-block; +} +.rickshaw_graph { + width: 100%; + padding: 0; +} +.prom_graph_hover_flipped.x_label { + right: 0; +} +.prom_graph_hover_flipped.item { + right: 10px; +} +.rickshaw_graph .detail .prom_graph_hover_flipped.item:before { + content: "\25b8"; + left: auto; + right: 1px; + font-size: 0.8em; +} +.prom_graph_ytitle { + -webkit-transform: rotate(-90deg); + -moz-transform: rotate(-90deg); + font-size: 11px; + font-family: Arial; + max-width: 13px; + white-space: nowrap; +} +.prom_graph_xtitle { + text-align: center; + font-size: 11px; + font-family: Arial; +} +.prom_graph_loading { + position: absolute; + top: 0px; + left: 0px; +} +.prom_graph_error { + font-family: Arial; + text-align: center; +} +a.prom_graph_link { + text-decoration: none; + color: black; +} + + +.prom_graph_timecontrol { + background: #222; + position: fixed; + padding: 2px; + bottom: 0; + left: 0; + z-index: 2; + width: 100%; +} + +.prom_graph_timecontrol_inner { + text-align: center; + height: 31px; +} + +.prom_graph_timecontrol_group { + display: inline-block; + margin: 1px 5px; +} +.prom_graph_timecontrol_group .btn { + font-size: 0.8em; + border-radius: 0; + height: 29px; + border-color: #ccc; +} + +.prom_graph_timecontrol_group .input { + height: 29px; + padding: 6px 12px; + font-size: 14px; + line-height: 1.42857143; + color: #555; + background-color: #fff; + background-image: none; + border: 1px solid #ccc; + text-align: center; +} + +.prom_graph_timecontrol_group .dropdown-menu { + min-width: 90px; +} +.prom_graph_timecontrol_group .dropdown-menu li { + text-align: center; +} diff --git a/bundles/prometheus/files/console_assets/prom_console.js b/bundles/prometheus/files/console_assets/prom_console.js new file mode 100644 index 0000000..cb3501c --- /dev/null +++ b/bundles/prometheus/files/console_assets/prom_console.js @@ -0,0 +1,687 @@ +/* + * Functions to make it easier to write prometheus consoles, such + * as graphs. + * + */ +PromConsole = {}; + +PromConsole.NumberFormatter = {}; +PromConsole.NumberFormatter.prefixesBig = ["k", "M", "G", "T", "P", "E", "Z", "Y"]; +PromConsole.NumberFormatter.prefixesBig1024 = ["ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"]; +PromConsole.NumberFormatter.prefixesSmall = ["m", "u", "n", "p", "f", "a", "z", "y"]; + +PromConsole._stripTrailingZero = function(x) { + if (x.indexOf("e") == -1) { + // It's not safe to strip if it's scientific notation. + return x.replace(/\.?0*$/, ''); + } + return x; +}; + +// Humanize a number. +PromConsole.NumberFormatter.humanize = function(x) { + if (x === null) { + return "null"; + } + var ret = PromConsole.NumberFormatter._humanize( + x, PromConsole.NumberFormatter.prefixesBig, + PromConsole.NumberFormatter.prefixesSmall, 1000); + x = ret[0]; + var prefix = ret[1]; + if (Math.abs(x) < 1) { + return x.toExponential(3) + prefix; + } + return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix; +}; + +// Humanize a number, don't use milli/micro/etc. prefixes. +PromConsole.NumberFormatter.humanizeNoSmallPrefix = function(x) { + if (x === null) { + return "null"; + } + if (Math.abs(x) < 1) { + return PromConsole._stripTrailingZero(x.toPrecision(3)); + } + var ret = PromConsole.NumberFormatter._humanize( + x, PromConsole.NumberFormatter.prefixesBig, + [], 1000); + x = ret[0]; + var prefix = ret[1]; + return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix; +}; + +// Humanize a number with 1024 as the base, rather than 1000. +PromConsole.NumberFormatter.humanize1024 = function(x) { + if (x === null) { + return "null"; + } + var ret = PromConsole.NumberFormatter._humanize( + x, PromConsole.NumberFormatter.prefixesBig1024, + [], 1024); + x = ret[0]; + var prefix = ret[1]; + if (Math.abs(x) < 1) { + return x.toExponential(3) + prefix; + } + return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix; +}; + +// Humanize a number, returning an exact representation. +PromConsole.NumberFormatter.humanizeExact = function(x) { + if (x === null) { + return "null"; + } + var ret = PromConsole.NumberFormatter._humanize( + x, PromConsole.NumberFormatter.prefixesBig, + PromConsole.NumberFormatter.prefixesSmall, 1000); + return ret[0] + ret[1]; +}; + +PromConsole.NumberFormatter._humanize = function(x, prefixesBig, prefixesSmall, factor) { + var prefix = ""; + if (x === 0) { + /* Do nothing. */ + } else if (Math.abs(x) >= 1) { + for (var i = 0; i < prefixesBig.length && Math.abs(x) >= factor; ++i) { + x /= factor; + prefix = prefixesBig[i]; + } + } else { + for (var i = 0; i < prefixesSmall.length && Math.abs(x) < 1; ++i) { + x *= factor; + prefix = prefixesSmall[i]; + } + } + return [x, prefix]; +}; + + +PromConsole.TimeControl = function() { + document.getElementById("prom_graph_duration_shrink").onclick = this.decreaseDuration.bind(this); + document.getElementById("prom_graph_duration_grow").onclick = this.increaseDuration.bind(this); + document.getElementById("prom_graph_time_back").onclick = this.decreaseEnd.bind(this); + document.getElementById("prom_graph_time_forward").onclick = this.increaseEnd.bind(this); + document.getElementById("prom_graph_refresh_button").onclick = this.refresh.bind(this); + this.durationElement = document.getElementById("prom_graph_duration"); + this.endElement = document.getElementById("prom_graph_time_end"); + this.durationElement.oninput = this.dispatch.bind(this); + this.endElement.oninput = this.dispatch.bind(this); + this.endElement.oninput = this.dispatch.bind(this); + this.refreshValueElement = document.getElementById("prom_graph_refresh_button_value"); + + var refreshList = document.getElementById("prom_graph_refresh_intervals"); + var refreshIntervals = ["Off", "1m", "5m", "15m", "1h"]; + for (var i = 0; i < refreshIntervals.length; ++i) { + var li = document.createElement("li"); + li.onclick = this.setRefresh.bind(this, refreshIntervals[i]); + li.textContent = refreshIntervals[i]; + refreshList.appendChild(li); + } + + this.durationElement.value = PromConsole.TimeControl.prototype.getHumanDuration( + PromConsole.TimeControl._initialValues.duration); + if (!PromConsole.TimeControl._initialValues.endTimeNow) { + this.endElement.value = PromConsole.TimeControl.prototype.getHumanDate( + new Date(PromConsole.TimeControl._initialValues.endTime * 1000)); + } +}; + +PromConsole.TimeControl.timeFactors = { + "y": 60 * 60 * 24 * 365, + "w": 60 * 60 * 24 * 7, + "d": 60 * 60 * 24, + "h": 60 * 60, + "m": 60, + "s": 1 +}; + +PromConsole.TimeControl.stepValues = [ + "10s", "1m", "5m", "15m", "30m", "1h", "2h", "6h", "12h", "1d", "2d", + "1w", "2w", "4w", "8w", "1y", "2y" +]; + +PromConsole.TimeControl.prototype._setHash = function() { + var duration = this.parseDuration(this.durationElement.value); + var endTime = this.getEndDate() / 1000; + window.location.hash = "#pctc" + encodeURIComponent(JSON.stringify( + { duration: duration, endTime: endTime })); +}; + +PromConsole.TimeControl._initialValues = function() { + var hash = window.location.hash; + var values; + if (hash.indexOf('#pctc') === 0) { + values = JSON.parse(decodeURIComponent(hash.substring(5))); + } else { + values = { duration: 3600, endTime: 0 }; + } + if (values.endTime == 0) { + values.endTime = new Date().getTime() / 1000; + values.endTimeNow = true; + } + return values; +}(); + +PromConsole.TimeControl.prototype.parseDuration = function(durationText) { + var durationRE = new RegExp("^([0-9]+)([ywdhms]?)$"); + var matches = durationText.match(durationRE); + if (!matches) { return 3600; } + var value = parseInt(matches[1]); + var unit = matches[2] || 's'; + return value * PromConsole.TimeControl.timeFactors[unit]; +}; + +PromConsole.TimeControl.prototype.getHumanDuration = function(duration) { + var units = []; + for (var key in PromConsole.TimeControl.timeFactors) { + units.push([PromConsole.TimeControl.timeFactors[key], key]); + } + units.sort(function(a, b) { return b[0] - a[0]; }); + for (var i = 0; i < units.length; ++i) { + if (duration % units[i][0] === 0) { + return (duration / units[i][0]) + units[i][1]; + } + } + return duration; +}; + +PromConsole.TimeControl.prototype.increaseDuration = function() { + var durationSeconds = this.parseDuration(this.durationElement.value); + for (var i = 0; i < PromConsole.TimeControl.stepValues.length; i++) { + if (durationSeconds < this.parseDuration(PromConsole.TimeControl.stepValues[i])) { + this.setDuration(PromConsole.TimeControl.stepValues[i]); + this.dispatch(); + return; + } + } +}; + +PromConsole.TimeControl.prototype.decreaseDuration = function() { + var durationSeconds = this.parseDuration(this.durationElement.value); + for (var i = PromConsole.TimeControl.stepValues.length - 1; i >= 0; i--) { + if (durationSeconds > this.parseDuration(PromConsole.TimeControl.stepValues[i])) { + this.setDuration(PromConsole.TimeControl.stepValues[i]); + this.dispatch(); + return; + } + } +}; + +PromConsole.TimeControl.prototype.setDuration = function(duration) { + this.durationElement.value = duration; + this._setHash(); +}; + +PromConsole.TimeControl.prototype.getEndDate = function() { + if (this.endElement.value === '') { + return null; + } + var dateParts = this.endElement.value.split(/[- :]/) + return Date.UTC(dateParts[0], dateParts[1] - 1, dateParts[2], dateParts[3], dateParts[4]); +}; + +PromConsole.TimeControl.prototype.getOrSetEndDate = function() { + var date = this.getEndDate(); + if (date) { + return date; + } + date = new Date(); + this.setEndDate(date); + return date.getTime(); +}; + +PromConsole.TimeControl.prototype.getHumanDate = function(date) { + var hours = date.getUTCHours() < 10 ? '0' + date.getUTCHours() : date.getUTCHours(); + var minutes = date.getUTCMinutes() < 10 ? '0' + date.getUTCMinutes() : date.getUTCMinutes(); + return date.getUTCFullYear() + "-" + (date.getUTCMonth() + 1) + "-" + date.getUTCDate() + " " + + hours + ":" + minutes; +}; + +PromConsole.TimeControl.prototype.setEndDate = function(date) { + this.setRefresh("Off"); + this.endElement.value = this.getHumanDate(date); + this._setHash(); +}; + +PromConsole.TimeControl.prototype.increaseEnd = function() { + // Increase duration 25% range & convert ms to s. + this.setEndDate(new Date(this.getOrSetEndDate() + this.parseDuration(this.durationElement.value) * 1000 / 4)); + this.dispatch(); +}; + +PromConsole.TimeControl.prototype.decreaseEnd = function() { + this.setEndDate(new Date(this.getOrSetEndDate() - this.parseDuration(this.durationElement.value) * 1000 / 4)); + this.dispatch(); +}; + +PromConsole.TimeControl.prototype.refresh = function() { + this.endElement.value = ''; + this._setHash(); + this.dispatch(); +}; + +PromConsole.TimeControl.prototype.dispatch = function() { + var durationSeconds = this.parseDuration(this.durationElement.value); + var end = this.getEndDate(); + if (end === null) { + end = new Date().getTime(); + } + for (var i = 0; i < PromConsole._graph_registry.length; i++) { + var graph = PromConsole._graph_registry[i]; + graph.params.duration = durationSeconds; + graph.params.endTime = end / 1000; + graph.dispatch(); + } +}; + +PromConsole.TimeControl.prototype._refreshInterval = null; + +PromConsole.TimeControl.prototype.setRefresh = function(duration) { + if (this._refreshInterval !== null) { + window.clearInterval(this._refreshInterval); + this._refreshInterval = null; + } + if (duration != "Off") { + if (this.endElement.value !== '') { + this.refresh(); + } + var durationSeconds = this.parseDuration(duration); + this._refreshInterval = window.setInterval(this.dispatch.bind(this), durationSeconds * 1000); + } + this.refreshValueElement.textContent = duration; +}; + +// List of all graphs, used by time controls. +PromConsole._graph_registry = []; + +PromConsole.graphDefaults = { + expr: null, // Expression to graph. Can be a list of strings. + node: null, // DOM node to place graph under. + // How long the graph is over, in seconds. + duration: PromConsole.TimeControl._initialValues.duration, + // The unixtime the graph ends at. + endTime: PromConsole.TimeControl._initialValues.endTime, + width: null, // Height of the graph div, excluding titles and legends. + // Defaults to auto-detection. + height: 200, // Height of the graph div, excluding titles and legends. + min: "auto", // Minimum Y-axis value, defaults to lowest data value. + max: undefined, // Maximum Y-axis value, defaults to highest data value. + renderer: 'line', // Type of graphs, options are 'line' and 'area'. + name: null, // What to call plots, defaults to trying to do + // something reasonable. + // If a string, it'll use that. [[ label ]] will be substituted. + // If a function it'll be called with a map of keys to values, + // and should return the name to use. + // Can be a list of strings/functions, each element + // will be applied to the plots from the corresponding + // element of the expr list. + xTitle: "Time", // The title of the x axis. + yUnits: "", // The units of the y axis. + yTitle: "", // The title of the y axis. + // Number formatter for y axis. + yAxisFormatter: PromConsole.NumberFormatter.humanize, + // Number of ticks (horizontal lines) for the y axis + yAxisTicks: null, + // Number formatter for y values hover detail. + yHoverFormatter: PromConsole.NumberFormatter.humanizeExact, + // Color scheme to be used by the plots. Can be either a list of hex color + // codes or one of the color scheme names supported by Rickshaw. + colorScheme: null, +}; + +PromConsole.Graph = function(params) { + for (var k in PromConsole.graphDefaults) { + if (!(k in params)) { + params[k] = PromConsole.graphDefaults[k]; + } + } + if (typeof params.expr == "string") { + params.expr = [params.expr]; + } + if (typeof params.name == "string" || typeof params.name == "function") { + var name = []; + for (var i = 0; i < params.expr.length; i++) { + name.push(params.name); + } + params.name = name; + } + + this.params = params; + this.rendered_data = null; + // Keep a reference so that further updates (e.g. annotations) can be made + // by the user in their templates. + this.rickshawGraph = null; + PromConsole._graph_registry.push(this); + + /* + * Table layout: + * | yTitle | Graph | + * | | xTitle | + * | /graph | Legend | + */ + var table = document.createElement("table"); + table.className = "prom_graph_table"; + params.node.appendChild(table); + var tr = document.createElement("tr"); + table.appendChild(tr); + var yTitleTd = document.createElement("td"); + tr.appendChild(yTitleTd); + var yTitleDiv = document.createElement("td"); + yTitleTd.appendChild(yTitleDiv); + yTitleDiv.className = "prom_graph_ytitle"; + yTitleDiv.textContent = params.yTitle + (params.yUnits ? " (" + params.yUnits.trim() + ")" : ""); + + this.graphTd = document.createElement("td"); + tr.appendChild(this.graphTd); + this.graphTd.className = "rickshaw_graph"; + this.graphTd.width = params.width; + this.graphTd.height = params.height; + + tr = document.createElement("tr"); + table.appendChild(tr); + tr.appendChild(document.createElement("td")); + var xTitleTd = document.createElement("td"); + tr.appendChild(xTitleTd); + xTitleTd.className = "prom_graph_xtitle"; + xTitleTd.textContent = params.xTitle; + + tr = document.createElement("tr"); + table.appendChild(tr); + var graphLinkTd = document.createElement("td"); + tr.appendChild(graphLinkTd); + var graphLinkA = document.createElement("a"); + graphLinkTd.appendChild(graphLinkA); + graphLinkA.className = "prom_graph_link"; + graphLinkA.textContent = "+"; + graphLinkA.href = PromConsole._graphsToSlashGraphURL(params.expr); + var legendTd = document.createElement("td"); + tr.appendChild(legendTd); + this.legendDiv = document.createElement("div"); + legendTd.width = params.width; + legendTd.appendChild(this.legendDiv); + + window.addEventListener('resize', function() { + if (this.rendered_data !== null) { + this._render(this.rendered_data); + } + }.bind(this)); + + this.dispatch(); +}; + +PromConsole.Graph.prototype._parseValue = function(value) { + var val = parseFloat(value); + if (isNaN(val)) { + // "+Inf", "-Inf", "+Inf" will be parsed into NaN by parseFloat(). The + // can't be graphed, so show them as gaps (null). + return null; + } + return val; +}; + +PromConsole.Graph.prototype._escapeHTML = function(string) { + var entityMap = { + "&": "&", + "<": "<", + ">": ">", + '"': '"', + "'": ''', + "/": '/' + }; + + return string.replace(/[&<>"'\/]/g, function(s) { + return entityMap[s]; + }); +}; + +PromConsole.Graph.prototype._render = function(data) { + var self = this; + var palette = new Rickshaw.Color.Palette({ scheme: this.params.colorScheme }); + var series = []; + + // This will be used on resize. + this.rendered_data = data; + + + var nameFuncs = []; + if (this.params.name === null) { + var chooser = PromConsole._chooseNameFunction(data); + for (var i = 0; i < this.params.expr.length; i++) { + nameFuncs.push(chooser); + } + } else { + for (var i = 0; i < this.params.name.length; i++) { + if (typeof this.params.name[i] == "string") { + nameFuncs.push(function(i, metric) { + return PromConsole._interpolateName(this.params.name[i], metric); + }.bind(this, i)); + } else { + nameFuncs.push(this.params.name[i]); + } + } + } + + // Get the data into the right format. + var seriesLen = 0; + + for (var e = 0; e < data.length; e++) { + for (var i = 0; i < data[e].data.result.length; i++) { + series[seriesLen] = { + data: data[e].data.result[i].values.map(function(s) { return { x: s[0], y: self._parseValue(s[1]) }; }), + color: palette.color(), + name: self._escapeHTML(nameFuncs[e](data[e].data.result[i].metric)), + }; + // Insert nulls for all missing steps. + var newSeries = []; + var pos = 0; + var start = self.params.endTime - self.params.duration; + var step = Math.floor(self.params.duration / this.graphTd.offsetWidth * 1000) / 1000; + for (var t = start; t <= self.params.endTime; t += step) { + // Allow for floating point inaccuracy. + if (series[seriesLen].data.length > pos && series[seriesLen].data[pos].x < t + step / 100) { + newSeries.push(series[seriesLen].data[pos]); + pos++; + } else { + newSeries.push({ x: t, y: null }); + } + } + series[seriesLen].data = newSeries; + seriesLen++; + } + } + this._clearGraph(); + if (!series.length) { + var errorText = document.createElement("div"); + errorText.className = 'prom_graph_error'; + errorText.textContent = 'No timeseries returned'; + this.graphTd.appendChild(errorText); + return; + } + // Render. + var graph = new Rickshaw.Graph({ + interpolation: "linear", + width: this.graphTd.offsetWidth, + height: this.params.height, + element: this.graphTd, + renderer: this.params.renderer, + max: this.params.max, + min: this.params.min, + series: series + }); + var hoverDetail = new Rickshaw.Graph.HoverDetail({ + graph: graph, + onRender: function() { + var xLabel = this.element.getElementsByClassName("x_label")[0]; + var item = this.element.getElementsByClassName("item")[0]; + if (xLabel.offsetWidth + xLabel.offsetLeft + this.element.offsetLeft > graph.element.offsetWidth || + item.offsetWidth + item.offsetLeft + this.element.offsetLeft > graph.element.offsetWidth) { + xLabel.classList.add("prom_graph_hover_flipped"); + item.classList.add("prom_graph_hover_flipped"); + } else { + xLabel.classList.remove("prom_graph_hover_flipped"); + item.classList.remove("prom_graph_hover_flipped"); + } + }, + yFormatter: function(y) { + if (y === null) { + return ""; + } + return this.params.yHoverFormatter(y) + this.params.yUnits; + }.bind(this) + }); + var yAxis = new Rickshaw.Graph.Axis.Y({ + graph: graph, + tickFormat: this.params.yAxisFormatter, + ticks: this.params.yAxisTicks + }); + var xAxis = new Rickshaw.Graph.Axis.Time({ + graph: graph, + }); + var legend = new Rickshaw.Graph.Legend({ + graph: graph, + element: this.legendDiv + }); + xAxis.render(); + yAxis.render(); + graph.render(); + + this.rickshawGraph = graph; +}; + +PromConsole.Graph.prototype._clearGraph = function() { + while (this.graphTd.lastChild) { + this.graphTd.removeChild(this.graphTd.lastChild); + } + while (this.legendDiv.lastChild) { + this.legendDiv.removeChild(this.legendDiv.lastChild); + } + this.rickshawGraph = null; +}; + +PromConsole.Graph.prototype._xhrs = []; + +PromConsole.Graph.prototype.buildQueryUrl = function(expr) { + var p = this.params; + return PATH_PREFIX + "/api/v1/query_range?query=" + + encodeURIComponent(expr) + + "&step=" + Math.floor(p.duration / this.graphTd.offsetWidth * 1000) / 1000 + + "&start=" + (p.endTime - p.duration) + "&end=" + p.endTime; +}; + +PromConsole.Graph.prototype.dispatch = function() { + for (var j = 0; j < this._xhrs.length; j++) { + this._xhrs[j].abort(); + } + var all_data = new Array(this.params.expr.length); + this._xhrs = new Array(this.params.expr.length); + var pending_requests = this.params.expr.length; + for (var i = 0; i < this.params.expr.length; ++i) { + var endTime = this.params.endTime; + var url = this.buildQueryUrl(this.params.expr[i]); + var xhr = new XMLHttpRequest(); + xhr.open('get', url, true); + xhr.responseType = 'json'; + xhr.onerror = function(xhr, i) { + this._clearGraph(); + var errorText = document.createElement("div"); + errorText.className = 'prom_graph_error'; + errorText.textContent = 'Error loading data'; + this.graphTd.appendChild(errorText); + console.log('Error loading data for ' + this.params.expr[i]); + pending_requests = 0; + // onabort gets any aborts. + for (var j = 0; j < pending_requests; j++) { + this._xhrs[j].abort(); + } + }.bind(this, xhr, i); + xhr.onload = function(xhr, i) { + if (pending_requests === 0) { + // Got an error before this success. + return; + } + var data = xhr.response; + if (typeof data !== "object") { + data = JSON.parse(xhr.responseText); + } + pending_requests -= 1; + all_data[i] = data; + if (pending_requests === 0) { + this._xhrs = []; + this._render(all_data); + } + }.bind(this, xhr, i); + xhr.send(); + this._xhrs[i] = xhr; + } + + var loadingImg = document.createElement("img"); + loadingImg.src = PATH_PREFIX + '/user/ajax-loader.gif'; + loadingImg.alt = 'Loading...'; + loadingImg.className = 'prom_graph_loading'; + this.graphTd.appendChild(loadingImg); +}; + +// Substitute the value of 'label' for [[ label ]]. +PromConsole._interpolateName = function(name, metric) { + var re = /(.*?)\[\[\s*(\w+)+\s*\]\](.*?)/g; + var result = ''; + while (match = re.exec(name)) { + result = result + match[1] + metric[match[2]] + match[3]; + } + if (!result) { + return name; + } + return result; +}; + +// Given the data returned by the API, return an appropriate function +// to return plot names. +PromConsole._chooseNameFunction = function(data) { + // By default, use the full metric name. + var nameFunc = function(metric) { + var name = metric.__name__ + "{"; + for (var label in metric) { + if (label.substring(0, 2) == "__") { + continue; + } + name += label + "='" + metric[label] + "',"; + } + return name + "}"; + }; + + // If only one label varies, use that value. + var labelValues = {}; + for (var e = 0; e < data.length; e++) { + for (var i = 0; i < data[e].data.result.length; i++) { + for (var label in data[e].data.result[i].metric) { + if (!(label in labelValues)) { + labelValues[label] = {}; + } + labelValues[label][data[e].data.result[i].metric[label]] = 1; + } + } + } + + var multiValueLabels = []; + for (var label in labelValues) { + if (Object.keys(labelValues[label]).length > 1) { + multiValueLabels.push(label); + } + } + if (multiValueLabels.length == 1) { + nameFunc = function(metric) { + return metric[multiValueLabels[0]]; + }; + } + return nameFunc; +}; + +// Given a list of expressions, produce the /graph url for them. +PromConsole._graphsToSlashGraphURL = function(exprs) { + var data = []; + for (var i = 0; i < exprs.length; ++i) { + data.push({ 'expr': exprs[i], 'tab': 0 }); + } + return PATH_PREFIX + '/graph#' + encodeURIComponent(JSON.stringify(data)); + +}; diff --git a/bundles/prometheus/files/console_libraries/menu.lib b/bundles/prometheus/files/console_libraries/menu.lib index dc87f9f..d9e4baf 100644 --- a/bundles/prometheus/files/console_libraries/menu.lib +++ b/bundles/prometheus/files/console_libraries/menu.lib @@ -1,64 +1,51 @@ -{{/* vim: set ft=html: */}} +{{/* vim: set ft=gotmpl: */}} {{/* Navbar, should be passed . */}} {{ define "navbar" }} - {{ end }} {{/* Helper, pass (args . path name) */}} {{ define "_menuItem" }} {{ $pagelink := .arg0.Path }} {{ if .arg0.Params.instance }}{{ $pagelink = (printf "%v?instance=%v" .arg0.Path .arg0.Params.instance) }}{{ end }} -
  • {{ .arg2 }}
    • +
  • + + {{ if eq .arg2 "overview" }} + + {{ else }} + {{ .arg2 }} + {{ end }} + +
    • {{ end }} diff --git a/bundles/prometheus/files/console_libraries/prom.lib b/bundles/prometheus/files/console_libraries/prom.lib index f934d43..c00bb6b 100644 --- a/bundles/prometheus/files/console_libraries/prom.lib +++ b/bundles/prometheus/files/console_libraries/prom.lib @@ -1,35 +1,76 @@ -{{/* vim: set ft=html: */}} +{{/* vim: set ft=gotmpl: */}} {{/* Load Prometheus console library JS/CSS. Should go in head */}} {{ define "prom_console_head" }} - - - - - - - - - - - + + +Prometheus — Console + + + + + + + + + + + + + + + - {{ end }} {{/* Top of all pages. */}} {{ define "head" -}} - + - -{{ template "prom_console_head" }} - - -{{ template "navbar" . }} - -{{ template "menu" . }} + + {{ template "prom_console_head" }} + + + {{ template "navbar" . }} + {{ template "menu" . }} {{ end }} {{ define "__prom_query_drilldown_noop" }}{{ . }}{{ end }} @@ -49,91 +90,64 @@ renderTemplate is the name of the template to use to render the value. */}} {{ define "prom_query_drilldown" }} {{ $expr := .arg0 }}{{ $suffix := (or .arg1 "") }}{{ $renderTemplate := (or .arg2 "__prom_query_drilldown_noop") }} -{{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }} + {{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }} {{ end }} {{ define "prom_path" }}/consoles/{{ .Path }}?{{ range $param, $value := .Params }}{{ $param }}={{ $value }}&{{ end }}{{ end }}" -{{ define "prom_right_table_head" }} -
    - -{{ end }} -{{ define "prom_right_table_tail" }} -
    -
    -{{ end }} - -{{/* RHS table head, pass job name. Should be used after prom_right_table_head. */}} -{{ define "prom_right_table_job_head" }} - - {{ . }} - {{ template "prom_query_drilldown" (args (printf "sum(up{job='%s'})" .)) }} / {{ template "prom_query_drilldown" (args (printf "count(up{job='%s'})" .)) }} - - - CPU - {{ template "prom_query_drilldown" (args (printf "avg by(job)(irate(process_cpu_seconds_total{job='%s'}[5m]))" .) "s/s" "humanizeNoSmallPrefix") }} - - - Memory - {{ template "prom_query_drilldown" (args (printf "avg by(job)(process_resident_memory_bytes{job='%s'})" .) "B" "humanize1024") }} - -{{ end }} - - {{ define "prom_content_head" }} -
    -
    -{{ template "prom_graph_timecontrol" . }} +
    + {{ template "prom_graph_timecontrol" . }} {{ end }} {{ define "prom_content_tail" }} -
    -
    +
    {{ end }} {{ define "prom_graph_timecontrol" }} -
    -
    -
    - -
    -
    - -
    -
    -
    - - - + + +
    +
    +
    + + + +
    +
    +
    -
    - - {{ end }} {{/* Bottom of all pages. */}} + {{ define "tail" }} - + {{ end }} diff --git a/bundles/prometheus/files/consoles/cpu.html b/bundles/prometheus/files/consoles/cpu.html new file mode 100644 index 0000000..64a2cad --- /dev/null +++ b/bundles/prometheus/files/consoles/cpu.html @@ -0,0 +1,60 @@ +{{ template "head" . }} + +{{ template "prom_right_table_head" }} + + CPU(s): {{ template "prom_query_drilldown" (args (printf "scalar(count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'})))" .Params.instance)) }} + +{{ range printf "sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='%s'}[5m])) * 100 / scalar(count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'})))" .Params.instance .Params.instance | query | sortByLabel "mode" }} + + {{ .Labels.mode | title }} CPU + {{ .Value | printf "%.1f" }}% + +{{ end }} + Misc + + Processes Running + {{ template "prom_query_drilldown" (args (printf "node_procs_running{job='node-exporter',instance='%s'}" .Params.instance) "" "humanize") }} + + + Processes Blocked + {{ template "prom_query_drilldown" (args (printf "node_procs_blocked{job='node-exporter',instance='%s'}" .Params.instance) "" "humanize") }} + + + Forks + {{ template "prom_query_drilldown" (args (printf "irate(node_forks_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} + + + Context Switches + {{ template "prom_query_drilldown" (args (printf "irate(node_context_switches_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} + + + Interrupts + {{ template "prom_query_drilldown" (args (printf "irate(node_intr_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} + + + 1m Loadavg + {{ template "prom_query_drilldown" (args (printf "node_load1{job='node-exporter',instance='%s'}" .Params.instance)) }} + + + +{{ template "prom_right_table_tail" }} + +{{ template "prom_content_head" . }} +

    Node CPU - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    + +

    CPU Usage

    +
    + +{{ template "prom_content_tail" . }} + +{{ template "tail" }} diff --git a/bundles/prometheus/files/consoles/disk.html b/bundles/prometheus/files/consoles/disk.html new file mode 100644 index 0000000..52afac7 --- /dev/null +++ b/bundles/prometheus/files/consoles/disk.html @@ -0,0 +1,78 @@ +{{ template "head" . }} + +{{ template "prom_right_table_head" }} + + Disks + +{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }} + {{ .Labels.device }} + + Utilization + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} + + + Throughput + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} + + + Avg Read Time + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) / irate(node_disk_reads_completed_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} + + + Avg Write Time + {{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) / irate(node_disk_writes_completed_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} + +{{ end }} + + Filesystem Fullness + +{{ define "roughlyNearZero" }} +{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} +{{ end }} +{{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} + + {{ .Labels.mountpoint }} + {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} + +{{ end }} + + +{{ template "prom_right_table_tail" }} + +{{ template "prom_content_head" . }} +

    Node Disk - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    + +

    Disk I/O Utilization

    +
    + +

    Filesystem Usage

    +
    + +{{ template "prom_content_tail" . }} + +{{ template "tail" }} diff --git a/bundles/prometheus/files/consoles/index.html b/bundles/prometheus/files/consoles/index.html index 29c2884..f0b102d 100644 --- a/bundles/prometheus/files/consoles/index.html +++ b/bundles/prometheus/files/consoles/index.html @@ -1,81 +1,102 @@ -{{/* vim: set ft=html: */}} +{{/* vim: set ft=gotmpl: */}} -{{ define "menu" }} -foobar +{{ define "s6_rc_services_count" }} +{{ $instance := .arg0 }}{{ $bundle := .arg1 }} +{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{instance='%s',bundle='%s',service=~'.*-srv'}" $instance $bundle)) }} {{ end }} {{ template "head" . }} -{{ template "prom_right_table_head" . }} - - S6 - inazuma - - - default - {{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='default'}")) }} - - - containers - {{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='containers'}")) }} - - - www - {{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='www'}")) }} - - - services - {{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='services'}")) }} - - - S6 - shimakaze - - - default - {{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='default'}")) }} - - - containers - {{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='containers'}")) }} - - - www - {{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='www'}")) }} - - - services - {{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='services'}")) }} - -{{ template "prom_right_table_tail" . }} - {{ template "prom_content_head" . }}
    -

    Overview

    -

    Traefik Requests OK - inazuma

    -
    - -

    Jellyfin HTTP Requests - shimakaze

    -
    - +
    +

    Overview

    + IZ: {{ template "s6_rc_services_count" (args "inazuma" "default") }} + SK: {{ template "s6_rc_services_count" (args "shimakaze" "default") }} +
    +
    +

    Traefik Latency

    +
    + +
    +
    +

    Node CPU Utilisation

    +
    + +
    +
    +

    Node CPU Saturation

    +
    + +
    +
    +

    Node Memory Utilisation

    +
    + +
    +
    +

    Node Memory Saturation

    +
    + +
    {{ template "prom_content_tail" . }} diff --git a/bundles/prometheus/files/consoles/node-cpu.html b/bundles/prometheus/files/consoles/node-cpu.html deleted file mode 100644 index 64a2cad..0000000 --- a/bundles/prometheus/files/consoles/node-cpu.html +++ /dev/null @@ -1,60 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - - CPU(s): {{ template "prom_query_drilldown" (args (printf "scalar(count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'})))" .Params.instance)) }} - -{{ range printf "sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='%s'}[5m])) * 100 / scalar(count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'})))" .Params.instance .Params.instance | query | sortByLabel "mode" }} - - {{ .Labels.mode | title }} CPU - {{ .Value | printf "%.1f" }}% - -{{ end }} - Misc - - Processes Running - {{ template "prom_query_drilldown" (args (printf "node_procs_running{job='node-exporter',instance='%s'}" .Params.instance) "" "humanize") }} - - - Processes Blocked - {{ template "prom_query_drilldown" (args (printf "node_procs_blocked{job='node-exporter',instance='%s'}" .Params.instance) "" "humanize") }} - - - Forks - {{ template "prom_query_drilldown" (args (printf "irate(node_forks_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} - - - Context Switches - {{ template "prom_query_drilldown" (args (printf "irate(node_context_switches_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} - - - Interrupts - {{ template "prom_query_drilldown" (args (printf "irate(node_intr_total{job='node-exporter',instance='%s'}[5m])" .Params.instance) "/s" "humanize") }} - - - 1m Loadavg - {{ template "prom_query_drilldown" (args (printf "node_load1{job='node-exporter',instance='%s'}" .Params.instance)) }} - - - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Node CPU - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    - -

    CPU Usage

    -
    - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/bundles/prometheus/files/consoles/node-disk.html b/bundles/prometheus/files/consoles/node-disk.html deleted file mode 100644 index 52afac7..0000000 --- a/bundles/prometheus/files/consoles/node-disk.html +++ /dev/null @@ -1,78 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - - Disks - -{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }} - {{ .Labels.device }} - - Utilization - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} - - - Throughput - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} - - - Avg Read Time - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) / irate(node_disk_reads_completed_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} - - - Avg Write Time - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_write_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) / irate(node_disk_writes_completed_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "s" "humanize") }} - -{{ end }} - - Filesystem Fullness - -{{ define "roughlyNearZero" }} -{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} -{{ end }} -{{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} - - {{ .Labels.mountpoint }} - {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} - -{{ end }} - - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Node Disk - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    - -

    Disk I/O Utilization

    -
    - -

    Filesystem Usage

    -
    - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/bundles/prometheus/files/consoles/node-overview.html b/bundles/prometheus/files/consoles/node-overview.html deleted file mode 100644 index edeca2e..0000000 --- a/bundles/prometheus/files/consoles/node-overview.html +++ /dev/null @@ -1,121 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - Overview - - User CPU - {{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }} - - - System CPU - {{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }} - - - Memory Total - {{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }} - - - Memory Free - {{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }} - - - Network - -{{ range printf "node_network_receive_bytes_total{job='node-exporter',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Received - {{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }} - - - {{ .Labels.device }} Transmitted - {{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }} - -{{ end }} - - Disks - -{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Utilization - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }} - -{{ end }} -{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }} - - {{ .Labels.device }} Throughput - {{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }} - -{{ end }} - - Filesystem Fullness - -{{ define "roughlyNearZero" }} -{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} -{{ end }} -{{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} - - {{ .Labels.mountpoint }} - {{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }} - -{{ end }} - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -

    Node Overview - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    - -

    CPU Usage

    -
    - - -

    Disk I/O Utilization

    -
    - - -

    Memory

    -
    - - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/bundles/prometheus/files/consoles/node.html b/bundles/prometheus/files/consoles/node.html new file mode 100644 index 0000000..9ef79ce --- /dev/null +++ b/bundles/prometheus/files/consoles/node.html @@ -0,0 +1,133 @@ +{{/* vim: set ft=gotmpl: */}} + +{{ template "head" . }} + +{{ template "prom_content_head" . }} +
    +
    +

    {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}

    + + + + + + + + + + + + + + + + + + + + + + +
    Overview
    User CPU{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}
    System CPU{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}
    Memory Total{{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}
    Memory Free{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}
    + + + {{ range printf "node_network_receive_bytes_total{job='node-exporter',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }} + + + + + + + + + {{ end }} +
    Network
    {{ .Labels.device }} Received{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}
    {{ .Labels.device }} Transmitted{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}
    + + + {{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }} + + + + + {{ end }} + {{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }} + + + + + {{ end }} +
    Disks
    {{ .Labels.device }} Utilization{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}
    {{ .Labels.device }} Throughput{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}
    + + + {{ define "roughlyNearZero" }} + {{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} + {{ end }} + {{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} + + + + + {{ end }} +
    Filesystem Fullness
    {{ .Labels.mountpoint }}{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}
    +
    +
    +

    CPU Usage

    +
    + +
    +
    +

    Disk I/O Utilization

    +
    + +
    +
    +

    Memory

    +
    + +
    +
    +{{ template "prom_content_tail" . }} + +{{ template "tail" . }} diff --git a/bundles/prometheus/files/consoles/prom.html b/bundles/prometheus/files/consoles/prom.html new file mode 100644 index 0000000..4bc5eb7 --- /dev/null +++ b/bundles/prometheus/files/consoles/prom.html @@ -0,0 +1,88 @@ +{{/* vim: set ft=gotmpl: */}} + +{{ template "head" . }} + +{{ template "prom_content_head" . }} +
    +
    +

    Prometheus

    + + + + + + + + + + + + + + +
    Overview
    CPU{{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }}
    Memory{{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Params.instance) "B" "humanize1024") }}
    Version{{ with query (printf "prometheus_build_info{job='prometheus',instance='%s'}" .Params.instance) }}{{. | first | label "version"}}{{end}}
    + + + + + + + + + + + + + + +
    Storage
    Ingested Samples{{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }}
    Head Series{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_head_series{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}
    Blocks Loaded{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_blocks_loaded{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}
    + + + + + + + + + + + + + + +
    Rules
    Evaluation Duration{{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_evaluator_duration_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}
    Notification Latency{{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_notifications_latency_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}
    Notification Queue{{ template "prom_query_drilldown" (args (printf "prometheus_notifications_queue_length{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}
    +
    +
    +

    Ingested Samples

    +
    + +
    +
    +

    HTTP Server

    +
    + +
    +
    +{{ template "prom_content_tail" . }} + +{{ template "tail" }} diff --git a/bundles/prometheus/files/consoles/prometheus-overview.html b/bundles/prometheus/files/consoles/prometheus-overview.html deleted file mode 100644 index 08e027d..0000000 --- a/bundles/prometheus/files/consoles/prometheus-overview.html +++ /dev/null @@ -1,96 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - - Overview - - - CPU - {{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }} - - - Memory - {{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Params.instance) "B" "humanize1024") }} - - - Version - {{ with query (printf "prometheus_build_info{job='prometheus',instance='%s'}" .Params.instance) }}{{. | first | label "version"}}{{end}} - - - - Storage - - - Ingested Samples - {{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }} - - - Head Series - {{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_head_series{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }} - - - Blocks Loaded - {{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_blocks_loaded{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }} - - - Rules - - - Evaluation Duration - {{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_evaluator_duration_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }} - - - Notification Latency - {{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_notifications_latency_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }} - - - Notification Queue - {{ template "prom_query_drilldown" (args (printf "prometheus_notifications_queue_length{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }} - - - HTTP Server - -{{ range printf "prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s'}" .Params.instance | query | sortByLabel "handler" }} - - {{ .Labels.handler }} - {{ template "prom_query_drilldown" (args (printf "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s',handler='%s'}[5m])" .Labels.instance .Labels.handler) "/s" "humanizeNoSmallPrefix") }} - -{{ end }} - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} -
    -

    Prometheus Overview - {{ .Params.instance }}

    - -

    Ingested Samples

    -
    - - -

    HTTP Server

    -
    - -
    -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/bundles/prometheus/files/kube.yaml b/bundles/prometheus/files/kube.yaml index 952f485..7a7f961 100644 --- a/bundles/prometheus/files/kube.yaml +++ b/bundles/prometheus/files/kube.yaml @@ -9,40 +9,97 @@ metadata: traefik.http.routers.prometheus.entrypoints: http,https traefik.http.routers.prometheus.rule: Host(`prom.gzr.im`) traefik.http.routers.prometheus.service: prometheus + traefik.http.routers.prometheus.middlewares: prometheus-consoles-redirect traefik.http.services.prometheus.loadbalancer.server.port: 9090 + traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.regex: ^https://prom.gzr.im/consoles(|/)$ + traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.replacement: https://prom.gzr.im/consoles/index.html + traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.permanent: true spec: - restartPolicy: Never dnsPolicy: Default dnsConfig: searches: - tail2d6ce.ts.net + securityContext: + runAsUser: 0 containers: - name: prom - image: quay.io/prometheus/prometheus:v2.52.0 + image: quay.io/prometheus/prometheus:v3.7.3 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=30d + - --storage.tsdb.retention.size=6GB + - --web.page-title=Prometheus + - --web.console.templates=/usr/share/prometheus/consoles + - --web.console.libraries=/usr/share/prometheus/console_libraries + - --web.user-assets=/usr/share/prometheus/console_assets volumeMounts: - - mountPath: /etc/prometheus:z + - mountPath: /etc/prometheus name: config readOnly: true - - mountPath: /usr/share/prometheus/consoles:z + - mountPath: /usr/share/prometheus/consoles name: config subPath: consoles readOnly: true - - mountPath: /usr/share/prometheus/console_libraries:z + - mountPath: /usr/share/prometheus/console_libraries name: config subPath: console_libraries readOnly: true + - mountPath: /usr/share/prometheus/console_assets + name: config + subPath: console_assets + readOnly: true + - mountPath: /prometheus + name: prom-data ports: - containerPort: 9090 protocol: TCP + livenessProbe: + exec: + command: + - sh + - -c + - wget -qO - http://localhost:9090/-/healthy | grep -q 'is Healthy' resources: limits: cpu: 100m - memory: 200Mi + memory: 256Mi requests: cpu: 25m - memory: 100Mi + memory: 128Mi + + - name: alert + image: quay.io/prometheus/alertmanager:v0.29.0 + volumeMounts: + - mountPath: /etc/alertmanager + name: config + readOnly: true + - mountPath: /alertmanager + name: alert-data + ports: + - containerPort: 9093 + protocol: TCP + livenessProbe: + exec: + command: + - sh + - -c + - wget -qO - http://localhost:9093/-/healthy | grep -q 'OK' + resources: + limits: + cpu: 50m + memory: 64Mi + volumes: - name: config hostPath: path: /etc/prometheus type: Directory + - name: prom-data + hostPath: + path: /var/lib/prometheus/prom + type: DirectoryOrCreate + - name: alert-data + hostPath: + path: /var/lib/prometheus/alert + type: DirectoryOrCreate diff --git a/bundles/prometheus/files/prometheus.yml b/bundles/prometheus/files/prometheus.yml index 0614c8c..38841e5 100644 --- a/bundles/prometheus/files/prometheus.yml +++ b/bundles/prometheus/files/prometheus.yml @@ -9,16 +9,24 @@ global: # label_name_length_limit: 200 # label_value_length_limit: 200 +# TODO: pending release +# storage: +# tsdb: +# retention: +# time: 30d +# size: 6GB + rule_files: - - rule_*.yml + - /etc/prometheus/rules/*.json + - /etc/prometheus/rules/*.yaml + - /etc/prometheus/rules/*.yml scrape_configs: - - job_name: prometheus static_configs: - - targets: [localhost:9090] - labels: - instance: inazuma + - targets: [localhost:9090] + labels: + instance: inazuma - job_name: service-discovery file_sd_configs: @@ -26,3 +34,9 @@ scrape_configs: - /etc/prometheus/services/*.json - /etc/prometheus/services/*.yaml - /etc/prometheus/services/*.yml + +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: [localhost:9093] diff --git a/bundles/prometheus/files/rules.yml b/bundles/prometheus/files/rules.yml new file mode 100644 index 0000000..45b130a --- /dev/null +++ b/bundles/prometheus/files/rules.yml @@ -0,0 +1,218 @@ +groups: + - name: Prometheus + rules: + - alert: PrometheusTargetMissing + expr: up == 0 + for: 10m + labels: + severity: critical + annotations: + summary: |- + Prometheus target missing (instance {{ $labels.instance }}) + description: |- + A Prometheus target has disappeared. An exporter might be crashed. + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - name: Host + rules: + - alert: HostOutOfMemory + expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < .10) + for: 2m + labels: + severity: warning + annotations: + summary: |- + Host out of memory (instance {{ $labels.instance }}) + description: |- + Node memory is filling up (< 10% left) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostMemoryUnderMemoryPressure + expr: (rate(node_vmstat_pgmajfault[5m]) > 1000) + for: 0m + labels: + severity: warning + annotations: + summary: |- + Host memory under memory pressure (instance {{ $labels.instance }}) + description: |- + The node is under heavy memory pressure. High rate of loading memory pages from disk. + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostUnusualDiskReadRate + expr: (rate(node_disk_io_time_seconds_total[5m]) > .80) + for: 0m + labels: + severity: warning + annotations: + summary: |- + Host unusual disk read rate (instance {{ $labels.instance }}) + description: |- + Disk is too busy (IO wait > 80%) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostOutOfDiskSpace + expr: (node_filesystem_avail_bytes{fstype!~"^(fuse.*|-tmpfs|cifs|nfs)",mountpoint!="/media/usb"} / node_filesystem_size_bytes < .10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) + for: 2m + labels: + severity: critical + annotations: + summary: |- + Host out of disk space (instance {{ $labels.instance }}) + description: |- + Disk is almost full (< 10% left) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostHighCpuLoad + expr: 1 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > .80 + for: 10m + labels: + severity: warning + annotations: + summary: |- + Host high CPU load (instance {{ $labels.instance }}) + description: |- + CPU load is > 80% + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostUnusualNetworkThroughputOut + expr: ((rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80) + for: 0m + labels: + severity: warning + annotations: + summary: |- + Host unusual outbound network traffic (instance {{ $labels.instance }}) + description: |- + Host transmit bandwidth is high (>80%) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostUnusualNetworkThroughputIn + expr: ((rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80) + for: 0m + labels: + severity: warning + annotations: + summary: |- + Host unusual inbound network traffic (instance {{ $labels.instance }}) + description: |- + Host receive bandwidth is high (>80%) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostServiceDown + expr: count(s6_rc_service_up{bundle="default", service=~".*-srv"}) by (instance, service) == 0 + for: 0m + labels: + severity: critical + annotations: + summary: |- + s6-rc service down (instance {{ $labels.instance }}) + description: |- + s6-rc service {{ $labels.service }} on {{ $labels.instance }} is down! + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - name: Traefik + rules: + - alert: TraefikHighHttp4xxErrorRateService + expr: sum(rate(traefik_service_requests_total{service!~".*go-away@.*",code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5 + for: 5m + labels: + severity: critical + annotations: + summary: |- + Traefik high HTTP 4xx error rate service (instance {{ $labels.instance }}) + description: |- + Traefik service 4xx error rate is above 5% + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: TraefikHighHttp5xxErrorRateService + expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5 + for: 5m + labels: + severity: critical + annotations: + summary: |- + Traefik high HTTP 5xx error rate service (instance {{ $labels.instance }}) + description: |- + Traefik service 5xx error rate is above 5% + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: TraefikLatencyHigh + expr: histogram_quantile(0.99, sum(rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~"https?"}[3m])) by (le, instance)) > 10 + for: 5m + labels: + severity: warning + annotations: + summary: |- + Traefik latency high (instance {{ $labels.instance }}) + description: |- + Traefik p99 latency is higher than 10 seconds + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - name: Podman + rules: + - alert: ContainerHighMemoryUsage + expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) > 80 + for: 2m + labels: + severity: warning + annotations: + summary: |- + Container High Memory usage (instance {{ $labels.instance }}) + description: |- + Container Memory usage is above 80% + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: ContainerLowMemoryUsage + expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) < 20 + for: 7d + labels: + severity: info + annotations: + summary: |- + Container Low Memory usage (instance {{ $labels.instance }}) + description: |- + Container Memory usage is under 20% for 1 week. Consider reducing the allocated memory. + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - name: BadBots + rules: + - alert: BadBotsFail2BanRate + expr: rate(f2b_jail_banned_current[15m]) > 10 + for: 0m + labels: + severity: warning + annotations: + summary: |- + Increased rate of bans + description: |- + Fail2Ban has banned more than 10 IPs in the last 15 minutes. + VALUE = {{ $value }} + LABELS = {{ $labels }} + + # - alert: BadBotsGoAway... + # expr: rate(f2b_jail_banned_current[15m]) > 10 + # for: 0m + # labels: + # severity: warning + # annotations: + # summary: |- + # Increased rate of bans + # description: |- + # Fail2Ban has banned more than 10 IPs in the last 15 minutes. + # VALUE = {{ $value }} + # LABELS = {{ $labels }} diff --git a/bundles/prometheus/items.py b/bundles/prometheus/items.py index 234aca5..99aecbc 100644 --- a/bundles/prometheus/items.py +++ b/bundles/prometheus/items.py @@ -1,16 +1,34 @@ from sys import exit from os.path import join +actions = { + "reload_prometheus": { + "command": "podman kill -s HUP prometheus-prom", + "triggered": True, + }, + "reload_alertmanager": { + "command": "podman kill -s HUP prometheus-alert", + "triggered": True, + }, +} + files = { "/etc/deployments/prometheus/kube.yaml": { - "content_type": "text", - "source": "kube.yaml", + "content_type": "mako", + "context": node.metadata.get("containers")["prometheus"], "triggers": {"svc_s6rc:prometheus:restart"}, }, "/etc/prometheus/prometheus.yml": { "content_type": "text", - "source": "prometheus.yml", - "triggers": {"svc_s6rc:prometheus:restart"}, + "triggers": {"action:reload_prometheus"}, + }, + "/etc/prometheus/rules/generic.yml": { + "source": "rules.yml", + "triggers": {"action:reload_prometheus"}, + }, + "/etc/prometheus/alertmanager.yml": { + "content_type": "mako", + "triggers": {"action:reload_alertmanager"}, }, } @@ -26,11 +44,19 @@ repo.libs.gen.add_files_recursive( target_root="/etc/prometheus", files_root=join(repo.path, "bundles", "prometheus", "files"), ) +repo.libs.gen.add_files_recursive( + files, + join(repo.path, "bundles", "prometheus", "files", "console_assets"), + target_root="/etc/prometheus", + files_root=join(repo.path, "bundles", "prometheus", "files"), +) directories = { - "/etc/prometheus/services": {}, - "/etc/prometheus/consoles": {}, - "/etc/prometheus/console_libraries": {}, + "/etc/prometheus/services": {"purge": True}, + "/etc/prometheus/rules": {"purge": True}, + "/etc/prometheus/consoles": {"purge": True}, + "/etc/prometheus/console_libraries": {"purge": True}, + "/etc/prometheus/console_assets": {"purge": True}, } # NOTE: services that want to have their metrics scraped need to expose them -- cgit 1.4.1