diff options
19 files changed, 1898 insertions, 441 deletions
diff --git a/bundles/prometheus/files/alertmanager.yml b/bundles/prometheus/files/alertmanager.yml new file mode 100644 index 0000000..e725338 --- /dev/null +++ b/bundles/prometheus/files/alertmanager.yml @@ -0,0 +1,38 @@ +global: + +route: + group_by: [instance, alertname, severity] + group_wait: 30s + group_interval: 5m + repeat_interval: 4h + mute_time_intervals: + - offhours + routes: + - matchers: [severity=critical] + repeat_interval: 1h + - matchers: [severity=warning] + repeat_interval: 4h + - matchers: [severity=info] + repeat_interval: 24h + receiver: telegram + +time_intervals: + - name: offhours + time_intervals: + - location: Europe/Berlin + times: + - start_time: 23:00 + end_time: 09:00 + +receivers: + - name: telegram + telegram_configs: + - bot_token: ${repo.vault.cmd(f"rbw get tgnotify/token")} + chat_id: ${repo.vault.cmd(f"rbw get tgnotify/chatid")} + message: | + {{if eq .GroupLabels.severity "critical"}}🚨{{end -}} + {{if eq .GroupLabels.severity "warning"}}⚠️{{end -}} + {{if eq .GroupLabels.severity "info"}}📬{{end -}} + <b>{{with .GroupLabels.instance}}{{.}}/{{end}}{{.GroupLabels.alertname}}</b> + {{.CommonAnnotations.summary}} + {{.CommonAnnotations.description}} diff --git a/bundles/prometheus/files/console_assets/ajax-loader.gif b/bundles/prometheus/files/console_assets/ajax-loader.gif new file mode 100644 index 0000000..e192ca8 --- /dev/null +++ b/bundles/prometheus/files/console_assets/ajax-loader.gif Binary files differdiff --git a/bundles/prometheus/files/console_assets/menu.lib b/bundles/prometheus/files/console_assets/menu.lib new file mode 100644 index 0000000..e5cf31f --- /dev/null +++ b/bundles/prometheus/files/console_assets/menu.lib @@ -0,0 +1,51 @@ +{{/* vim: set ft=gotmpl: */}} + +{{/* Navbar, should be passed . */}} +{{ define "navbar" }} +<nav class="navbar sticky-top navbar-expand-sm navbar-dark bg-dark"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false" aria-controls="navbar-nav" aria-label="toggle navigation"> + <span class="navbar-toggler-icon"></span> + </button> + <a class="navbar-brand" href="{{ pathPrefix }}/">Prometheus</a> + </div> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav"> + <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/alerts">Alerts</a></li> + <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/graph">Graph</a></li> + <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/targets">Targets</a></li> + </ul> + </div> +</nav> +{{ end }} + +{{/* LHS menu, should be passed . */}} +{{ define "menu" }} +<nav aria-label="breadcrumb"> + <ol class="breadcrumb rounded-0"> + {{ template "_menuItem" (args . "index.html" "overview") }} + + {{ $ctx := . }} + + {{ template "_menuItem" (args $ctx "node.html?instance=inazuma" "inazuma") }} + {{ template "_menuItem" (args $ctx "node.html?instance=shimakaze" "shimakaze") }} + {{ template "_menuItem" (args $ctx "prom.html?instance=inazuma" "prometheus") }} + </ol> +</nav> +{{ end }} + +{{/* Helper, pass (args . path name) */}} +{{ define "_menuItem" }} +{{ $pagelink := .arg0.Path }} +{{ if .arg0.Params.instance }}{{ $pagelink = (printf "%v?instance=%v" .arg0.Path .arg0.Params.instance) }}{{ end }} +<li class="breadcrumb-item{{ if eq $pagelink .arg1 }} active{{ end }} text-lowercase"> + <a href="{{ .arg1 }}"> + {{ if eq .arg2 "overview" }} + <i class="glyphicon glyphicon-th" title="{{ .arg2 }}"></i> + {{ else }} + {{ .arg2 }} + {{ end }} + </a> +</li> +{{ end }} diff --git a/bundles/prometheus/files/console_assets/prom.lib b/bundles/prometheus/files/console_assets/prom.lib new file mode 100644 index 0000000..fbc39ff --- /dev/null +++ b/bundles/prometheus/files/console_assets/prom.lib @@ -0,0 +1,150 @@ +{{/* vim: set ft=gotmpl: */}} +{{/* Load Prometheus console library JS/CSS. Should go in head */}} + +{{ define "prom_console_head" }} +<meta name="viewport" content="width=device-width,initial-scale=1.0"> +<meta name="title" content="Prometheus Console"> +<title>Prometheus Console</title> +<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.css"> +<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/css/bootstrap.min.css"> +<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/css/prom_console.css"> +<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css"> +<style type="text/css"> + body { + background-color: #191d21 !important; + color: #fff !important; + } + .navbar { + margin-bottom: unset !important; + padding-left: 15px !important; + padding-right: 15px !important; + } + .prom_console_content { + padding-top: unset !important; + margin-bottom: unset !important; + } + .prom_content_div { + width: unset !important; + display: grid; + grid-template-columns: 1fr 1fr; + gap: 15px; + } + @media (max-width: 768px) { + .prom_content_div { + grid-template-columns: 1fr; + } + } + .prom_content_div > div { + padding: 10px; + border: 1px solid var(--gray); + } + .prom_content_div * { + background-color: var(--light) !important; + color: var(--dark) !important; + } + .prom_graph_table { + overflow: hidden; + } +</style> +<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.v3.js"></script> +<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.layout.min.js"></script> +<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.js"></script> +<script src="{{ pathPrefix }}/classic/static/vendor/js/jquery-3.5.1.min.js"></script> +<script src="{{ pathPrefix }}/classic/static/vendor/js/popper.min.js"></script> +<script src="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/js/bootstrap.min.js"></script> +<script> + var PATH_PREFIX = "{{ pathPrefix }}"; +</script> +<script src="{{ pathPrefix }}/classic/static/js/prom_console.js"></script> +{{ end }} + +{{/* Top of all pages. */}} +{{ define "head" -}} +<!DOCTYPE html> +<html lang="en"> + <head> + {{ template "prom_console_head" }} + </head> + <body> + {{ template "navbar" . }} + {{ template "menu" . }} +{{ end }} + +{{ define "__prom_query_drilldown_noop" }}{{ . }}{{ end }} +{{ define "humanize" }}{{ humanize . }}{{ end }} +{{ define "humanizeNoSmallPrefix" }}{{ if and (lt . 1.0) (gt . -1.0) }}{{ printf "%.3g" . }}{{ else }}{{ humanize . }}{{ end }}{{ end }} +{{ define "humanize1024" }}{{ humanize1024 . }}{{ end }} +{{ define "humanizeDuration" }}{{ humanizeDuration . }}{{ end }} +{{ define "humanizePercentage" }}{{ humanizePercentage . }}{{ end }} +{{ define "humanizeTimestamp" }}{{ humanizeTimestamp . }}{{ end }} +{{ define "printf.1f" }}{{ printf "%.1f" . }}{{ end }} +{{ define "printf.3g" }}{{ printf "%.3g" . }}{{ end }} + +{{/* prom_query_drilldown (args expr suffix? renderTemplate?) +Displays the result of the expression, with a link to /graph for it. + +renderTemplate is the name of the template to use to render the value. +*/}} +{{ define "prom_query_drilldown" }} +{{ $expr := .arg0 }}{{ $suffix := (or .arg1 "") }}{{ $renderTemplate := (or .arg2 "__prom_query_drilldown_noop") }} +<a class="prom_query_drilldown" href="{{ pathPrefix }}{{ graphLink $expr }}"> {{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }}</a> +{{ end }} + +{{ define "prom_path" }}/consoles/{{ .Path }}?{{ range $param, $value := .Params }}{{ $param }}={{ $value }}&{{ end }}{{ end }}" + +{{ define "prom_content_head" }} + <div class="prom_console_content"> + {{ template "prom_graph_timecontrol" . }} +{{ end }} +{{ define "prom_content_tail" }} + </div> +{{ end }} + +{{ define "prom_graph_timecontrol" }} + <div class="prom_graph_timecontrol"> + <div class="prom_graph_timecontrol_inner"> + <div class="prom_graph_timecontrol_group "> + <button class="btn btn-light pull-left" type="button" id="prom_graph_duration_shrink" title="Shrink the time range."> + <i class="glyphicon glyphicon-minus"></i> + </button><!-- Comments between elements to remove spaces + --><input class="input pull-left align-middle" size="3" title="Time range of graph" type="text" id="prom_graph_duration"><!-- + --><button class="btn btn-light pull-left" type="button" id="prom_graph_duration_grow" title="Grow the time range."> + <i class="glyphicon glyphicon-plus"></i> + </button> + </div> + <div class="prom_graph_timecontrol_group "> + <button class="btn btn-light pull-left" type="button" id="prom_graph_time_back" title="Rewind the end time."> + <i class="glyphicon glyphicon-backward"></i> + </button><!-- + --><input class="input pull-left align-middle" title="End time of graph" placeholder="Until" type="text" id="prom_graph_time_end" size="16" value=""><!-- + --><button class="btn btn-light pull-left" type="button" id="prom_graph_time_forward" title="Advance the end time."> + <i class="glyphicon glyphicon-forward"></i> + </button> + </div> + <div class="prom_graph_timecontrol_group "> + <div class="btn-group dropup prom_graph_timecontrol_refresh pull-left"> + <button type="button" class="btn btn-light pull-left" id="prom_graph_refresh_button" title="Refresh."> + <i class="glyphicon glyphicon-repeat"></i> + <span class="icon-repeat"></span> + (<span id="prom_graph_refresh_button_value">Off</span>) + </button> + <button type="button" class="btn btn-light pull-left dropdown-toggle" data-toggle="dropdown" title="Set autorefresh." aria-haspopup="true" aria-expanded="false"> + <span class="caret"></span> + </button> + <ul class="dropdown-menu" id="prom_graph_refresh_intervals" role="menu"> + </ul> + </div> + </div> + </div> + <script> + new PromConsole.TimeControl(); + </script> + </div> +{{ end }} + +{{/* Bottom of all pages. */}} + +{{ define "tail" }} + </body> +</html> +{{ end }} diff --git a/bundles/prometheus/files/console_assets/prom_console.css b/bundles/prometheus/files/console_assets/prom_console.css new file mode 100644 index 0000000..e5b8be7 --- /dev/null +++ b/bundles/prometheus/files/console_assets/prom_console.css @@ -0,0 +1,190 @@ +.navbar { + margin-bottom: 20px; +} + +div.prom_lhs_menu { + float: left; + margin-right: 2ex; + background: #222; + min-height: 100%; + overflow: auto; + width: auto; + margin-left: -40px; + margin-top: 55px; + height: 100%; +} +.prom_lhs_menu_nav { + padding-right: 0; +} +.prom_lhs_menu ul { + list-style: none; + padding-left: .5ex; + margin-left: .5ex; +} +.prom_lhs_menu li { + padding-right: 1ex; + padding-left: 1ex; + font-size: 0.9rem; +} +.prom_lhs_menu a, +.prom_lhs_menu li { + color: #999999; + display: block; +} +.prom_lhs_menu a { + text-decoration: none; + padding-top: 0; + padding-bottom: 0; +} +.prom_lhs_menu_selected a { + pointer-events: none; + cursor: default; +} +li.prom_lhs_menu_selected { + background: #555555; + background-clip: padding-box; +} +.prom_lhs_menu li:hover { + background: #666666; +} + +.prom_console_rhs { + float: right; + margin-left: 1ex; + height: 100%; + padding-top: 55px; +} +.prom_console_rhs table { + margin-top: 1ex; + margin-bottom: 32px; /* Space for time control. */ +} +.prom_console_rhs th { + text-align: center; +} +.prom_console_rhs td:nth-child(2) { + text-align: right; +} + +.prom_console_content { + overflow: visible; + margin-bottom: 32px; /* Space for time control. */ + padding-top: 70px; /* Space for navbar */ +} + +a.prom_query_drilldown { + text-decoration: none; + color: black; +} +a.prom_query_drilldown:hover, a.prom_query_drilldown:active { + text-decoration: underline; +} + +#samplesGraph { + width: 100%; +} + +.prom_content_div { + width: 80%; +} + +.prom_graph_table { +} +.rickshaw_legend { + padding: 2px; + margin-top: 1px; +} +.rickshaw_legend li { + min-width: 0; +} +.rickshaw_legend ul li { + list-style-type: none; + display: inline-block; +} +.rickshaw_graph { + width: 100%; + padding: 0; +} +.prom_graph_hover_flipped.x_label { + right: 0; +} +.prom_graph_hover_flipped.item { + right: 10px; +} +.rickshaw_graph .detail .prom_graph_hover_flipped.item:before { + content: "\25b8"; + left: auto; + right: 1px; + font-size: 0.8em; +} +.prom_graph_ytitle { + -webkit-transform: rotate(-90deg); + -moz-transform: rotate(-90deg); + font-size: 11px; + font-family: Arial; + max-width: 13px; + white-space: nowrap; +} +.prom_graph_xtitle { + text-align: center; + font-size: 11px; + font-family: Arial; +} +.prom_graph_loading { + position: absolute; + top: 0px; + left: 0px; +} +.prom_graph_error { + font-family: Arial; + text-align: center; +} +a.prom_graph_link { + text-decoration: none; + color: black; +} + + +.prom_graph_timecontrol { + background: #222; + position: fixed; + padding: 2px; + bottom: 0; + left: 0; + z-index: 2; + width: 100%; +} + +.prom_graph_timecontrol_inner { + text-align: center; + height: 31px; +} + +.prom_graph_timecontrol_group { + display: inline-block; + margin: 1px 5px; +} +.prom_graph_timecontrol_group .btn { + font-size: 0.8em; + border-radius: 0; + height: 29px; + border-color: #ccc; +} + +.prom_graph_timecontrol_group .input { + height: 29px; + padding: 6px 12px; + font-size: 14px; + line-height: 1.42857143; + color: #555; + background-color: #fff; + background-image: none; + border: 1px solid #ccc; + text-align: center; +} + +.prom_graph_timecontrol_group .dropdown-menu { + min-width: 90px; +} +.prom_graph_timecontrol_group .dropdown-menu li { + text-align: center; +} diff --git a/bundles/prometheus/files/console_assets/prom_console.js b/bundles/prometheus/files/console_assets/prom_console.js new file mode 100644 index 0000000..cb3501c --- /dev/null +++ b/bundles/prometheus/files/console_assets/prom_console.js @@ -0,0 +1,687 @@ +/* + * Functions to make it easier to write prometheus consoles, such + * as graphs. + * + */ +PromConsole = {}; + +PromConsole.NumberFormatter = {}; +PromConsole.NumberFormatter.prefixesBig = ["k", "M", "G", "T", "P", "E", "Z", "Y"]; +PromConsole.NumberFormatter.prefixesBig1024 = ["ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"]; +PromConsole.NumberFormatter.prefixesSmall = ["m", "u", "n", "p", "f", "a", "z", "y"]; + +PromConsole._stripTrailingZero = function(x) { + if (x.indexOf("e") == -1) { + // It's not safe to strip if it's scientific notation. + return x.replace(/\.?0*$/, ''); + } + return x; +}; + +// Humanize a number. +PromConsole.NumberFormatter.humanize = function(x) { + if (x === null) { + return "null"; + } + var ret = PromConsole.NumberFormatter._humanize( + x, PromConsole.NumberFormatter.prefixesBig, + PromConsole.NumberFormatter.prefixesSmall, 1000); + x = ret[0]; + var prefix = ret[1]; + if (Math.abs(x) < 1) { + return x.toExponential(3) + prefix; + } + return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix; +}; + +// Humanize a number, don't use milli/micro/etc. prefixes. +PromConsole.NumberFormatter.humanizeNoSmallPrefix = function(x) { + if (x === null) { + return "null"; + } + if (Math.abs(x) < 1) { + return PromConsole._stripTrailingZero(x.toPrecision(3)); + } + var ret = PromConsole.NumberFormatter._humanize( + x, PromConsole.NumberFormatter.prefixesBig, + [], 1000); + x = ret[0]; + var prefix = ret[1]; + return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix; +}; + +// Humanize a number with 1024 as the base, rather than 1000. +PromConsole.NumberFormatter.humanize1024 = function(x) { + if (x === null) { + return "null"; + } + var ret = PromConsole.NumberFormatter._humanize( + x, PromConsole.NumberFormatter.prefixesBig1024, + [], 1024); + x = ret[0]; + var prefix = ret[1]; + if (Math.abs(x) < 1) { + return x.toExponential(3) + prefix; + } + return PromConsole._stripTrailingZero(x.toFixed(3)) + prefix; +}; + +// Humanize a number, returning an exact representation. +PromConsole.NumberFormatter.humanizeExact = function(x) { + if (x === null) { + return "null"; + } + var ret = PromConsole.NumberFormatter._humanize( + x, PromConsole.NumberFormatter.prefixesBig, + PromConsole.NumberFormatter.prefixesSmall, 1000); + return ret[0] + ret[1]; +}; + +PromConsole.NumberFormatter._humanize = function(x, prefixesBig, prefixesSmall, factor) { + var prefix = ""; + if (x === 0) { + /* Do nothing. */ + } else if (Math.abs(x) >= 1) { + for (var i = 0; i < prefixesBig.length && Math.abs(x) >= factor; ++i) { + x /= factor; + prefix = prefixesBig[i]; + } + } else { + for (var i = 0; i < prefixesSmall.length && Math.abs(x) < 1; ++i) { + x *= factor; + prefix = prefixesSmall[i]; + } + } + return [x, prefix]; +}; + + +PromConsole.TimeControl = function() { + document.getElementById("prom_graph_duration_shrink").onclick = this.decreaseDuration.bind(this); + document.getElementById("prom_graph_duration_grow").onclick = this.increaseDuration.bind(this); + document.getElementById("prom_graph_time_back").onclick = this.decreaseEnd.bind(this); + document.getElementById("prom_graph_time_forward").onclick = this.increaseEnd.bind(this); + document.getElementById("prom_graph_refresh_button").onclick = this.refresh.bind(this); + this.durationElement = document.getElementById("prom_graph_duration"); + this.endElement = document.getElementById("prom_graph_time_end"); + this.durationElement.oninput = this.dispatch.bind(this); + this.endElement.oninput = this.dispatch.bind(this); + this.endElement.oninput = this.dispatch.bind(this); + this.refreshValueElement = document.getElementById("prom_graph_refresh_button_value"); + + var refreshList = document.getElementById("prom_graph_refresh_intervals"); + var refreshIntervals = ["Off", "1m", "5m", "15m", "1h"]; + for (var i = 0; i < refreshIntervals.length; ++i) { + var li = document.createElement("li"); + li.onclick = this.setRefresh.bind(this, refreshIntervals[i]); + li.textContent = refreshIntervals[i]; + refreshList.appendChild(li); + } + + this.durationElement.value = PromConsole.TimeControl.prototype.getHumanDuration( + PromConsole.TimeControl._initialValues.duration); + if (!PromConsole.TimeControl._initialValues.endTimeNow) { + this.endElement.value = PromConsole.TimeControl.prototype.getHumanDate( + new Date(PromConsole.TimeControl._initialValues.endTime * 1000)); + } +}; + +PromConsole.TimeControl.timeFactors = { + "y": 60 * 60 * 24 * 365, + "w": 60 * 60 * 24 * 7, + "d": 60 * 60 * 24, + "h": 60 * 60, + "m": 60, + "s": 1 +}; + +PromConsole.TimeControl.stepValues = [ + "10s", "1m", "5m", "15m", "30m", "1h", "2h", "6h", "12h", "1d", "2d", + "1w", "2w", "4w", "8w", "1y", "2y" +]; + +PromConsole.TimeControl.prototype._setHash = function() { + var duration = this.parseDuration(this.durationElement.value); + var endTime = this.getEndDate() / 1000; + window.location.hash = "#pctc" + encodeURIComponent(JSON.stringify( + { duration: duration, endTime: endTime })); +}; + +PromConsole.TimeControl._initialValues = function() { + var hash = window.location.hash; + var values; + if (hash.indexOf('#pctc') === 0) { + values = JSON.parse(decodeURIComponent(hash.substring(5))); + } else { + values = { duration: 3600, endTime: 0 }; + } + if (values.endTime == 0) { + values.endTime = new Date().getTime() / 1000; + values.endTimeNow = true; + } + return values; +}(); + +PromConsole.TimeControl.prototype.parseDuration = function(durationText) { + var durationRE = new RegExp("^([0-9]+)([ywdhms]?)$"); + var matches = durationText.match(durationRE); + if (!matches) { return 3600; } + var value = parseInt(matches[1]); + var unit = matches[2] || 's'; + return value * PromConsole.TimeControl.timeFactors[unit]; +}; + +PromConsole.TimeControl.prototype.getHumanDuration = function(duration) { + var units = []; + for (var key in PromConsole.TimeControl.timeFactors) { + units.push([PromConsole.TimeControl.timeFactors[key], key]); + } + units.sort(function(a, b) { return b[0] - a[0]; }); + for (var i = 0; i < units.length; ++i) { + if (duration % units[i][0] === 0) { + return (duration / units[i][0]) + units[i][1]; + } + } + return duration; +}; + +PromConsole.TimeControl.prototype.increaseDuration = function() { + var durationSeconds = this.parseDuration(this.durationElement.value); + for (var i = 0; i < PromConsole.TimeControl.stepValues.length; i++) { + if (durationSeconds < this.parseDuration(PromConsole.TimeControl.stepValues[i])) { + this.setDuration(PromConsole.TimeControl.stepValues[i]); + this.dispatch(); + return; + } + } +}; + +PromConsole.TimeControl.prototype.decreaseDuration = function() { + var durationSeconds = this.parseDuration(this.durationElement.value); + for (var i = PromConsole.TimeControl.stepValues.length - 1; i >= 0; i--) { + if (durationSeconds > this.parseDuration(PromConsole.TimeControl.stepValues[i])) { + this.setDuration(PromConsole.TimeControl.stepValues[i]); + this.dispatch(); + return; + } + } +}; + +PromConsole.TimeControl.prototype.setDuration = function(duration) { + this.durationElement.value = duration; + this._setHash(); +}; + +PromConsole.TimeControl.prototype.getEndDate = function() { + if (this.endElement.value === '') { + return null; + } + var dateParts = this.endElement.value.split(/[- :]/) + return Date.UTC(dateParts[0], dateParts[1] - 1, dateParts[2], dateParts[3], dateParts[4]); +}; + +PromConsole.TimeControl.prototype.getOrSetEndDate = function() { + var date = this.getEndDate(); + if (date) { + return date; + } + date = new Date(); + this.setEndDate(date); + return date.getTime(); +}; + +PromConsole.TimeControl.prototype.getHumanDate = function(date) { + var hours = date.getUTCHours() < 10 ? '0' + date.getUTCHours() : date.getUTCHours(); + var minutes = date.getUTCMinutes() < 10 ? '0' + date.getUTCMinutes() : date.getUTCMinutes(); + return date.getUTCFullYear() + "-" + (date.getUTCMonth() + 1) + "-" + date.getUTCDate() + " " + + hours + ":" + minutes; +}; + +PromConsole.TimeControl.prototype.setEndDate = function(date) { + this.setRefresh("Off"); + this.endElement.value = this.getHumanDate(date); + this._setHash(); +}; + +PromConsole.TimeControl.prototype.increaseEnd = function() { + // Increase duration 25% range & convert ms to s. + this.setEndDate(new Date(this.getOrSetEndDate() + this.parseDuration(this.durationElement.value) * 1000 / 4)); + this.dispatch(); +}; + +PromConsole.TimeControl.prototype.decreaseEnd = function() { + this.setEndDate(new Date(this.getOrSetEndDate() - this.parseDuration(this.durationElement.value) * 1000 / 4)); + this.dispatch(); +}; + +PromConsole.TimeControl.prototype.refresh = function() { + this.endElement.value = ''; + this._setHash(); + this.dispatch(); +}; + +PromConsole.TimeControl.prototype.dispatch = function() { + var durationSeconds = this.parseDuration(this.durationElement.value); + var end = this.getEndDate(); + if (end === null) { + end = new Date().getTime(); + } + for (var i = 0; i < PromConsole._graph_registry.length; i++) { + var graph = PromConsole._graph_registry[i]; + graph.params.duration = durationSeconds; + graph.params.endTime = end / 1000; + graph.dispatch(); + } +}; + +PromConsole.TimeControl.prototype._refreshInterval = null; + +PromConsole.TimeControl.prototype.setRefresh = function(duration) { + if (this._refreshInterval !== null) { + window.clearInterval(this._refreshInterval); + this._refreshInterval = null; + } + if (duration != "Off") { + if (this.endElement.value !== '') { + this.refresh(); + } + var durationSeconds = this.parseDuration(duration); + this._refreshInterval = window.setInterval(this.dispatch.bind(this), durationSeconds * 1000); + } + this.refreshValueElement.textContent = duration; +}; + +// List of all graphs, used by time controls. +PromConsole._graph_registry = []; + +PromConsole.graphDefaults = { + expr: null, // Expression to graph. Can be a list of strings. + node: null, // DOM node to place graph under. + // How long the graph is over, in seconds. + duration: PromConsole.TimeControl._initialValues.duration, + // The unixtime the graph ends at. + endTime: PromConsole.TimeControl._initialValues.endTime, + width: null, // Height of the graph div, excluding titles and legends. + // Defaults to auto-detection. + height: 200, // Height of the graph div, excluding titles and legends. + min: "auto", // Minimum Y-axis value, defaults to lowest data value. + max: undefined, // Maximum Y-axis value, defaults to highest data value. + renderer: 'line', // Type of graphs, options are 'line' and 'area'. + name: null, // What to call plots, defaults to trying to do + // something reasonable. + // If a string, it'll use that. [[ label ]] will be substituted. + // If a function it'll be called with a map of keys to values, + // and should return the name to use. + // Can be a list of strings/functions, each element + // will be applied to the plots from the corresponding + // element of the expr list. + xTitle: "Time", // The title of the x axis. + yUnits: "", // The units of the y axis. + yTitle: "", // The title of the y axis. + // Number formatter for y axis. + yAxisFormatter: PromConsole.NumberFormatter.humanize, + // Number of ticks (horizontal lines) for the y axis + yAxisTicks: null, + // Number formatter for y values hover detail. + yHoverFormatter: PromConsole.NumberFormatter.humanizeExact, + // Color scheme to be used by the plots. Can be either a list of hex color + // codes or one of the color scheme names supported by Rickshaw. + colorScheme: null, +}; + +PromConsole.Graph = function(params) { + for (var k in PromConsole.graphDefaults) { + if (!(k in params)) { + params[k] = PromConsole.graphDefaults[k]; + } + } + if (typeof params.expr == "string") { + params.expr = [params.expr]; + } + if (typeof params.name == "string" || typeof params.name == "function") { + var name = []; + for (var i = 0; i < params.expr.length; i++) { + name.push(params.name); + } + params.name = name; + } + + this.params = params; + this.rendered_data = null; + // Keep a reference so that further updates (e.g. annotations) can be made + // by the user in their templates. + this.rickshawGraph = null; + PromConsole._graph_registry.push(this); + + /* + * Table layout: + * | yTitle | Graph | + * | | xTitle | + * | /graph | Legend | + */ + var table = document.createElement("table"); + table.className = "prom_graph_table"; + params.node.appendChild(table); + var tr = document.createElement("tr"); + table.appendChild(tr); + var yTitleTd = document.createElement("td"); + tr.appendChild(yTitleTd); + var yTitleDiv = document.createElement("td"); + yTitleTd.appendChild(yTitleDiv); + yTitleDiv.className = "prom_graph_ytitle"; + yTitleDiv.textContent = params.yTitle + (params.yUnits ? " (" + params.yUnits.trim() + ")" : ""); + + this.graphTd = document.createElement("td"); + tr.appendChild(this.graphTd); + this.graphTd.className = "rickshaw_graph"; + this.graphTd.width = params.width; + this.graphTd.height = params.height; + + tr = document.createElement("tr"); + table.appendChild(tr); + tr.appendChild(document.createElement("td")); + var xTitleTd = document.createElement("td"); + tr.appendChild(xTitleTd); + xTitleTd.className = "prom_graph_xtitle"; + xTitleTd.textContent = params.xTitle; + + tr = document.createElement("tr"); + table.appendChild(tr); + var graphLinkTd = document.createElement("td"); + tr.appendChild(graphLinkTd); + var graphLinkA = document.createElement("a"); + graphLinkTd.appendChild(graphLinkA); + graphLinkA.className = "prom_graph_link"; + graphLinkA.textContent = "+"; + graphLinkA.href = PromConsole._graphsToSlashGraphURL(params.expr); + var legendTd = document.createElement("td"); + tr.appendChild(legendTd); + this.legendDiv = document.createElement("div"); + legendTd.width = params.width; + legendTd.appendChild(this.legendDiv); + + window.addEventListener('resize', function() { + if (this.rendered_data !== null) { + this._render(this.rendered_data); + } + }.bind(this)); + + this.dispatch(); +}; + +PromConsole.Graph.prototype._parseValue = function(value) { + var val = parseFloat(value); + if (isNaN(val)) { + // "+Inf", "-Inf", "+Inf" will be parsed into NaN by parseFloat(). The + // can't be graphed, so show them as gaps (null). + return null; + } + return val; +}; + +PromConsole.Graph.prototype._escapeHTML = function(string) { + var entityMap = { + "&": "&", + "<": "<", + ">": ">", + '"': '"', + "'": ''', + "/": '/' + }; + + return string.replace(/[&<>"'\/]/g, function(s) { + return entityMap[s]; + }); +}; + +PromConsole.Graph.prototype._render = function(data) { + var self = this; + var palette = new Rickshaw.Color.Palette({ scheme: this.params.colorScheme }); + var series = []; + + // This will be used on resize. + this.rendered_data = data; + + + var nameFuncs = []; + if (this.params.name === null) { + var chooser = PromConsole._chooseNameFunction(data); + for (var i = 0; i < this.params.expr.length; i++) { + nameFuncs.push(chooser); + } + } else { + for (var i = 0; i < this.params.name.length; i++) { + if (typeof this.params.name[i] == "string") { + nameFuncs.push(function(i, metric) { + return PromConsole._interpolateName(this.params.name[i], metric); + }.bind(this, i)); + } else { + nameFuncs.push(this.params.name[i]); + } + } + } + + // Get the data into the right format. + var seriesLen = 0; + + for (var e = 0; e < data.length; e++) { + for (var i = 0; i < data[e].data.result.length; i++) { + series[seriesLen] = { + data: data[e].data.result[i].values.map(function(s) { return { x: s[0], y: self._parseValue(s[1]) }; }), + color: palette.color(), + name: self._escapeHTML(nameFuncs[e](data[e].data.result[i].metric)), + }; + // Insert nulls for all missing steps. + var newSeries = []; + var pos = 0; + var start = self.params.endTime - self.params.duration; + var step = Math.floor(self.params.duration / this.graphTd.offsetWidth * 1000) / 1000; + for (var t = start; t <= self.params.endTime; t += step) { + // Allow for floating point inaccuracy. + if (series[seriesLen].data.length > pos && series[seriesLen].data[pos].x < t + step / 100) { + newSeries.push(series[seriesLen].data[pos]); + pos++; + } else { + newSeries.push({ x: t, y: null }); + } + } + series[seriesLen].data = newSeries; + seriesLen++; + } + } + this._clearGraph(); + if (!series.length) { + var errorText = document.createElement("div"); + errorText.className = 'prom_graph_error'; + errorText.textContent = 'No timeseries returned'; + this.graphTd.appendChild(errorText); + return; + } + // Render. + var graph = new Rickshaw.Graph({ + interpolation: "linear", + width: this.graphTd.offsetWidth, + height: this.params.height, + element: this.graphTd, + renderer: this.params.renderer, + max: this.params.max, + min: this.params.min, + series: series + }); + var hoverDetail = new Rickshaw.Graph.HoverDetail({ + graph: graph, + onRender: function() { + var xLabel = this.element.getElementsByClassName("x_label")[0]; + var item = this.element.getElementsByClassName("item")[0]; + if (xLabel.offsetWidth + xLabel.offsetLeft + this.element.offsetLeft > graph.element.offsetWidth || + item.offsetWidth + item.offsetLeft + this.element.offsetLeft > graph.element.offsetWidth) { + xLabel.classList.add("prom_graph_hover_flipped"); + item.classList.add("prom_graph_hover_flipped"); + } else { + xLabel.classList.remove("prom_graph_hover_flipped"); + item.classList.remove("prom_graph_hover_flipped"); + } + }, + yFormatter: function(y) { + if (y === null) { + return ""; + } + return this.params.yHoverFormatter(y) + this.params.yUnits; + }.bind(this) + }); + var yAxis = new Rickshaw.Graph.Axis.Y({ + graph: graph, + tickFormat: this.params.yAxisFormatter, + ticks: this.params.yAxisTicks + }); + var xAxis = new Rickshaw.Graph.Axis.Time({ + graph: graph, + }); + var legend = new Rickshaw.Graph.Legend({ + graph: graph, + element: this.legendDiv + }); + xAxis.render(); + yAxis.render(); + graph.render(); + + this.rickshawGraph = graph; +}; + +PromConsole.Graph.prototype._clearGraph = function() { + while (this.graphTd.lastChild) { + this.graphTd.removeChild(this.graphTd.lastChild); + } + while (this.legendDiv.lastChild) { + this.legendDiv.removeChild(this.legendDiv.lastChild); + } + this.rickshawGraph = null; +}; + +PromConsole.Graph.prototype._xhrs = []; + +PromConsole.Graph.prototype.buildQueryUrl = function(expr) { + var p = this.params; + return PATH_PREFIX + "/api/v1/query_range?query=" + + encodeURIComponent(expr) + + "&step=" + Math.floor(p.duration / this.graphTd.offsetWidth * 1000) / 1000 + + "&start=" + (p.endTime - p.duration) + "&end=" + p.endTime; +}; + +PromConsole.Graph.prototype.dispatch = function() { + for (var j = 0; j < this._xhrs.length; j++) { + this._xhrs[j].abort(); + } + var all_data = new Array(this.params.expr.length); + this._xhrs = new Array(this.params.expr.length); + var pending_requests = this.params.expr.length; + for (var i = 0; i < this.params.expr.length; ++i) { + var endTime = this.params.endTime; + var url = this.buildQueryUrl(this.params.expr[i]); + var xhr = new XMLHttpRequest(); + xhr.open('get', url, true); + xhr.responseType = 'json'; + xhr.onerror = function(xhr, i) { + this._clearGraph(); + var errorText = document.createElement("div"); + errorText.className = 'prom_graph_error'; + errorText.textContent = 'Error loading data'; + this.graphTd.appendChild(errorText); + console.log('Error loading data for ' + this.params.expr[i]); + pending_requests = 0; + // onabort gets any aborts. + for (var j = 0; j < pending_requests; j++) { + this._xhrs[j].abort(); + } + }.bind(this, xhr, i); + xhr.onload = function(xhr, i) { + if (pending_requests === 0) { + // Got an error before this success. + return; + } + var data = xhr.response; + if (typeof data !== "object") { + data = JSON.parse(xhr.responseText); + } + pending_requests -= 1; + all_data[i] = data; + if (pending_requests === 0) { + this._xhrs = []; + this._render(all_data); + } + }.bind(this, xhr, i); + xhr.send(); + this._xhrs[i] = xhr; + } + + var loadingImg = document.createElement("img"); + loadingImg.src = PATH_PREFIX + '/user/ajax-loader.gif'; + loadingImg.alt = 'Loading...'; + loadingImg.className = 'prom_graph_loading'; + this.graphTd.appendChild(loadingImg); +}; + +// Substitute the value of 'label' for [[ label ]]. +PromConsole._interpolateName = function(name, metric) { + var re = /(.*?)\[\[\s*(\w+)+\s*\]\](.*?)/g; + var result = ''; + while (match = re.exec(name)) { + result = result + match[1] + metric[match[2]] + match[3]; + } + if (!result) { + return name; + } + return result; +}; + +// Given the data returned by the API, return an appropriate function +// to return plot names. +PromConsole._chooseNameFunction = function(data) { + // By default, use the full metric name. + var nameFunc = function(metric) { + var name = metric.__name__ + "{"; + for (var label in metric) { + if (label.substring(0, 2) == "__") { + continue; + } + name += label + "='" + metric[label] + "',"; + } + return name + "}"; + }; + + // If only one label varies, use that value. + var labelValues = {}; + for (var e = 0; e < data.length; e++) { + for (var i = 0; i < data[e].data.result.length; i++) { + for (var label in data[e].data.result[i].metric) { + if (!(label in labelValues)) { + labelValues[label] = {}; + } + labelValues[label][data[e].data.result[i].metric[label]] = 1; + } + } + } + + var multiValueLabels = []; + for (var label in labelValues) { + if (Object.keys(labelValues[label]).length > 1) { + multiValueLabels.push(label); + } + } + if (multiValueLabels.length == 1) { + nameFunc = function(metric) { + return metric[multiValueLabels[0]]; + }; + } + return nameFunc; +}; + +// Given a list of expressions, produce the /graph url for them. +PromConsole._graphsToSlashGraphURL = function(exprs) { + var data = []; + for (var i = 0; i < exprs.length; ++i) { + data.push({ 'expr': exprs[i], 'tab': 0 }); + } + return PATH_PREFIX + '/graph#' + encodeURIComponent(JSON.stringify(data)); + +}; diff --git a/bundles/prometheus/files/console_libraries/menu.lib b/bundles/prometheus/files/console_libraries/menu.lib index dc87f9f..d9e4baf 100644 --- a/bundles/prometheus/files/console_libraries/menu.lib +++ b/bundles/prometheus/files/console_libraries/menu.lib @@ -1,64 +1,51 @@ -{{/* vim: set ft=html: */}} +{{/* vim: set ft=gotmpl: */}} {{/* Navbar, should be passed . */}} {{ define "navbar" }} -<nav class="navbar fixed-top navbar-expand-sm navbar-dark bg-dark"> - <div class="container-fluid"> - <!-- Brand and toggle get grouped for better mobile display --> - <div class="navbar-header"> - <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" - aria-expanded="false" aria-controls="navbar-nav" aria-label="toggle navigation"> - <span class="navbar-toggler-icon"></span> - </button> - <a class="navbar-brand" href="{{ pathPrefix }}/">Prometheus</a> - </div> - - <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> - <ul class="nav navbar-nav"> - <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/alerts">Alerts</a></li> - </ul> - </div> +<nav class="navbar sticky-top navbar-expand-sm navbar-dark bg-dark"> + <!-- Brand and toggle get grouped for better mobile display --> + <div class="navbar-header"> + <button type="button" class="navbar-toggler" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1" aria-expanded="false" aria-controls="navbar-nav" aria-label="toggle navigation"> + <span class="navbar-toggler-icon"></span> + </button> + <a class="navbar-brand" href="{{ pathPrefix }}/">Prometheus</a> + </div> + <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1"> + <ul class="nav navbar-nav"> + <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/alerts">Alerts</a></li> + <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/graph">Graph</a></li> + <li class="nav-item"><a class="nav-link" href="{{ pathPrefix }}/targets">Targets</a></li> + </ul> </div> </nav> {{ end }} {{/* LHS menu, should be passed . */}} {{ define "menu" }} -<div class="prom_lhs_menu row"> - <nav class="col-md-2 md-block bg-dark sidebar prom_lhs_menu_nav"> - <div class="sidebar-sticky"> - <ul class="nav flex-column"> - - {{ template "_menuItem" (args . "index.html" "Overview") }} +<nav aria-label="breadcrumb"> + <ol class="breadcrumb rounded-0"> + {{ template "_menuItem" (args . "index.html" "overview") }} - {{ $ctx := . }} + {{ $ctx := . }} - {{ range query "up{job='node-exporter'}" | sortByLabel "instance" }} - {{ $instance := .Labels.instance }} - <ul> - {{ template "_menuItem" (args $ctx (printf "node-overview.html?instance=%v" $instance) $instance) }} - <ul> - {{ template "_menuItem" (args $ctx (printf "node-cpu.html?instance=%v" $instance) "CPU") }} - {{ template "_menuItem" (args $ctx (printf "node-disk.html?instance=%v" $instance) "Disk") }} - </ul> - </ul> - {{ end }} - - {{/* XXX: there is only one prometheus instance in my setup */}} - {{ with query "up{job='prometheus'}" | first }} - {{ template "_menuItem" (args $ctx (printf "prometheus-overview.html?instance=%v" .Labels.instance) "Prometheus") }} - {{ end }} - - </ul> - </div> - </nav> -</div> + {{ template "_menuItem" (args $ctx "node.html?instance=inazuma" "inazuma") }} + {{ template "_menuItem" (args $ctx "node.html?instance=shimakaze" "shimakaze") }} + {{ template "_menuItem" (args $ctx "prom.html?instance=inazuma" "prometheus") }} + </ol> +</nav> {{ end }} {{/* Helper, pass (args . path name) */}} {{ define "_menuItem" }} {{ $pagelink := .arg0.Path }} {{ if .arg0.Params.instance }}{{ $pagelink = (printf "%v?instance=%v" .arg0.Path .arg0.Params.instance) }}{{ end }} -<li class="{{ if eq $pagelink .arg1 }}prom_lhs_menu_selected {{ end }}nav-item"><a class="nav-link" - href="{{ .arg1 }}">{{ .arg2 }}</a></li> +<li class="breadcrumb-item{{ if eq $pagelink .arg1 }} active{{ end }} text-lowercase"> + <a href="{{ .arg1 }}"> + {{ if eq .arg2 "overview" }} + <i class="bi bi-grid-3x3-gap-fill" title="{{ .arg2 }}"></i> + {{ else }} + {{ .arg2 }} + {{ end }} + </a> +</li> {{ end }} diff --git a/bundles/prometheus/files/console_libraries/prom.lib b/bundles/prometheus/files/console_libraries/prom.lib index f934d43..c00bb6b 100644 --- a/bundles/prometheus/files/console_libraries/prom.lib +++ b/bundles/prometheus/files/console_libraries/prom.lib @@ -1,35 +1,76 @@ -{{/* vim: set ft=html: */}} +{{/* vim: set ft=gotmpl: */}} {{/* Load Prometheus console library JS/CSS. Should go in head */}} {{ define "prom_console_head" }} -<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.css"> -<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/css/bootstrap.min.css"> -<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/css/prom_console.css"> -<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/classic/static/vendor/bootstrap4-glyphicons/css/bootstrap-glyphicons.min.css"> -<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.v3.js"></script> -<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/vendor/d3.layout.min.js"></script> -<script src="{{ pathPrefix }}/classic/static/vendor/rickshaw/rickshaw.min.js"></script> -<script src="{{ pathPrefix }}/classic/static/vendor/js/jquery-3.5.1.min.js"></script> -<script src="{{ pathPrefix }}/classic/static/vendor/js/popper.min.js"></script> -<script src="{{ pathPrefix }}/classic/static/vendor/bootstrap-4.5.2/js/bootstrap.min.js"></script> - +<meta name="viewport" content="width=device-width,initial-scale=1.0"> +<meta name="title" content="Prometheus — Console"> +<title>Prometheus — Console</title> +<!-- rickshaw --> +<script src="https://unpkg.com/rickshaw@1.7.1/vendor/d3.v3.js"></script> +<script src="https://unpkg.com/rickshaw@1.7.1/vendor/d3.layout.min.js"></script> +<link type="text/css" rel="stylesheet" href="https://unpkg.com/rickshaw@1.7.1/rickshaw.min.css"> +<script src="https://unpkg.com/rickshaw@1.7.1/rickshaw.min.js"></script> +<!-- bootstrap --> +<script src="https://unpkg.com/jquery@3.5.1/dist/jquery.min.js"></script> +<link type="text/css" rel="stylesheet" href="https://unpkg.com/bootstrap@4.5.2/dist/css/bootstrap.min.css"> +<script src="https://unpkg.com/bootstrap@4.5.2/dist/js/bootstrap.bundle.min.js"></script> +<!-- icons --> +<link type="text/css" rel="stylesheet" href="https://unpkg.com/bootstrap-icons@1.13.1/font/bootstrap-icons.min.css"> +<!-- user assets --> +<link type="text/css" rel="stylesheet" href="{{ pathPrefix }}/user/prom_console.css"> +<script src="{{ pathPrefix }}/user/prom_console.js"></script> +<style type="text/css"> + body { + background-color: #191d21 !important; + color: #fff !important; + } + .navbar { + margin-bottom: unset !important; + padding-left: 15px !important; + padding-right: 15px !important; + } + .prom_console_content { + padding-top: unset !important; + margin-bottom: unset !important; + } + .prom_content_div { + width: unset !important; + display: grid; + grid-template-columns: 1fr 1fr; + gap: 15px; + } + @media (max-width: 768px) { + .prom_content_div { + grid-template-columns: 1fr; + } + } + .prom_content_div > div { + padding: 10px; + border: 1px solid var(--gray); + } + .prom_content_div * { + background-color: var(--light) !important; + color: var(--dark) !important; + } + .prom_graph_table { + overflow: hidden; + } +</style> <script> -var PATH_PREFIX = "{{ pathPrefix }}"; + var PATH_PREFIX = "{{ pathPrefix }}"; </script> -<script src="{{ pathPrefix }}/classic/static/js/prom_console.js"></script> {{ end }} {{/* Top of all pages. */}} {{ define "head" -}} -<!doctype html> +<!DOCTYPE html> <html lang="en"> -<head> -{{ template "prom_console_head" }} -</head> -<body> -{{ template "navbar" . }} - -{{ template "menu" . }} + <head> + {{ template "prom_console_head" }} + </head> + <body> + {{ template "navbar" . }} + {{ template "menu" . }} {{ end }} {{ define "__prom_query_drilldown_noop" }}{{ . }}{{ end }} @@ -49,91 +90,64 @@ renderTemplate is the name of the template to use to render the value. */}} {{ define "prom_query_drilldown" }} {{ $expr := .arg0 }}{{ $suffix := (or .arg1 "") }}{{ $renderTemplate := (or .arg2 "__prom_query_drilldown_noop") }} -<a class="prom_query_drilldown" href="{{ pathPrefix }}{{ graphLink $expr }}">{{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }}</a> +<a class="prom_query_drilldown" href="{{ pathPrefix }}{{ graphLink $expr }}"> {{ with query $expr }}{{tmpl $renderTemplate ( . | first | value )}}{{ $suffix }}{{ else }}-{{ end }}</a> {{ end }} {{ define "prom_path" }}/consoles/{{ .Path }}?{{ range $param, $value := .Params }}{{ $param }}={{ $value }}&{{ end }}{{ end }}" -{{ define "prom_right_table_head" }} -<div class="prom_console_rhs"> -<table class="table table-bordered table-hover table-sm"> -{{ end }} -{{ define "prom_right_table_tail" }} -</table> -</div> -{{ end }} - -{{/* RHS table head, pass job name. Should be used after prom_right_table_head. */}} -{{ define "prom_right_table_job_head" }} -<tr> - <th>{{ . }}</th> - <th>{{ template "prom_query_drilldown" (args (printf "sum(up{job='%s'})" .)) }} / {{ template "prom_query_drilldown" (args (printf "count(up{job='%s'})" .)) }}</th> -</tr> -<tr> - <td>CPU</td> - <td>{{ template "prom_query_drilldown" (args (printf "avg by(job)(irate(process_cpu_seconds_total{job='%s'}[5m]))" .) "s/s" "humanizeNoSmallPrefix") }}</td> -</tr> -<tr> - <td>Memory</td> - <td>{{ template "prom_query_drilldown" (args (printf "avg by(job)(process_resident_memory_bytes{job='%s'})" .) "B" "humanize1024") }}</td> -</tr> -{{ end }} - - {{ define "prom_content_head" }} -<div class="prom_console_content"> - <div class="container-fluid"> -{{ template "prom_graph_timecontrol" . }} + <div class="prom_console_content"> + {{ template "prom_graph_timecontrol" . }} {{ end }} {{ define "prom_content_tail" }} - </div> -</div> + </div> {{ end }} {{ define "prom_graph_timecontrol" }} -<div class="prom_graph_timecontrol"> - <div class="prom_graph_timecontrol_inner"> - <div class="prom_graph_timecontrol_group "> - <button class="btn btn-light pull-left" type="button" id="prom_graph_duration_shrink" title="Shrink the time range."> - <i class="glyphicon glyphicon-minus"></i> - </button><!-- Comments between elements to remove spaces + <div class="prom_graph_timecontrol"> + <div class="prom_graph_timecontrol_inner"> + <div class="prom_graph_timecontrol_group "> + <button class="btn btn-light pull-left" type="button" id="prom_graph_duration_shrink" title="Shrink the time range."> + <i class="bi bi-dash-lg"></i> + </button><!-- Comments between elements to remove spaces --><input class="input pull-left align-middle" size="3" title="Time range of graph" type="text" id="prom_graph_duration"><!-- --><button class="btn btn-light pull-left" type="button" id="prom_graph_duration_grow" title="Grow the time range."> - <i class="glyphicon glyphicon-plus"></i> - </button> - </div> - <div class="prom_graph_timecontrol_group "> - <button class="btn btn-light pull-left" type="button" id="prom_graph_time_back" title="Rewind the end time."> - <i class="glyphicon glyphicon-backward"></i> - </button><!-- + <i class="bi bi-plus-lg"></i> + </button> + </div> + <div class="prom_graph_timecontrol_group "> + <button class="btn btn-light pull-left" type="button" id="prom_graph_time_back" title="Rewind the end time."> + <i class="bi bi-arrow-left"></i> + </button><!-- --><input class="input pull-left align-middle" title="End time of graph" placeholder="Until" type="text" id="prom_graph_time_end" size="16" value=""><!-- --><button class="btn btn-light pull-left" type="button" id="prom_graph_time_forward" title="Advance the end time."> - <i class="glyphicon glyphicon-forward"></i> - </button> - </div> - <div class="prom_graph_timecontrol_group "> - <div class="btn-group dropup prom_graph_timecontrol_refresh pull-left"> - <button type="button" class="btn btn-light pull-left" id="prom_graph_refresh_button" title="Refresh."> - <i class="glyphicon glyphicon-repeat"></i> - <span class="icon-repeat"></span> - (<span id="prom_graph_refresh_button_value">Off</span>) - </button> - <button type="button" class="btn btn-light pull-left dropdown-toggle" data-toggle="dropdown" title="Set autorefresh." aria-haspopup="true" aria-expanded="false"> - <span class="caret"></span> - </button> - <ul class="dropdown-menu" id="prom_graph_refresh_intervals" role="menu"> - </ul> + <i class="bi bi-arrow-right"></i> + </button> + </div> + <div class="prom_graph_timecontrol_group "> + <div class="btn-group dropup prom_graph_timecontrol_refresh pull-left"> + <button type="button" class="btn btn-light pull-left" id="prom_graph_refresh_button" title="Refresh."> + <i class="bi bi-arrow-clockwise"></i> + <span class="icon-repeat"></span> + (<span id="prom_graph_refresh_button_value">Off</span>) + </button> + <button type="button" class="btn btn-light pull-left" data-toggle="dropdown" title="Set autorefresh." aria-haspopup="true" aria-expanded="false"> + <i class="bi bi-caret-up-fill"></i> + </button> + <ul class="dropdown-menu" id="prom_graph_refresh_intervals" role="menu"> + </ul> + </div> + </div> </div> + <script> + new PromConsole.TimeControl(); + </script> </div> - </div> - <script> - new PromConsole.TimeControl(); - </script> -</div> {{ end }} {{/* Bottom of all pages. */}} + {{ define "tail" }} -</body> + </body> </html> {{ end }} diff --git a/bundles/prometheus/files/consoles/node-cpu.html b/bundles/prometheus/files/consoles/cpu.html index 64a2cad..64a2cad 100644 --- a/bundles/prometheus/files/consoles/node-cpu.html +++ b/bundles/prometheus/files/consoles/cpu.html diff --git a/bundles/prometheus/files/consoles/node-disk.html b/bundles/prometheus/files/consoles/disk.html index 52afac7..52afac7 100644 --- a/bundles/prometheus/files/consoles/node-disk.html +++ b/bundles/prometheus/files/consoles/disk.html diff --git a/bundles/prometheus/files/consoles/index.html b/bundles/prometheus/files/consoles/index.html index 29c2884..f0b102d 100644 --- a/bundles/prometheus/files/consoles/index.html +++ b/bundles/prometheus/files/consoles/index.html @@ -1,81 +1,102 @@ -{{/* vim: set ft=html: */}} +{{/* vim: set ft=gotmpl: */}} -{{ define "menu" }} -foobar +{{ define "s6_rc_services_count" }} +{{ $instance := .arg0 }}{{ $bundle := .arg1 }} +{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{instance='%s',bundle='%s',service=~'.*-srv'}" $instance $bundle)) }} {{ end }} {{ template "head" . }} -{{ template "prom_right_table_head" . }} -<tr> - <th colspan="2">S6 - inazuma</th> -</tr> -<tr> - <td>default</td> - <td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='default'}")) }}</td> -</tr> -<tr> - <td>containers</td> - <td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='containers'}")) }}</td> -</tr> -<tr> - <td>www</td> - <td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='www'}")) }}</td> -</tr> -<tr> - <td>services</td> - <td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='inazuma',bundle='services'}")) }}</td> -</tr> -<tr> - <th colspan="2">S6 - shimakaze</th> -</tr> -<tr> - <td>default</td> - <td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='default'}")) }}</td> -</tr> -<tr> - <td>containers</td> - <td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='containers'}")) }}</td> -</tr> -<tr> - <td>www</td> - <td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='www'}")) }}</td> -</tr> -<tr> - <td>services</td> - <td>{{ template "prom_query_drilldown" (args (printf "s6_rc_services_count{job='node-exporter',instance='shimakaze',bundle='services'}")) }}</td> -</tr> -{{ template "prom_right_table_tail" . }} - {{ template "prom_content_head" . }} <div class="prom_content_div"> - <h1>Overview</h1> - <h3>Traefik Requests OK - inazuma</h3> - <div id="traefikGraph"></div> - <script> - new PromConsole.Graph({ - node: document.querySelector("#traefikGraph"), - expr: "irate(traefik_service_request_duration_seconds_count{job='traefik',instance='inazuma',code='200'}[5m])", - name: '[[service]]', - yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yTitle: "Requests", - yUnits: "/s", - }) - </script> - <h3>Jellyfin HTTP Requests - shimakaze</h3> - <div id="jellyfinGraph"></div> - <script> - new PromConsole.Graph({ - node: document.querySelector("#jellyfinGraph"), - expr: "irate(http_request_duration_seconds_count{job='jellyfin',instance='shimakaze'}[5m])", - name: '[[action]]', - yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yTitle: "Requests", - yUnits: "/s", - }) - </script> + <div> + <h2>Overview</h2> + <code>IZ: {{ template "s6_rc_services_count" (args "inazuma" "default") }}</code> + <code>SK: {{ template "s6_rc_services_count" (args "shimakaze" "default") }}</code> + </div> + <div> + <h4>Traefik Latency</h4> + <div id="traefikLatencyGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#traefikLatencyGraph"), + expr: [ + `histogram_quantile(0.99, sum(rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~"https?"}[10m])) by (le, instance))`, + ], + name: '[[instance]]', + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yTitle: "Latency (p99)", + yUnits: "s", + }) + </script> + </div> + <div> + <h4>Node CPU Utilisation</h4> + <div id="cpuUtilGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#cpuUtilGraph"), + expr: [ + `(1 - avg(rate(node_cpu_seconds_total{mode="idle"}[1m]))) * 100.0`, + ], + name: '[[instance]]', + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yUnits: "%", + min: 0, + max: 100, + }) + </script> + </div> + <div> + <h4>Node CPU Saturation</h4> + <div id="cpuSatGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#cpuSatGraph"), + expr: [ + `sum by (instance) (node_load15) / count by (instance) (sum by (instance, cpu) (node_cpu_seconds_total))`, + ], + name: '[[instance]]', + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + }) + </script> + </div> + <div> + <h4>Node Memory Utilisation</h4> + <div id="memUtilGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#memUtilGraph"), + expr: [ + `(1 - sum by (instance) (node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / sum by (instance) (node_memory_MemTotal_bytes)) * 100.0`, + ], + name: '[[instance]]', + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yUnits: "%", + min: 0, + max: 100, + }) + </script> + </div> + <div> + <h4>Node Memory Saturation</h4> + <div id="memSatGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#memSatGraph"), + expr: [ + `1e3 * sum by (instance) (rate(node_vmstat_pgpgin[2m]) + rate(node_vmstat_pgpgout[2m]))`, + ], + name: '[[instance]]', + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + }) + </script> + </div> </div> {{ template "prom_content_tail" . }} diff --git a/bundles/prometheus/files/consoles/node-overview.html b/bundles/prometheus/files/consoles/node-overview.html deleted file mode 100644 index edeca2e..0000000 --- a/bundles/prometheus/files/consoles/node-overview.html +++ /dev/null @@ -1,121 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - <tr><th colspan="2">Overview</th></tr> - <tr> - <td>User CPU</td> - <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td> - </tr> - <tr> - <td>System CPU</td> - <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td> - </tr> - <tr> - <td>Memory Total</td> - <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td> - </tr> - <tr> - <td>Memory Free</td> - <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td> - </tr> -<tr> - <th colspan="2">Network</th> -</tr> -{{ range printf "node_network_receive_bytes_total{job='node-exporter',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }} - <tr> - <td>{{ .Labels.device }} Received</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td> - </tr> - <tr> - <td>{{ .Labels.device }} Transmitted</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td> - </tr> -{{ end }} -<tr> - <th colspan="2">Disks</th> -</tr> -{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }} - <tr> - <td>{{ .Labels.device }} Utilization</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td> - </tr> -{{ end }} -{{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }} - <tr> - <td>{{ .Labels.device }} Throughput</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td> - </tr> -{{ end }} -<tr> - <th colspan="2">Filesystem Fullness</th> -</tr> -{{ define "roughlyNearZero" }} -{{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} -{{ end }} -{{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} - <tr> - <td>{{ .Labels.mountpoint }}</td> - <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td> - </tr> -{{ end }} - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} - <h1>Node Overview - {{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1> - - <h3>CPU Usage</h3> - <div id="cpuGraph"></div> - <script> - new PromConsole.Graph({ - node: document.querySelector("#cpuGraph"), - expr: "sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))", - renderer: 'area', - max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}}, - yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yTitle: 'Cores' - }) - </script> - - <h3>Disk I/O Utilization</h3> - <div id="diskioGraph"></div> - <script> - new PromConsole.Graph({ - node: document.querySelector("#diskioGraph"), - expr: [ - "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',device!~'^(md\\\\d+$|dm-)'}[5m]) * 100", - ], - min: 0, - name: '[[ device ]]', - yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yUnits: "%", - yTitle: 'Disk I/O Utilization' - }) - </script> - - <h3>Memory</h3> - <div id="memoryGraph"></div> - <script> - new PromConsole.Graph({ - node: document.querySelector("#memoryGraph"), - renderer: 'area', - expr: [ - "node_memory_Cached_bytes{job='node-exporter',instance='{{ .Params.instance }}'}", - "node_memory_Buffers_bytes{job='node-exporter',instance='{{ .Params.instance }}'}", - "node_memory_MemTotal_bytes{job='node-exporter',instance='{{ .Params.instance }}'} - node_memory_MemFree_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Buffers_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Cached_bytes{job='node-exporter',instance='{{.Params.instance}}'}", - "node_memory_MemFree{job='node-exporter',instance='{{ .Params.instance }}'}", - ], - name: ["Cached", "Buffers", "Used", "Free"], - min: 0, - yUnits: "B", - yAxisFormatter: PromConsole.NumberFormatter.humanize1024, - yHoverFormatter: PromConsole.NumberFormatter.humanize1024, - yTitle: 'Memory' - }) - </script> - -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/bundles/prometheus/files/consoles/node.html b/bundles/prometheus/files/consoles/node.html new file mode 100644 index 0000000..9ef79ce --- /dev/null +++ b/bundles/prometheus/files/consoles/node.html @@ -0,0 +1,133 @@ +{{/* vim: set ft=gotmpl: */}} + +{{ template "head" . }} + +{{ template "prom_content_head" . }} +<div class="prom_content_div"> + <div style="columns:2;"> + <h1 style="margin-bottom:10px;">{{ reReplaceAll "(.*?://)([^:/]+?)(:\\d+)?/.*" "$2" .Params.instance }}</h1> + <table class="table"> + <thead class="thead-dark"> + <tr><th colspan="2">Overview</th></tr> + </thead> + <tbody> + <tr> + <td>User CPU</td> + <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='user'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td> + </tr> + <tr> + <td>System CPU</td> + <td>{{ template "prom_query_drilldown" (args (printf "sum(irate(node_cpu_seconds_total{job='node-exporter',instance='%s',mode='system'}[5m])) * 100 / count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance .Params.instance) "%" "printf.1f") }}</td> + </tr> + <tr> + <td>Memory Total</td> + <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemTotal_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td> + </tr> + <tr> + <td>Memory Free</td> + <td>{{ template "prom_query_drilldown" (args (printf "node_memory_MemFree_bytes{job='node-exporter',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td> + </tr> + </tbody> + </table> + <table class="table"> + <tr><th colspan="2">Network</th></tr> + {{ range printf "node_network_receive_bytes_total{job='node-exporter',instance='%s',device!='lo'}" .Params.instance | query | sortByLabel "device" }} + <tr> + <td>{{ .Labels.device }} Received</td> + <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_receive_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td> + </tr> + <tr> + <td>{{ .Labels.device }} Transmitted</td> + <td>{{ template "prom_query_drilldown" (args (printf "irate(node_network_transmit_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device) "B/s" "humanize") }}</td> + </tr> + {{ end }} + </table> + <table class="table"> + <tr><th colspan="2">Disks</th></tr> + {{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device!~'^(md\\\\d+$|dm-)'}" .Params.instance | query | sortByLabel "device" }} + <tr> + <td>{{ .Labels.device }} Utilization</td> + <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_io_time_seconds_total{job='node-exporter',instance='%s',device='%s'}[5m]) * 100" .Labels.instance .Labels.device) "%" "printf.1f") }}</td> + </tr> + {{ end }} + {{ range printf "node_disk_io_time_seconds_total{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "device" }} + <tr> + <td>{{ .Labels.device }} Throughput</td> + <td>{{ template "prom_query_drilldown" (args (printf "irate(node_disk_read_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m]) + irate(node_disk_written_bytes_total{job='node-exporter',instance='%s',device='%s'}[5m])" .Labels.instance .Labels.device .Labels.instance .Labels.device) "B/s" "humanize") }}</td> + </tr> + {{ end }} + </table> + <table class="table"> + <tr><th colspan="2">Filesystem Fullness</th></tr> + {{ define "roughlyNearZero" }} + {{ if gt .1 . }}~0{{ else }}{{ printf "%.1f" . }}{{ end }} + {{ end }} + {{ range printf "node_filesystem_size_bytes{job='node-exporter',instance='%s'}" .Params.instance | query | sortByLabel "mountpoint" }} + <tr> + <td>{{ .Labels.mountpoint }}</td> + <td>{{ template "prom_query_drilldown" (args (printf "100 - node_filesystem_avail_bytes{job='node-exporter',instance='%s',mountpoint='%s'} / node_filesystem_size_bytes{job='node-exporter'} * 100" .Labels.instance .Labels.mountpoint) "%" "roughlyNearZero") }}</td> + </tr> + {{ end }} + </table> + </div> + <div> + <h3>CPU Usage</h3> + <div id="cpuGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#cpuGraph"), + expr: [ + `sum by (mode)(irate(node_cpu_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',mode!='idle',mode!='iowait',mode!='steal'}[5m]))`, + ], + renderer: 'area', + max: {{ with printf "count(count by (cpu)(node_cpu_seconds_total{job='node-exporter',instance='%s'}))" .Params.instance | query }}{{ . | first | value }}{{ else}}undefined{{end}}, + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yTitle: 'Cores' + }) + </script> + </div> + <div> + <h3>Disk I/O Utilization</h3> + <div id="diskioGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#diskioGraph"), + expr: [ + `irate(node_disk_io_time_seconds_total{job='node-exporter',instance='{{ .Params.instance }}',device!~'^(md\\\\d+$|dm-)'}[5m]) * 100`, + ], + min: 0, + name: '[[ device ]]', + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yUnits: "%", + yTitle: 'Disk I/O Utilization' + }) + </script> + </div> + <div> + <h3>Memory</h3> + <div id="memoryGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#memoryGraph"), + renderer: 'area', + expr: [ + `node_memory_Cached_bytes{job='node-exporter',instance='{{ .Params.instance }}'}`, + `node_memory_Buffers_bytes{job='node-exporter',instance='{{ .Params.instance }}'}`, + `node_memory_MemTotal_bytes{job='node-exporter',instance='{{ .Params.instance }}'} - node_memory_MemFree_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Buffers_bytes{job='node-exporter',instance='{{.Params.instance}}'} - node_memory_Cached_bytes{job='node-exporter',instance='{{.Params.instance}}'}`, + `node_memory_MemFree{job='node-exporter',instance='{{ .Params.instance }}'}`, + ], + name: ["Cached", "Buffers", "Used", "Free"], + min: 0, + yUnits: "B", + yAxisFormatter: PromConsole.NumberFormatter.humanize1024, + yHoverFormatter: PromConsole.NumberFormatter.humanize1024, + yTitle: 'Memory' + }) + </script> + </div> +</div> +{{ template "prom_content_tail" . }} + +{{ template "tail" . }} diff --git a/bundles/prometheus/files/consoles/prom.html b/bundles/prometheus/files/consoles/prom.html new file mode 100644 index 0000000..4bc5eb7 --- /dev/null +++ b/bundles/prometheus/files/consoles/prom.html @@ -0,0 +1,88 @@ +{{/* vim: set ft=gotmpl: */}} + +{{ template "head" . }} + +{{ template "prom_content_head" . }} +<div class="prom_content_div"> + <div style="columns:2;"> + <h1 style="margin-bottom:10px;">Prometheus</h1> + <table> + <tr><th colspan="2">Overview</th></tr> + <tr> + <td>CPU</td> + <td>{{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }}</td> + </tr> + <tr> + <td>Memory</td> + <td>{{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td> + </tr> + <tr> + <td>Version</td> + <td>{{ with query (printf "prometheus_build_info{job='prometheus',instance='%s'}" .Params.instance) }}{{. | first | label "version"}}{{end}}</td> + </tr> + </table> + <table> + <tr><th colspan="2">Storage</th></tr> + <tr> + <td>Ingested Samples</td> + <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }}</td> + </tr> + <tr> + <td>Head Series</td> + <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_head_series{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td> + </tr> + <tr> + <td>Blocks Loaded</td> + <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_blocks_loaded{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td> + </tr> + </table> + <table> + <tr><th colspan="2">Rules</th></tr> + <tr> + <td>Evaluation Duration</td> + <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_evaluator_duration_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td> + </tr> + <tr> + <td>Notification Latency</td> + <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_notifications_latency_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td> + </tr> + <tr> + <td>Notification Queue</td> + <td>{{ template "prom_query_drilldown" (args (printf "prometheus_notifications_queue_length{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td> + </tr> + </table> + </div> + <div> + <h3>Ingested Samples</h3> + <div id="samplesGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#samplesGraph"), + expr: "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='{{ .Params.instance }}'}[5m])", + name: 'Ingested Samples', + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yTitle: "Samples", + yUnits: "/s", + }) + </script> + </div> + <div> + <h3>HTTP Server</h3> + <div id="serverGraph"></div> + <script> + new PromConsole.Graph({ + node: document.querySelector("#serverGraph"), + expr: "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='{{ .Params.instance }}'}[5m])", + name: '[[handler]]', + yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, + yTitle: "Requests", + yUnits: "/s", + }) + </script> + </div> +</div> +{{ template "prom_content_tail" . }} + +{{ template "tail" }} diff --git a/bundles/prometheus/files/consoles/prometheus-overview.html b/bundles/prometheus/files/consoles/prometheus-overview.html deleted file mode 100644 index 08e027d..0000000 --- a/bundles/prometheus/files/consoles/prometheus-overview.html +++ /dev/null @@ -1,96 +0,0 @@ -{{ template "head" . }} - -{{ template "prom_right_table_head" }} - <tr> - <th colspan="2">Overview</th> - </tr> - <tr> - <td>CPU</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(process_cpu_seconds_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "s/s" "humanizeNoSmallPrefix") }}</td> - </tr> - <tr> - <td>Memory</td> - <td>{{ template "prom_query_drilldown" (args (printf "process_resident_memory_bytes{job='prometheus',instance='%s'}" .Params.instance) "B" "humanize1024") }}</td> - </tr> - <tr> - <td>Version</td> - <td>{{ with query (printf "prometheus_build_info{job='prometheus',instance='%s'}" .Params.instance) }}{{. | first | label "version"}}{{end}}</td> - </tr> - - <tr> - <th colspan="2">Storage</th> - </tr> - <tr> - <td>Ingested Samples</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='%s'}[5m])" .Params.instance) "/s" "humanizeNoSmallPrefix") }}</td> - </tr> - <tr> - <td>Head Series</td> - <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_head_series{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td> - </tr> - <tr> - <td>Blocks Loaded</td> - <td>{{ template "prom_query_drilldown" (args (printf "prometheus_tsdb_blocks_loaded{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td> - </tr> - <tr> - <th colspan="2">Rules</th> - </tr> - <tr> - <td>Evaluation Duration</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_evaluator_duration_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_evaluator_duration_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td> - </tr> - <tr> - <td>Notification Latency</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_notifications_latency_seconds_sum{job='prometheus',instance='%s'}[5m]) / irate(prometheus_notifications_latency_seconds_count{job='prometheus',instance='%s'}[5m])" .Params.instance .Params.instance) "" "humanizeDuration") }}</td> - </tr> - <tr> - <td>Notification Queue</td> - <td>{{ template "prom_query_drilldown" (args (printf "prometheus_notifications_queue_length{job='prometheus',instance='%s'}" .Params.instance) "" "humanize") }}</td> - </tr> - <tr> - <th colspan="2">HTTP Server</th> - </tr> -{{ range printf "prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s'}" .Params.instance | query | sortByLabel "handler" }} - <tr> - <td>{{ .Labels.handler }}</td> - <td>{{ template "prom_query_drilldown" (args (printf "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='%s',handler='%s'}[5m])" .Labels.instance .Labels.handler) "/s" "humanizeNoSmallPrefix") }}</td> - </tr> -{{ end }} - -{{ template "prom_right_table_tail" }} - -{{ template "prom_content_head" . }} - <div class="prom_content_div"> - <h1>Prometheus Overview - {{ .Params.instance }}</h1> - - <h3>Ingested Samples</h3> - <div id="samplesGraph"></div> - <script> - new PromConsole.Graph({ - node: document.querySelector("#samplesGraph"), - expr: "irate(prometheus_tsdb_head_samples_appended_total{job='prometheus',instance='{{ .Params.instance }}'}[5m])", - name: 'Ingested Samples', - yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yTitle: "Samples", - yUnits: "/s", - }) - </script> - - <h3>HTTP Server</h3> - <div id="serverGraph"></div> - <script> - new PromConsole.Graph({ - node: document.querySelector("#serverGraph"), - expr: "irate(prometheus_http_request_duration_seconds_count{job='prometheus',instance='{{ .Params.instance }}'}[5m])", - name: '[[handler]]', - yAxisFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yHoverFormatter: PromConsole.NumberFormatter.humanizeNoSmallPrefix, - yTitle: "Requests", - yUnits: "/s", - }) - </script> - </div> -{{ template "prom_content_tail" . }} - -{{ template "tail" }} diff --git a/bundles/prometheus/files/kube.yaml b/bundles/prometheus/files/kube.yaml index 952f485..7a7f961 100644 --- a/bundles/prometheus/files/kube.yaml +++ b/bundles/prometheus/files/kube.yaml @@ -9,40 +9,97 @@ metadata: traefik.http.routers.prometheus.entrypoints: http,https traefik.http.routers.prometheus.rule: Host(`prom.gzr.im`) traefik.http.routers.prometheus.service: prometheus + traefik.http.routers.prometheus.middlewares: prometheus-consoles-redirect traefik.http.services.prometheus.loadbalancer.server.port: 9090 + traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.regex: ^https://prom.gzr.im/consoles(|/)$ + traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.replacement: https://prom.gzr.im/consoles/index.html + traefik.http.middlewares.prometheus-consoles-redirect.redirectregex.permanent: true spec: - restartPolicy: Never dnsPolicy: Default dnsConfig: searches: - tail2d6ce.ts.net + securityContext: + runAsUser: 0 containers: - name: prom - image: quay.io/prometheus/prometheus:v2.52.0 + image: quay.io/prometheus/prometheus:v3.7.3 + args: + - --config.file=/etc/prometheus/prometheus.yml + - --storage.tsdb.path=/prometheus + - --storage.tsdb.retention.time=30d + - --storage.tsdb.retention.size=6GB + - --web.page-title=Prometheus + - --web.console.templates=/usr/share/prometheus/consoles + - --web.console.libraries=/usr/share/prometheus/console_libraries + - --web.user-assets=/usr/share/prometheus/console_assets volumeMounts: - - mountPath: /etc/prometheus:z + - mountPath: /etc/prometheus name: config readOnly: true - - mountPath: /usr/share/prometheus/consoles:z + - mountPath: /usr/share/prometheus/consoles name: config subPath: consoles readOnly: true - - mountPath: /usr/share/prometheus/console_libraries:z + - mountPath: /usr/share/prometheus/console_libraries name: config subPath: console_libraries readOnly: true + - mountPath: /usr/share/prometheus/console_assets + name: config + subPath: console_assets + readOnly: true + - mountPath: /prometheus + name: prom-data ports: - containerPort: 9090 protocol: TCP + livenessProbe: + exec: + command: + - sh + - -c + - wget -qO - http://localhost:9090/-/healthy | grep -q 'is Healthy' resources: limits: cpu: 100m - memory: 200Mi + memory: 256Mi requests: cpu: 25m - memory: 100Mi + memory: 128Mi + + - name: alert + image: quay.io/prometheus/alertmanager:v0.29.0 + volumeMounts: + - mountPath: /etc/alertmanager + name: config + readOnly: true + - mountPath: /alertmanager + name: alert-data + ports: + - containerPort: 9093 + protocol: TCP + livenessProbe: + exec: + command: + - sh + - -c + - wget -qO - http://localhost:9093/-/healthy | grep -q 'OK' + resources: + limits: + cpu: 50m + memory: 64Mi + volumes: - name: config hostPath: path: /etc/prometheus type: Directory + - name: prom-data + hostPath: + path: /var/lib/prometheus/prom + type: DirectoryOrCreate + - name: alert-data + hostPath: + path: /var/lib/prometheus/alert + type: DirectoryOrCreate diff --git a/bundles/prometheus/files/prometheus.yml b/bundles/prometheus/files/prometheus.yml index 0614c8c..38841e5 100644 --- a/bundles/prometheus/files/prometheus.yml +++ b/bundles/prometheus/files/prometheus.yml @@ -9,16 +9,24 @@ global: # label_name_length_limit: 200 # label_value_length_limit: 200 +# TODO: pending release +# storage: +# tsdb: +# retention: +# time: 30d +# size: 6GB + rule_files: - - rule_*.yml + - /etc/prometheus/rules/*.json + - /etc/prometheus/rules/*.yaml + - /etc/prometheus/rules/*.yml scrape_configs: - - job_name: prometheus static_configs: - - targets: [localhost:9090] - labels: - instance: inazuma + - targets: [localhost:9090] + labels: + instance: inazuma - job_name: service-discovery file_sd_configs: @@ -26,3 +34,9 @@ scrape_configs: - /etc/prometheus/services/*.json - /etc/prometheus/services/*.yaml - /etc/prometheus/services/*.yml + +alerting: + alertmanagers: + - scheme: http + static_configs: + - targets: [localhost:9093] diff --git a/bundles/prometheus/files/rules.yml b/bundles/prometheus/files/rules.yml new file mode 100644 index 0000000..45b130a --- /dev/null +++ b/bundles/prometheus/files/rules.yml @@ -0,0 +1,218 @@ +groups: + - name: Prometheus + rules: + - alert: PrometheusTargetMissing + expr: up == 0 + for: 10m + labels: + severity: critical + annotations: + summary: |- + Prometheus target missing (instance {{ $labels.instance }}) + description: |- + A Prometheus target has disappeared. An exporter might be crashed. + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - name: Host + rules: + - alert: HostOutOfMemory + expr: (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes < .10) + for: 2m + labels: + severity: warning + annotations: + summary: |- + Host out of memory (instance {{ $labels.instance }}) + description: |- + Node memory is filling up (< 10% left) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostMemoryUnderMemoryPressure + expr: (rate(node_vmstat_pgmajfault[5m]) > 1000) + for: 0m + labels: + severity: warning + annotations: + summary: |- + Host memory under memory pressure (instance {{ $labels.instance }}) + description: |- + The node is under heavy memory pressure. High rate of loading memory pages from disk. + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostUnusualDiskReadRate + expr: (rate(node_disk_io_time_seconds_total[5m]) > .80) + for: 0m + labels: + severity: warning + annotations: + summary: |- + Host unusual disk read rate (instance {{ $labels.instance }}) + description: |- + Disk is too busy (IO wait > 80%) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostOutOfDiskSpace + expr: (node_filesystem_avail_bytes{fstype!~"^(fuse.*|-tmpfs|cifs|nfs)",mountpoint!="/media/usb"} / node_filesystem_size_bytes < .10 and on (instance, device, mountpoint) node_filesystem_readonly == 0) + for: 2m + labels: + severity: critical + annotations: + summary: |- + Host out of disk space (instance {{ $labels.instance }}) + description: |- + Disk is almost full (< 10% left) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostHighCpuLoad + expr: 1 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > .80 + for: 10m + labels: + severity: warning + annotations: + summary: |- + Host high CPU load (instance {{ $labels.instance }}) + description: |- + CPU load is > 80% + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostUnusualNetworkThroughputOut + expr: ((rate(node_network_transmit_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80) + for: 0m + labels: + severity: warning + annotations: + summary: |- + Host unusual outbound network traffic (instance {{ $labels.instance }}) + description: |- + Host transmit bandwidth is high (>80%) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostUnusualNetworkThroughputIn + expr: ((rate(node_network_receive_bytes_total[5m]) / on(instance, device) node_network_speed_bytes) > .80) + for: 0m + labels: + severity: warning + annotations: + summary: |- + Host unusual inbound network traffic (instance {{ $labels.instance }}) + description: |- + Host receive bandwidth is high (>80%) + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: HostServiceDown + expr: count(s6_rc_service_up{bundle="default", service=~".*-srv"}) by (instance, service) == 0 + for: 0m + labels: + severity: critical + annotations: + summary: |- + s6-rc service down (instance {{ $labels.instance }}) + description: |- + s6-rc service {{ $labels.service }} on {{ $labels.instance }} is down! + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - name: Traefik + rules: + - alert: TraefikHighHttp4xxErrorRateService + expr: sum(rate(traefik_service_requests_total{service!~".*go-away@.*",code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5 + for: 5m + labels: + severity: critical + annotations: + summary: |- + Traefik high HTTP 4xx error rate service (instance {{ $labels.instance }}) + description: |- + Traefik service 4xx error rate is above 5% + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: TraefikHighHttp5xxErrorRateService + expr: sum(rate(traefik_service_requests_total{code=~"5.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5 + for: 5m + labels: + severity: critical + annotations: + summary: |- + Traefik high HTTP 5xx error rate service (instance {{ $labels.instance }}) + description: |- + Traefik service 5xx error rate is above 5% + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: TraefikLatencyHigh + expr: histogram_quantile(0.99, sum(rate(traefik_entrypoint_request_duration_seconds_bucket{entrypoint=~"https?"}[3m])) by (le, instance)) > 10 + for: 5m + labels: + severity: warning + annotations: + summary: |- + Traefik latency high (instance {{ $labels.instance }}) + description: |- + Traefik p99 latency is higher than 10 seconds + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - name: Podman + rules: + - alert: ContainerHighMemoryUsage + expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) > 80 + for: 2m + labels: + severity: warning + annotations: + summary: |- + Container High Memory usage (instance {{ $labels.instance }}) + description: |- + Container Memory usage is above 80% + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - alert: ContainerLowMemoryUsage + expr: (sum(podman_container_mem_usage_bytes{name!~".*-(infra|-service)$"}) BY (instance, pod_name, name) / sum(podman_container_mem_limit_bytes > 0) BY (instance, pod_name, name) * 100) < 20 + for: 7d + labels: + severity: info + annotations: + summary: |- + Container Low Memory usage (instance {{ $labels.instance }}) + description: |- + Container Memory usage is under 20% for 1 week. Consider reducing the allocated memory. + VALUE = {{ $value }} + LABELS = {{ $labels }} + + - name: BadBots + rules: + - alert: BadBotsFail2BanRate + expr: rate(f2b_jail_banned_current[15m]) > 10 + for: 0m + labels: + severity: warning + annotations: + summary: |- + Increased rate of bans + description: |- + Fail2Ban has banned more than 10 IPs in the last 15 minutes. + VALUE = {{ $value }} + LABELS = {{ $labels }} + + # - alert: BadBotsGoAway... + # expr: rate(f2b_jail_banned_current[15m]) > 10 + # for: 0m + # labels: + # severity: warning + # annotations: + # summary: |- + # Increased rate of bans + # description: |- + # Fail2Ban has banned more than 10 IPs in the last 15 minutes. + # VALUE = {{ $value }} + # LABELS = {{ $labels }} diff --git a/bundles/prometheus/items.py b/bundles/prometheus/items.py index 234aca5..99aecbc 100644 --- a/bundles/prometheus/items.py +++ b/bundles/prometheus/items.py @@ -1,16 +1,34 @@ from sys import exit from os.path import join +actions = { + "reload_prometheus": { + "command": "podman kill -s HUP prometheus-prom", + "triggered": True, + }, + "reload_alertmanager": { + "command": "podman kill -s HUP prometheus-alert", + "triggered": True, + }, +} + files = { "/etc/deployments/prometheus/kube.yaml": { - "content_type": "text", - "source": "kube.yaml", + "content_type": "mako", + "context": node.metadata.get("containers")["prometheus"], "triggers": {"svc_s6rc:prometheus:restart"}, }, "/etc/prometheus/prometheus.yml": { "content_type": "text", - "source": "prometheus.yml", - "triggers": {"svc_s6rc:prometheus:restart"}, + "triggers": {"action:reload_prometheus"}, + }, + "/etc/prometheus/rules/generic.yml": { + "source": "rules.yml", + "triggers": {"action:reload_prometheus"}, + }, + "/etc/prometheus/alertmanager.yml": { + "content_type": "mako", + "triggers": {"action:reload_alertmanager"}, }, } @@ -26,11 +44,19 @@ repo.libs.gen.add_files_recursive( target_root="/etc/prometheus", files_root=join(repo.path, "bundles", "prometheus", "files"), ) +repo.libs.gen.add_files_recursive( + files, + join(repo.path, "bundles", "prometheus", "files", "console_assets"), + target_root="/etc/prometheus", + files_root=join(repo.path, "bundles", "prometheus", "files"), +) directories = { - "/etc/prometheus/services": {}, - "/etc/prometheus/consoles": {}, - "/etc/prometheus/console_libraries": {}, + "/etc/prometheus/services": {"purge": True}, + "/etc/prometheus/rules": {"purge": True}, + "/etc/prometheus/consoles": {"purge": True}, + "/etc/prometheus/console_libraries": {"purge": True}, + "/etc/prometheus/console_assets": {"purge": True}, } # NOTE: services that want to have their metrics scraped need to expose them |
