From f7e8d60800b0ac5923bb8d8b1b75fc904fb3b5ed Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 25 Jun 2019 13:08:32 +0100 Subject: [PATCH 001/126] _data/rules.yml: Added Prometheus error alerts. --- _data/rules.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 364deed..5d64560 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -7,6 +7,42 @@ services: description: Prometheus exporter down query: 'up == 0' severity: warning + - name: Prometheus rule evaluation failures + description: 'Prometheus encountered {{ $value }} rule evaluation failures' + query: 'prometheus_rule_evaluation_failures_total > 0' + severity: error + - name: Prometheus template text expansion failures + description: 'Prometheus encountered {{ $value }} template text expansion failures' + query: 'prometheus_template_text_expansion_failures_total > 0' + severity: error + - name: Prometheus TSDB checkpoint creation failures + description: 'Prometheus encountered {{ $value }} checkpoint creation failures' + query: 'prometheus_tsdb_checkpoint_creations_failed_total > 0' + severity: error + - name: Prometheus TSDB checkpoint deletion failures + description: 'Prometheus encountered {{ $value }} checkpoint deletion failures' + query: 'prometheus_tsdb_checkpoint_deletions_failed_total > 0' + severity: error + - name: Prometheus TSDB compactions failed + description: 'Prometheus encountered {{ $value }} TSDB compactions failures' + query: 'prometheus_tsdb_compactions_failed_total > 0' + severity: error + - name: Prometheus TSDB head truncations failed + description: 'Prometheus encountered {{ $value }} TSDB head truncation failures' + query: 'prometheus_tsdb_head_truncations_failed_total > 0' + severity: error + - name: Prometheus TSDB reload failures + description: 'Prometheus encountered {{ $value }} TSDB reload failures' + query: 'prometheus_tsdb_reloads_failures_total > 0' + severity: error + - name: Prometheus TSDB WAL corruptions + description: 'Prometheus encountered {{ $value }} TSDB WAL corruptions' + query: 'prometheus_tsdb_wal_corruptions_total > 0' + severity: error + - name: Prometheus TSDB WAL truncations failed + description: 'Prometheus encountered {{ $value }} TSDB WAL truncation failures' + query: 'prometheus_tsdb_wal_truncations_failed_total > 0' + severity: error - name: Host exporters: From d789cc314cc1f5f8cf5c19359f347fea2e7c0594 Mon Sep 17 00:00:00 2001 From: Jonas Kongslund Date: Thu, 25 Jul 2019 13:01:47 +0400 Subject: [PATCH 002/126] Add ProbeFailed alert for the Blackbox exporter --- _data/rules.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index eba3892..ec90e0b 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -506,6 +506,10 @@ services: - name: prometheus/blackbox_exporter doc_url: https://github.com/prometheus/blackbox_exporter rules: + - name: Probe failed + description: Probe failed + query: probe_success == 0 + severity: error - name: Status Code description: HTTP status code is not 200-299 query: 'probe_http_status_code <= 199 OR probe_http_status_code >= 300' From e9f247783b11cd0a7b50057c3c60ced04526ae7a Mon Sep 17 00:00:00 2001 From: louis Date: Thu, 8 Aug 2019 14:32:47 +0200 Subject: [PATCH 003/126] add alerts for traefik --- _data/rules.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index ec90e0b..0fa3a06 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -357,6 +357,14 @@ services: - name: Traefik exporters: - rules: + - name: Traefik backend down + description: All Traefik backends are down + query: 'count(traefik_backend_server_up) by (backend) == 0' + severity: critical + - name: Traefik backend errors + description: Traefik backend error rate is above 10% + query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[5m])) by (backend) / sum(rate(traefik_backend_requests_total[5m])) by (backend) > 0.1' + severity: critical - name: PHP-FPM exporters: From 9bd8b3698f3b48d1904f8e2db83cffd2face1aca Mon Sep 17 00:00:00 2001 From: Jonas Kongslund Date: Thu, 22 Aug 2019 13:52:15 +0400 Subject: [PATCH 004/126] Add CollectorError alert for WMI exporter --- _data/rules.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 0fa3a06..fc87e96 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -544,6 +544,10 @@ services: - name: martinlindhe/wmi_exporter doc_url: https://github.com/martinlindhe/wmi_exporter rules: + - name: Collector Error + description: 'Collector {{ $labels.collector }} was not successful' + query: 'wmi_exporter_collector_success == 0' + severity: error - name: Service Status description: Windows Service state is not OK query: 'wmi_service_status{status="ok"} != 1' From 5b7ecd2ce0a7c8aca5f1dcec7f99e29c17e5f043 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2019 11:23:49 +0000 Subject: [PATCH 005/126] Bump nokogiri from 1.8.5 to 1.10.4 Bumps [nokogiri](https://github.com/sparklemotion/nokogiri) from 1.8.5 to 1.10.4. - [Release notes](https://github.com/sparklemotion/nokogiri/releases) - [Changelog](https://github.com/sparklemotion/nokogiri/blob/master/CHANGELOG.md) - [Commits](https://github.com/sparklemotion/nokogiri/compare/v1.8.5...v1.10.4) Signed-off-by: dependabot[bot] --- Gemfile.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 46cf81f..91a191d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -198,15 +198,15 @@ GEM rb-inotify (~> 0.9, >= 0.9.7) ruby_dep (~> 1.2) mercenary (0.3.6) - mini_portile2 (2.3.0) + mini_portile2 (2.4.0) minima (2.5.0) jekyll (~> 3.5) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) minitest (5.11.3) multipart-post (2.0.0) - nokogiri (1.8.5) - mini_portile2 (~> 2.3.0) + nokogiri (1.10.4) + mini_portile2 (~> 2.4.0) octokit (4.13.0) sawyer (~> 0.8.0, >= 0.5.3) pathutil (0.16.1) From 51e7231b3d520264d78bb201135e0f8865ebad7d Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Thu, 29 Aug 2019 19:03:54 +0200 Subject: [PATCH 006/126] fix(blackbox exporter): alert when http >= 400 instead of 300 --- _data/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index fc87e96..bf1203d 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -519,8 +519,8 @@ services: query: probe_success == 0 severity: error - name: Status Code - description: HTTP status code is not 200-299 - query: 'probe_http_status_code <= 199 OR probe_http_status_code >= 300' + description: HTTP status code is not 200-399 + query: 'probe_http_status_code <= 199 OR probe_http_status_code >= 400' severity: error - name: SSL certificate will expire soon description: SSL certificate expires in 30 days From 37ef9a6f5c42e8b27dd9a095e2c2a96e543c5bc5 Mon Sep 17 00:00:00 2001 From: timfeirg Date: Mon, 2 Sep 2019 16:20:08 +0800 Subject: [PATCH 007/126] free memory should include node_memory_Slab_bytes --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index bf1203d..c4c8a1b 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -23,7 +23,7 @@ services: rules: - name: Out of memory description: Node memory is filling up (< 10% left) - query: '(node_memory_MemFree_bytes + node_memory_Cached_bytes + node_memory_Buffers_bytes) / node_memory_MemTotal_bytes * 100 < 10' + query: 'node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10' severity: warning - name: Unusual network throughput in description: Host network interfaces are probably receiving too much data (> 100 MB/s) From f7f94ed81ed389d0e29e8996cebc6b984e920da8 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Fri, 13 Sep 2019 18:08:04 +0200 Subject: [PATCH 008/126] Fixed time interval (10min->10m) --- _data/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index c4c8a1b..826827b 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -281,7 +281,7 @@ services: severity: warning - name: MongoDB cursors timeouts description: Too many cursors are timing out - query: 'increase(mongodb_metrics_cursor_timed_out_total[10min]) > 100' + query: 'increase(mongodb_metrics_cursor_timed_out_total[10m]) > 100' severity: warning - name: MongoDB too many connections description: Too many connections @@ -588,5 +588,5 @@ services: - rules: - name: CoreDNS Panic Count description: Number of CoreDNS panics encountered - query: 'increase(coredns_panic_count_total[10min]) > 0' + query: 'increase(coredns_panic_count_total[10m]) > 0' severity: error From 6d9866cefb9b7ec1316df915818db99ad9416d71 Mon Sep 17 00:00:00 2001 From: Andrey Dudin Date: Wed, 25 Sep 2019 02:42:44 +0300 Subject: [PATCH 009/126] Fix typo in query of PG DeadLocks --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 826827b..ca788e8 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -201,7 +201,7 @@ services: severity: warning - name: Dead locks description: PostgreSQL has dead-locks - query: 'rate(pg_stat_database_deadlocks{pg_stat_database_de}[1m]) > 0' + query: 'rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0' severity: warning - name: Slow queries description: PostgreSQL executes slow queries (> 1min) From 1f4a1f80522e79d3256443a1534f67f06fe32e2c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Wed, 25 Sep 2019 14:23:16 +0200 Subject: [PATCH 010/126] Updating Traefik -> Traefik v1.* --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index ca788e8..11abd12 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -354,7 +354,7 @@ services: doc_url: https://github.com/prometheus/haproxy_exporter rules: - - name: Traefik + - name: Traefik v1.* exporters: - rules: - name: Traefik backend down From e3628c5ba85b238e84125215f76b5aaca117056d Mon Sep 17 00:00:00 2001 From: olivier beyler Date: Wed, 25 Sep 2019 16:04:24 +0200 Subject: [PATCH 011/126] Add OpenEBS and Minio alert Signed-off-by: olivier beyler --- README.md | 2 ++ _data/rules.yml | 19 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/README.md b/README.md index 7fada6b..7096ecc 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ - [Windows](https://awesome-prometheus-alerts.grep.to/rules#windows-server) - [Juniper](https://awesome-prometheus-alerts.grep.to/rules#juniper) - [CoreDNS](https://awesome-prometheus-alerts.grep.to/rules#coredns) +- [OpenEBS](https://awesome-prometheus-alerts.grep.to/rules#openebs) +- [Minio](https://awesome-prometheus-alerts.grep.to/rules#minio) ## Contributing diff --git a/_data/rules.yml b/_data/rules.yml index 11abd12..e018736 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -565,6 +565,25 @@ services: query: '100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{} / 1024 / 1024)) > 80' severity: error + - name: OpenEBS + exporters: + - name: OpenEBS + rules: + - name: Used pool capacity + description: 'OpenEBS Pool use more than 80% of his capacity\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' + query: '(openebs_used_pool_capacity_percent) > 80' + severity: warning + + - name: Minio + exporters: + - name: Minio + rules: + - name: Disk down + description: 'Minio Disk is down\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' + query: 'minio_offline_disks > 0' + severity: error + + - name: Juniper exporters: - name: czerwonk/junos_exporter From 8f6c85774a8b606f0f0a9aa017527f87aeb9128e Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Wed, 25 Sep 2019 16:36:10 +0200 Subject: [PATCH 012/126] Clean data file --- _data/rules.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index e018736..c9b21b8 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -69,8 +69,8 @@ services: description: Swap is filling up (>80%) query: '(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80' severity: warning - - name: SystemD service failed - description: 'Service {{ $labels.name }} failed' + - name: SystemD service crashed + description: 'SystemD service crashed' query: 'node_systemd_unit_state{state="failed"} == 1' severity: warning @@ -567,8 +567,7 @@ services: - name: OpenEBS exporters: - - name: OpenEBS - rules: + - rules: - name: Used pool capacity description: 'OpenEBS Pool use more than 80% of his capacity\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' query: '(openebs_used_pool_capacity_percent) > 80' @@ -576,14 +575,12 @@ services: - name: Minio exporters: - - name: Minio - rules: + - rules: - name: Disk down description: 'Minio Disk is down\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' query: 'minio_offline_disks > 0' severity: error - - name: Juniper exporters: - name: czerwonk/junos_exporter From 3040fe56d5943bbfd75e5c7adc8bfc4172a011f5 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Wed, 25 Sep 2019 16:36:37 +0200 Subject: [PATCH 013/126] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7096ecc..bfa61de 100644 --- a/README.md +++ b/README.md @@ -38,9 +38,9 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ - [Blackbox](https://awesome-prometheus-alerts.grep.to/rules#blackbox) - [Windows](https://awesome-prometheus-alerts.grep.to/rules#windows-server) - [Juniper](https://awesome-prometheus-alerts.grep.to/rules#juniper) -- [CoreDNS](https://awesome-prometheus-alerts.grep.to/rules#coredns) - [OpenEBS](https://awesome-prometheus-alerts.grep.to/rules#openebs) - [Minio](https://awesome-prometheus-alerts.grep.to/rules#minio) +- [CoreDNS](https://awesome-prometheus-alerts.grep.to/rules#coredns) ## Contributing From 0139c3728fdc3ccdbcf2d433e64891f61ab1384c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sat, 26 Oct 2019 16:41:11 +0200 Subject: [PATCH 014/126] feat(ui): adding copy buttons --- CONTRIBUTING.md | 14 +++++++++++++- _layouts/default.html | 5 +++++ assets/css/app.css | 11 +++++++++++ assets/js/app.js | 16 ++++++++++++++++ docker-compose.yml | 11 +++++++++++ rules.md | 16 +++++++++++----- 6 files changed, 67 insertions(+), 6 deletions(-) create mode 100644 assets/css/app.css create mode 100644 assets/js/app.js create mode 100644 docker-compose.yml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a968f03..7b14678 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,7 +10,19 @@ Rules are here: `_data/rules.yml`. ``` gem install bundler bundle install -bundle exec jekyll serve +jekyll serve +``` + +Or with Docker: + +``` +docker run --rm -it -p 4000:4000 -v $(pwd):/srv/jekyll jekyll/jekyll jekyll serve +``` + +Or with Docker-Compose: + +``` +docker-compose up -d ``` ## Guidelines diff --git a/_layouts/default.html b/_layouts/default.html index 8cfbf89..cf68b00 100644 --- a/_layouts/default.html +++ b/_layouts/default.html @@ -18,7 +18,12 @@ + + + + + +
+

⚠️ Disclamer ⚠️

+ +

+ Alert threshold depends on nature of application. +
+ Some query may have arbitrary tolerance threshold. +

+ Building an efficient an battle-tested monitoring platform takes time. 😉 +

+
+ +

0. Prometheus global configuration

+ {% highlight yaml %} # prometheus.yml From de778a9aec36c40ae157c617dadaaaa693d3ca8c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sat, 7 Mar 2020 20:12:03 +0100 Subject: [PATCH 047/126] don't ask french people to write in english without error --- rules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rules.md b/rules.md index 04d6bc8..9d7089a 100644 --- a/rules.md +++ b/rules.md @@ -8,9 +8,9 @@

⚠️ Disclamer ⚠️

- Alert threshold depends on nature of application. + Alert thresholds depend on nature of applications.
- Some query may have arbitrary tolerance threshold. + Some queries may have arbitrary tolerance threshold.

Building an efficient an battle-tested monitoring platform takes time. 😉

From 072a435f326aa7269319472fced9c5ebfc74e8bd Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 14:41:36 +0100 Subject: [PATCH 048/126] Fixing @jpds queries ;) :rocket: --- _data/rules.yml | 47 +++++++++++++++++++++++------------------------ 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index ea9165b..8dfa6f7 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -2,57 +2,57 @@ services: - name: Prometheus internals exporters: - rules: - - name: Prometheus configuration reload + - name: Prometheus configuration reload failure description: Prometheus configuration reload error query: "prometheus_config_last_reload_successful != 1" - severity: error + severity: warning + - name: AlertManager configuration reload failure + description: AlertManager configuration reload error + query: "alertmanager_config_last_reload_successful != 1" + severity: warning - name: Prometheus not connected to alertmanager description: Prometheus cannot connect the alertmanager query: "prometheus_notifications_alertmanagers_discovered < 1" severity: error - - name: AlertManager configuration reload - description: AlertManager configuration reload error - query: "alertmanager_config_last_reload_successful != 1" - severity: error - name: Exporter down description: Prometheus exporter down query: "up == 0" - severity: warning + severity: error - name: Prometheus rule evaluation failures - description: 'Prometheus encountered {{ $value }} rule evaluation failures' - query: 'prometheus_rule_evaluation_failures_total > 0' + description: 'Prometheus encountered {{ $value }} rule evaluation failures. leading to potentially ignored alerts.' + query: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0' severity: error - name: Prometheus template text expansion failures description: 'Prometheus encountered {{ $value }} template text expansion failures' - query: 'prometheus_template_text_expansion_failures_total > 0' + query: 'increase(prometheus_template_text_expansion_failures_total[3m]) > 0' severity: error - name: Prometheus TSDB checkpoint creation failures description: 'Prometheus encountered {{ $value }} checkpoint creation failures' - query: 'prometheus_tsdb_checkpoint_creations_failed_total > 0' + query: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0' severity: error - name: Prometheus TSDB checkpoint deletion failures description: 'Prometheus encountered {{ $value }} checkpoint deletion failures' - query: 'prometheus_tsdb_checkpoint_deletions_failed_total > 0' + query: 'increase(prometheus_tsdb_checkpoint_deletions_failed_total[3m]) > 0' severity: error - name: Prometheus TSDB compactions failed description: 'Prometheus encountered {{ $value }} TSDB compactions failures' - query: 'prometheus_tsdb_compactions_failed_total > 0' + query: 'increase(prometheus_tsdb_compactions_failed_total[3m]) > 0' severity: error - name: Prometheus TSDB head truncations failed description: 'Prometheus encountered {{ $value }} TSDB head truncation failures' - query: 'prometheus_tsdb_head_truncations_failed_total > 0' + query: 'increase(prometheus_tsdb_head_truncations_failed_total[3m]) > 0' severity: error - name: Prometheus TSDB reload failures description: 'Prometheus encountered {{ $value }} TSDB reload failures' - query: 'prometheus_tsdb_reloads_failures_total > 0' + query: 'increase(prometheus_tsdb_reloads_failures_total[3m]) > 0' severity: error - name: Prometheus TSDB WAL corruptions description: 'Prometheus encountered {{ $value }} TSDB WAL corruptions' - query: 'prometheus_tsdb_wal_corruptions_total > 0' + query: 'increase(prometheus_tsdb_wal_corruptions_total[3m]) > 0' severity: error - name: Prometheus TSDB WAL truncations failed description: 'Prometheus encountered {{ $value }} TSDB WAL truncation failures' - query: 'prometheus_tsdb_wal_truncations_failed_total > 0' + query: 'increase(prometheus_tsdb_wal_truncations_failed_total[3m]) > 0' severity: error - name: Host and hardware @@ -127,17 +127,16 @@ services: - name: Node overtemperature alarm description: "Physical node temperature alarm triggered" query: "node_hwmon_temp_alarm == 1" - severity: critical + severity: error - name: RAID array got inactive description: 'RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.' query: 'node_md_state{state="inactive"} > 0' - severity: critical + severity: error - name: RAID disk failure description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap' query: 'node_md_disks{state="fail"} > 0' severity: warning - - name: Docker containers exporters: - name: cAdvisor @@ -426,7 +425,7 @@ services: - name: Cassandra hints count description: Cassandra hints count has changed on {{ $labels.instance }} some nodes may go down query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:totalhints:count"}[1m]) > 3' - severity: critical + severity: error - name: Cassandra compaction task pending description: Many Cassandra compaction tasks are pending. You might need to increase I/O capacity by adding nodes to the cluster. query: 'avg_over_time(cassandra_stats{name="org:apache:cassandra:metrics:compaction:pendingtasks:value"}[30m]) > 100' @@ -442,7 +441,7 @@ services: - name: Cassandra node down description: Cassandra node down query: 'sum(cassandra_stats{name="org:apache:cassandra:net:failuredetector:downendpointcount"}) by (service,group,cluster,env) > 0' - severity: critical + severity: error - name: Cassandra commitlog pending tasks description: Unexpected number of Cassandra commitlog pending tasks query: 'cassandra_stats{name="org:apache:cassandra:metrics:commitlog:pendingtasks:value"} > 15' @@ -466,11 +465,11 @@ services: - name: Cassandra connection timeouts total description: Some connection between nodes are ending in timeout query: 'rate(cassandra_stats{name="org:apache:cassandra:metrics:connection:totaltimeouts:count"}[1m]) > 5' - severity: critical + severity: error - name: Cassandra storage exceptions description: Something is going wrong with cassandra storage query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:exceptions:count"}[1m]) > 1' - severity: critical + severity: error - name: Apache exporters: From 718a039313ca2283c782d8d5c18b306fb03d572d Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 15:08:11 +0100 Subject: [PATCH 049/126] Adding an alert for prometheus internals: rule evaluation slowing down --- _data/rules.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 8dfa6f7..5368371 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -19,13 +19,17 @@ services: query: "up == 0" severity: error - name: Prometheus rule evaluation failures - description: 'Prometheus encountered {{ $value }} rule evaluation failures. leading to potentially ignored alerts.' + description: 'Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.' query: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0' severity: error - name: Prometheus template text expansion failures description: 'Prometheus encountered {{ $value }} template text expansion failures' query: 'increase(prometheus_template_text_expansion_failures_total[3m]) > 0' severity: error + - name: Prometheus rule evaluation slow + description: 'Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query.' + query: 'prometheus_rule_group_last_duration_seconds < prometheus_rule_group_interval_seconds' + severity: error - name: Prometheus TSDB checkpoint creation failures description: 'Prometheus encountered {{ $value }} checkpoint creation failures' query: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0' From 90a9a08b7c40075e4f9bc1173fb0506f64b44f4c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 15:17:55 +0100 Subject: [PATCH 050/126] Improves readme and contributing guidelines --- CONTRIBUTING.md | 21 ++++++++++++--------- README.md | 27 ++++++++++++++++++--------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7b14678..001d352 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,18 @@ Rules are here: `_data/rules.yml`. -## Run localy +### Guidelines + +Please ensure your pull request adheres to the following guidelines: + +- Search previous suggestions before making a new one, as yours may be a duplicate. +- Keep descriptions short and simple, but descriptive. +- Description must be factual (the "what?") and should provide root cause suggestions (the "why?"), for faster resolution. +- Queries must be tested on latest exporter version. + +## Improving Github page + +### Run localy ``` gem install bundler @@ -24,11 +35,3 @@ Or with Docker-Compose: ``` docker-compose up -d ``` - -## Guidelines - -Please ensure your pull request adheres to the following guidelines: - -- Search previous suggestions before making a new one, as yours may be a duplicate. -- Keep descriptions short and simple, but descriptive. -- Queries must be tested on latest exporter version. diff --git a/README.md b/README.md index f1d42ce..4c2eb05 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,18 @@ -# Awesome Prometheus Alerts [![Awesome](https://awesome.re/badge-flat.svg)](https://awesome.re) +# 👋 Awesome Prometheus Alerts [![Awesome](https://awesome.re/badge-flat.svg)](https://awesome.re) > Most alerting rules are common to any Prometheus setup. We need a place to find them all. 🤘 🚨 📊 Collection available here: **[https://awesome-prometheus-alerts.grep.to](https://awesome-prometheus-alerts.grep.to)** -## Contents +## ✨ Contents - [Rules](#rules) +- [Contributing](#-contributing) - [Improvements](#improvements) +- [Help us](#-show-your-support) +- [License](#-license) -## Rules +## 🚨 Rules - [Prometheus internals](https://awesome-prometheus-alerts.grep.to/rules#prometheus-internals) - [Host/Hardware](https://awesome-prometheus-alerts.grep.to/rules#host-and-hardware) @@ -43,20 +46,26 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ - [Minio](https://awesome-prometheus-alerts.grep.to/rules#minio) - [CoreDNS](https://awesome-prometheus-alerts.grep.to/rules#coredns) -## Contributing +## 🤝 Contributing -Contributions for common alerting rules are most welcome! +Contributions from community (you!) are most welcome! -For few months, many exporters have changed metric names, in order to follow community conventions. -Please create an issue or a small pull-request with corrected metric name. 🙏 +There are many ways to contribute: writing code, alerting rules, documentation, reporting issues, discussing better error tracking... [Instructions here](CONTRIBUTING.md) -## Improvements +## 🏋️ Improvements - Create an alert rule builder in Jekyll for custom alerts (severity, thresholds, instances...) +- Add resolution suggestions to rule descriptions, for faster incident resolution ([#85](https://github.com/samber/awesome-prometheus-alerts/issues/85)). -## License +## 💫 Show your support + +Give a ⭐️ if this project helped you! + +[![support us](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/samber) + +## 📝 License [![CC4](https://mirrors.creativecommons.org/presskit/cc.srr.primary.svg)](https://creativecommons.org/licenses/by/4.0/legalcode) From c4d35090eb423613b575ff2500fe96b2f2c4162f Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 15:19:48 +0100 Subject: [PATCH 051/126] Improves readme and contributing guidelines --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 4c2eb05..9901f57 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,9 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ ## ✨ Contents -- [Rules](#rules) +- [Rules](#-rules) - [Contributing](#-contributing) -- [Improvements](#improvements) +- [Improvements](#-improvements) - [Help us](#-show-your-support) - [License](#-license) From 7dbbbb0e09a8340bd41f7a9ae67858522efbab26 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 16:10:33 +0100 Subject: [PATCH 052/126] Doc: organizing lb and reverse proxy --- README.md | 2 +- _data/rules.yml | 46 ++++++++++++++++++++++++++++++---------------- 2 files changed, 31 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 9901f57..d6bbc5a 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ - [Prometheus internals](https://awesome-prometheus-alerts.grep.to/rules#prometheus-internals) - [Host/Hardware](https://awesome-prometheus-alerts.grep.to/rules#host-and-hardware) - [Docker Containers](https://awesome-prometheus-alerts.grep.to/rules#docker-containers) -- [Nginx](https://awesome-prometheus-alerts.grep.to/rules#nginx) - [RabbitMQ](https://awesome-prometheus-alerts.grep.to/rules#rabbitmq) - [MySQL](https://awesome-prometheus-alerts.grep.to/rules#mysql) - [PostgreSQL](https://awesome-prometheus-alerts.grep.to/rules#postgresql) @@ -25,6 +24,7 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ - [MongoDB](https://awesome-prometheus-alerts.grep.to/rules#mongodb) - [Elasticsearch](https://awesome-prometheus-alerts.grep.to/rules#elasticsearch) - [Cassandra](https://awesome-prometheus-alerts.grep.to/rules#cassandra) +- [Nginx](https://awesome-prometheus-alerts.grep.to/rules#nginx) - [Apache](https://awesome-prometheus-alerts.grep.to/rules#apache) - [HaProxy](https://awesome-prometheus-alerts.grep.to/rules#haproxy) - [Traefik](https://awesome-prometheus-alerts.grep.to/rules#traefik) diff --git a/_data/rules.yml b/_data/rules.yml index 5368371..bb4f0ef 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -167,20 +167,6 @@ services: query: "(sum(container_fs_io_current) BY (instance, name) * 100) > 80" severity: warning - - name: Nginx - exporters: - - name: nginx-lua-prometheus - doc_url: https://github.com/knyar/nginx-lua-prometheus - rules: - - name: HTTP errors 4xx - description: Too many HTTP requests with status 4xx (> 5%) - query: 'sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' - severity: error - - name: HTTP errors 5xx - description: Too many HTTP requests with status 5xx (> 5%) - query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' - severity: error - - name: RabbitMQ exporters: - name: kbudde/rabbitmq-exporter @@ -475,13 +461,33 @@ services: query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:exceptions:count"}[1m]) > 1' severity: error + - name: Nginx + exporters: + - name: nginx-lua-prometheus + doc_url: https://github.com/knyar/nginx-lua-prometheus + rules: + - name: HTTP errors 4xx + description: Too many HTTP requests with status 4xx (> 5%) + query: 'sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' + severity: error + - name: HTTP errors 5xx + description: Too many HTTP requests with status 5xx (> 5%) + query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' + severity: error + - name: Apache exporters: - name: Lusitaniae/apache_exporter doc_url: https://github.com/Lusitaniae/apache_exporter rules: - - name: HaProxy + - name: HaProxy v2.* + exporters: + - name: Embedded exporter + doc_url: https://github.com/haproxy/haproxy/tree/master/contrib/prometheus-exporter + rules: + + - name: HaProxy v1.* exporters: - name: prometheus/haproxy_exporter doc_url: https://github.com/prometheus/haproxy_exporter @@ -489,7 +495,9 @@ services: - name: Traefik v1.* exporters: - - rules: + - name: Embedded exporter + doc_url: https://docs.traefik.io/v1.7/configuration/metrics/ + rules: - name: Traefik backend down description: All Traefik backends are down query: "count(traefik_backend_server_up) by (backend) == 0" @@ -499,6 +507,12 @@ services: query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[5m])) by (backend) / sum(rate(traefik_backend_requests_total[5m])) by (backend) > 0.1' severity: error + - name: Traefik v2.* + exporters: + - name: Embedded exporter + doc_url: https://docs.traefik.io/observability/metrics/prometheus/ + rules: + - name: PHP-FPM exporters: - name: bakins/php-fpm-exporter From 953878df031bbe4316ebe245312789c69576c20c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 17:17:06 +0100 Subject: [PATCH 053/126] HAProxy 1.*: adding rules --- _data/rules.yml | 78 +++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index bb4f0ef..5c61cdb 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -466,11 +466,11 @@ services: - name: nginx-lua-prometheus doc_url: https://github.com/knyar/nginx-lua-prometheus rules: - - name: HTTP errors 4xx + - name: Nginx high HTTP 4xx error rate description: Too many HTTP requests with status 4xx (> 5%) query: 'sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' severity: error - - name: HTTP errors 5xx + - name: Nginx high HTTP 5xx error rate description: Too many HTTP requests with status 5xx (> 5%) query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' severity: error @@ -492,6 +492,70 @@ services: - name: prometheus/haproxy_exporter doc_url: https://github.com/prometheus/haproxy_exporter rules: + - name: HAProxy down + description: HAProxy down + query: 'haproxy_up = 0' + severity: error + - name: HAProxy high HTTP 4xx error rate backend + description: Too many HTTP requests with status 4xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} + query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' + severity: error + - name: HAProxy high HTTP 4xx error rate backend + description: Too many HTTP requests with status 5xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} + query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' + severity: error + - name: HAProxy high HTTP 4xx error rate server + description: Too many HTTP requests with status 4xx (> 5%) on server {{ $labels.server }} + query: 'sum by (server) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' + severity: error + - name: HAProxy high HTTP 5xx error rate server + description: Too many HTTP requests with status 5xx (> 5%) on server {{ $labels.server }} + query: 'sum by (server) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' + severity: error + - name: HAProxy backend connection errors + description: Too many connection errors to {{ $labels.fqdn }}/{{ $labels.backend }} backend (> 5%). Request throughput may be to high. + query: 'sum by (backend) rate(haproxy_backend_connection_errors_total[1m]) * 100 > 5' + severity: error + - name: HAProxy server response errors + description: Too many response errors to {{ $labels.server }} server (> 5%). + query: 'sum by (server) rate(haproxy_server_response_errors_total[1m]) * 100 > 5' + severity: error + - name: HAProxy server connection errors + description: Too many connection errors to {{ $labels.server }} server (> 5%). Request throughput may be to high. + query: 'sum by (server) rate(haproxy_server_connection_errors_total[1m]) * 100 > 5' + severity: error + - name: HAProxy backend max active session + description: HAproxy backend {{ $labels.fqdn }}/{{ $labels.backend }} is reaching session limit (> 80%). + query: 'avg_over_time((sum by (backend) (haproxy_server_max_sessions) / sum by (backend) (haproxy_server_limit_sessions)) [2m]) * 100 > 80' + severity: warning + - name: HAProxy pending requests + description: Some HAProxy requests are pending on {{ $labels.fqdn }}/{{ $labels.backend }} backend + query: 'sum by (backend) haproxy_backend_current_queue > 0' + severity: warning + - name: HAProxy HTTP slowing down + description: Average request time is increasing + query: 'avg by (backend) (haproxy_backend_http_total_time_average_seconds) > 2' + severity: warning + - name: HAProxy retry high + description: High rate of retry on {{ $labels.fqdn }}/{{ $labels.backend }} backend + query: 'rate(sum by (backend) (haproxy_backend_retry_warnings_total)) > 10' + severity: warning + - name: HAProxy backend down + description: HAProxy backend is down + query: 'haproxy_backend_up = 0' + severity: error + - name: HAProxy server down + description: HAProxy server is down + query: 'haproxy_server_up = 0' + severity: error + - name: HAProxy frontend security blocked requests + description: HAProxy is blocking requests for security reason + query: 'rate(sum by (frontend) (haproxy_frontend_requests_denied_total)) > 10' + severity: warning + - name: HAProxy server healthcheck failure + description: Some server healthcheck are failing on {{ $labels.server }} + query: 'increase(haproxy_server_check_failures_total) > 0' + severity: warning - name: Traefik v1.* exporters: @@ -502,9 +566,13 @@ services: description: All Traefik backends are down query: "count(traefik_backend_server_up) by (backend) == 0" severity: error - - name: Traefik backend errors - description: Traefik backend error rate is above 10% - query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[5m])) by (backend) / sum(rate(traefik_backend_requests_total[5m])) by (backend) > 0.1' + - name: Traefik high HTTP 4xx error rate backend + description: Traefik backend 4xx error rate is above 5% + query: 'sum(rate(traefik_backend_requests_total{code=~"4.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' + severity: error + - name: Traefik high HTTP 5xx error rate backend + description: Traefik backend 5xx error rate is above 5% + query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' severity: error - name: Traefik v2.* From 5bace111076c85d3e6eea04935a6bde8ec6a1949 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 17:24:39 +0100 Subject: [PATCH 054/126] data: ensure alert name prefix --- _data/rules.yml | 175 ++++++++++++++++++++++++------------------------ 1 file changed, 88 insertions(+), 87 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 5c61cdb..a559fb8 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -64,51 +64,51 @@ services: - name: node-exporter doc_url: https://github.com/prometheus/node_exporter rules: - - name: Out of memory + - name: Host out of memory description: Node memory is filling up (< 10% left) query: "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10" severity: warning - - name: Unusual network throughput in + - name: Host unusual network throughput in description: Host network interfaces are probably receiving too much data (> 100 MB/s) query: "sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100" severity: warning - - name: Unusual network throughput out + - name: Host unusual network throughput out description: Host network interfaces are probably sending too much data (> 100 MB/s) query: "sum by (instance) (irate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100" severity: warning - - name: Unusual disk read rate + - name: Host unusual disk read rate description: Disk is probably reading too much data (> 50 MB/s) query: "sum by (instance) (irate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50" severity: warning - - name: Unusual disk write rate + - name: Host unusual disk write rate description: Disk is probably writing too much data (> 50 MB/s) query: "sum by (instance) (irate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50" severity: warning - - name: Out of disk space + - name: Host out of disk space description: Disk is almost full (< 10% left) query: '(node_filesystem_avail_bytes{mountpoint="/rootfs"} * 100) / node_filesystem_size_bytes{mountpoint="/rootfs"} < 10' severity: warning - - name: Disk will fill in 4 hours + - name: Host disk will fill in 4 hours description: Disk will fill in 4 hours at current write rate query: 'predict_linear(node_filesystem_free_bytes{fstype!~"tmpfs"}[1h], 4 * 3600) < 0' severity: warning - - name: Out of inodes + - name: Host out of inodes description: Disk is almost running out of available inodes (< 10% left) query: 'node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint ="/rootfs"} * 100 < 10' severity: warning - - name: Unusual disk read latency + - name: Host unusual disk read latency description: Disk latency is growing (read operations > 100ms) query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 100" severity: warning - - name: Unusual disk write latency + - name: Host unusual disk write latency description: Disk latency is growing (write operations > 100ms) query: "rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 100" severity: warning - - name: High CPU load + - name: Host high CPU load description: CPU load is > 80% query: '100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80' severity: warning - - name: Context switching + - name: Host context switching description: Context switching is growing on node (> 1000 / s) query: "rate(node_context_switches_total[5m]) > 1000" severity: warning @@ -116,27 +116,27 @@ services: 1000 context switches is an arbitrary number. Alert threshold depends on nature of application. Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58 - - name: Swap is filling up + - name: Host swap is filling up description: Swap is filling up (>80%) query: "(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80" severity: warning - - name: SystemD service crashed + - name: Host SystemD service crashed description: "SystemD service crashed" query: 'node_systemd_unit_state{state="failed"} == 1' severity: warning - - name: Physical component too hot + - name: Host physical component too hot description: "Physical hardware component too hot" query: "node_hwmon_temp_celsius > 75" severity: warning - - name: Node overtemperature alarm + - name: Host node overtemperature alarm description: "Physical node temperature alarm triggered" query: "node_hwmon_temp_alarm == 1" severity: error - - name: RAID array got inactive + - name: Host RAID array got inactive description: 'RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.' query: 'node_md_state{state="inactive"} > 0' severity: error - - name: RAID disk failure + - name: Host RAID disk failure description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap' query: 'node_md_disks{state="fail"} > 0' severity: warning @@ -176,43 +176,43 @@ services: description: RabbitMQ node down query: "rabbitmq_up == 0" severity: error - - name: Cluster down + - name: Rabbitmq cluster down description: Less than 3 nodes running in RabbitMQ cluster query: "sum(rabbitmq_running) < 3" severity: error - - name: Cluster partition + - name: Rabbitmq cluster partition description: Cluster partition query: "rabbitmq_partitions > 0" severity: error - - name: Out of memory + - name: Rabbitmq out of memory description: Memory available for RabbmitMQ is low (< 10%) query: "rabbitmq_node_mem_used / rabbitmq_node_mem_limit * 100 > 90" severity: warning - - name: Too many connections + - name: Rabbitmq too many connections description: RabbitMQ instance has too many connections (> 1000) query: "rabbitmq_connectionsTotal > 1000" severity: warning - - name: Dead letter queue filling up + - name: Rabbitmq dead letter queue filling up description: Dead letter queue is filling up (> 10 msgs) query: 'rabbitmq_queue_messages{queue="my-dead-letter-queue"} > 10' severity: error - - name: Too many messages in queue + - name: Rabbitmq too many messages in queue description: Queue is filling up (> 1000 msgs) query: 'rabbitmq_queue_messages_ready{queue="my-queue"} > 1000' severity: warning - - name: Slow queue consuming + - name: Rabbitmq slow queue consuming description: Queue messages are consumed slowly (> 60s) query: 'time() - rabbitmq_queue_head_message_timestamp{queue="my-queue"} > 60' severity: warning - - name: No consumer + - name: Rabbitmq no consumer description: Queue has no consumer query: "rabbitmq_queue_consumers == 0" severity: error - - name: Too many consumers + - name: Rabbitmq too many consumers description: Queue should have only 1 consumer query: "rabbitmq_queue_consumers > 1" severity: error - - name: Unactive exchange + - name: Rabbitmq unactive exchange description: Exchange receive less than 5 msgs per second query: 'rate(rabbitmq_exchange_messages_published_in_total{exchange="my-exchange"}[1m]) < 5' severity: warning @@ -241,31 +241,31 @@ services: in order to monitor lag on standby servers only. Exporter does not guarantee a NaN value for pg_replication_log on promoted master nodes. See https://github.com/samber/awesome-prometheus-alerts/issues/74 - - name: Table not vaccumed + - name: Postgresql table not vaccumed description: Table has not been vaccum for 24 hours query: "time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24" severity: warning - - name: Table not analyzed + - name: Postgresql table not analyzed description: Table has not been analyzed for 24 hours query: "time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24" severity: warning - - name: Too many connections + - name: Postgresql too many connections description: PostgreSQL instance has too many connections query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) > 100' severity: warning - - name: Not enough connections + - name: Postgresql not enough connections description: PostgreSQL instance should have more connections (> 5) query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) < 5' severity: warning - - name: Dead locks + - name: Postgresql dead locks description: PostgreSQL has dead-locks query: 'rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0' severity: warning - - name: Slow queries + - name: Postgresql slow queries description: PostgreSQL executes slow queries (> 1min) query: 'avg(rate(pg_stat_activity_max_tx_duration{datname!~"template.*"}[1m])) BY (datname) > 60' severity: warning - - name: High rollback rate + - name: Postgresql high rollback rate description: Ratio of transactions being aborted compared to committed is > 2 % query: 'rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02' severity: warning @@ -279,27 +279,27 @@ services: description: Redis instance is down query: "redis_up == 0" severity: error - - name: Missing backup + - name: Redis missing backup description: Redis has not been backuped for 24 hours query: "time() - redis_rdb_last_save_timestamp_seconds > 60 * 60 * 24" severity: error - - name: Out of memory + - name: Redis out of memory description: Redis is running out of memory (> 90%) query: "redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90" severity: warning - - name: Replication broken + - name: Redis replication broken description: Redis instance lost a slave query: "delta(redis_connected_slaves[1m]) < 0" severity: error - - name: Too many connections + - name: Redis too many connections description: Redis instance has too many connections query: "redis_connected_clients > 100" severity: warning - - name: Not enough connections + - name: Redis not enough connections description: Redis instance should have more connections (> 5) query: "redis_connected_clients < 5" severity: warning - - name: Rejected connections + - name: Redis rejected connections description: Some connections to Redis has been rejected query: "increase(redis_rejected_connections_total[1m]) > 0" severity: error @@ -359,47 +359,47 @@ services: - name: justwatchcom/elasticsearch_exporter doc_url: https://github.com/justwatchcom/elasticsearch_exporter rules: - - name: Elastic Heap Usage Too High + - name: Elasticsearch Heap Usage Too High description: "The heap usage is over 90% for 5m" query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90' severity: error - - name: Elastic Heap Usage warning + - name: Elasticsearch Heap Usage warning description: "The heap usage is over 80% for 5m" query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80' severity: warning - - name: Elastic Cluster Red + - name: Elasticsearch Cluster Red description: Elastic Cluster Red status query: 'elasticsearch_cluster_health_status{color="red"} == 1' severity: error - - name: Elastic Cluster Yellow + - name: Elasticsearch Cluster Yellow description: Elastic Cluster Yellow status query: 'elasticsearch_cluster_health_status{color="yellow"} == 1' severity: warning - - name: Number of Elastic Healthy Nodes + - name: Elasticsearch Healthy Nodes description: "Number Healthy Nodes less then number_of_nodes" query: "elasticsearch_cluster_health_number_of_nodes < number_of_nodes" severity: error - - name: Number of Elastic Healthy Data Nodes + - name: Elasticsearch Healthy Data Nodes description: "Number Healthy Data Nodes less then number_of_data_nodes" query: "elasticsearch_cluster_health_number_of_data_nodes < number_of_data_nodes" severity: error - - name: Number of relocation shards + - name: Elasticsearch relocation shards description: "Number of relocation shards for 20 min" query: "elasticsearch_cluster_health_relocating_shards > 0" severity: error - - name: Number of initializing shards + - name: Elasticsearch initializing shards description: "Number of initializing shards for 10 min" query: "elasticsearch_cluster_health_initializing_shards > 0" - severity: error - - name: Number of unassigned shards + severity: warning + - name: Elasticsearch unassigned shards description: "Number of unassigned shards for 2 min" query: "elasticsearch_cluster_health_unassigned_shards > 0" severity: error - - name: Number of pending tasks + - name: Elasticsearch pending tasks description: "Number of pending tasks for 10 min. Cluster works slowly." query: "elasticsearch_cluster_health_number_of_pending_tasks > 0" severity: warning - - name: Elastic no new documents + - name: Elasticsearch no new documents description: No new documents for 10 min! query: 'rate(elasticsearch_indices_docs{es_data_node="true"}[10m]) < 1' severity: warning @@ -409,6 +409,7 @@ services: - name: instaclustr/cassandra-exporter doc_url: https://github.com/instaclustr/cassandra-exporter rules: + - name: criteo/cassandra_exporter doc_url: https://github.com/criteo/cassandra_exporter rules: @@ -632,15 +633,15 @@ services: description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending" query: 'kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1' severity: warning - - name: Volume out of disk space + - name: Kubernetes Volume out of disk space description: Volume is almost full (< 10% left) query: "kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10" severity: warning - - name: Volume full in four days + - name: Kubernetes Volume full in four days description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available." query: "100 * (kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) < 15 and predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0" severity: error - - name: StatefulSet down + - name: Kubernetes StatefulSet down description: A StatefulSet went down query: "(kube_statefulset_status_replicas_ready / kube_statefulset_status_replicas_current) != 1" severity: error @@ -656,11 +657,11 @@ services: - name: prometheus/consul_exporter doc_url: https://github.com/prometheus/consul_exporter rules: - - name: Service healthcheck failed + - name: Consul service healthcheck failed description: "Service: `{{ $labels.service_name }}` Healthcheck: `{{ $labels.service_id }}`" query: "consul_catalog_service_node_healthy == 0" severity: error - - name: Missing Consul master node + - name: Consul missing master node description: Numbers of consul raft peers less then expected query: "consul_raft_peers < number_of_consul_master" severity: error @@ -668,39 +669,39 @@ services: - name: Etcd exporters: - rules: - - name: Insufficient Members + - name: Etcd insufficient Members description: Etcd cluster should have an odd number of members query: "count(etcd_server_id) % 2 == 0" severity: error - - name: No Leader + - name: Etcd no Leader description: Etcd cluster have no leader query: "etcd_server_has_leader == 0" severity: error - - name: High number of leader changes + - name: Etcd high number of leader changes description: Etcd leader changed more than 3 times during last hour query: "increase(etcd_server_leader_changes_seen_total[1h]) > 3" severity: warning - - name: High number of failed GRPC requests + - name: Etcd high number of failed GRPC requests description: More than 1% GRPC request failure detected in Etcd for 5 minutes query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.01' severity: warning - - name: High number of failed GRPC requests + - name: Etcd high number of failed GRPC requests description: More than 5% GRPC request failure detected in Etcd for 5 minutes query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.05' severity: error - - name: GRPC requests slow + - name: Etcd GRPC requests slow description: GRPC requests slowing down, 99th percentil is over 0.15s for 5 minutes query: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15' severity: warning - - name: High number of failed HTTP requests + - name: Etcd high number of failed HTTP requests description: More than 1% HTTP failure detected in Etcd for 5 minutes query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.01" severity: warning - - name: High number of failed HTTP requests + - name: Etcd high number of failed HTTP requests description: More than 5% HTTP failure detected in Etcd for 5 minutes query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.05" severity: error - - name: HTTP requests slow + - name: Etcd HTTP requests slow description: HTTP requests slowing down, 99th percentil is over 0.15s for 5 minutes query: "histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15" severity: warning @@ -708,15 +709,15 @@ services: description: Etcd member communication slowing down, 99th percentil is over 0.15s for 5 minutes query: "histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) > 0.15" severity: warning - - name: High number of failed proposals + - name: Etcd high number of failed proposals description: Etcd server got more than 5 failed proposals past hour query: "increase(etcd_server_proposals_failed_total[1h]) > 5" severity: warning - - name: High fsync durations + - name: Etcd high fsync durations description: Etcd WAL fsync duration increasing, 99th percentil is over 0.5s for 5 minutes query: "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 0.5" severity: warning - - name: High commit durations + - name: Etcd high commit durations description: Etcd commit duration increasing, 99th percentil is over 0.25s for 5 minutes query: "histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) > 0.25" severity: warning @@ -754,31 +755,31 @@ services: - name: prometheus/blackbox_exporter doc_url: https://github.com/prometheus/blackbox_exporter rules: - - name: Probe failed + - name: Blackbox probe failed description: Probe failed query: probe_success == 0 severity: error - - name: Slow probe + - name: Blackbox slow probe description: Blackbox probe took more than 1s to complete query: "avg_over_time(probe_duration_seconds[1m]) > 1" severity: warning - - name: HTTP Status Code + - name: Blackbox HTTP Status Code description: HTTP status code is not 200-399 query: "probe_http_status_code <= 199 OR probe_http_status_code >= 400" severity: error - - name: SSL certificate will expire soon + - name: Blackbox SSL certificate will expire soon description: SSL certificate expires in 30 days query: "probe_ssl_earliest_cert_expiry - time() < 86400 * 30" severity: warning - - name: SSL certificate expired + - name: Blackbox SSL certificate expired description: SSL certificate has expired already query: "probe_ssl_earliest_cert_expiry - time() <= 0" severity: error - - name: HTTP slow requests + - name: Blackbox HTTP slow requests description: HTTP request took more than 1s query: "avg_over_time(probe_http_duration_seconds[1m]) > 1" severity: warning - - name: Slow ping + - name: Blackbox slow ping description: Blackbox ping took more than 1s query: "avg_over_time(probe_icmp_duration_seconds[1m]) > 1" severity: warning @@ -788,23 +789,23 @@ services: - name: martinlindhe/wmi_exporter doc_url: https://github.com/martinlindhe/wmi_exporter rules: - - name: Collector Error + - name: Windows Server collector Error description: "Collector {{ $labels.collector }} was not successful" query: "wmi_exporter_collector_success == 0" severity: error - - name: Service Status + - name: Windows Server service Status description: Windows Service state is not OK query: 'wmi_service_status{status="ok"} != 1' severity: error - - name: CPU Usage + - name: Windows Server CPU Usage description: CPU Usage is more than 80% query: '100 - (avg by (instance) (irate(wmi_cpu_time_total{mode="idle"}[2m])) * 100) > 80' severity: warning - - name: Memory Usage + - name: Windows Server memory Usage description: Memory Usage is more than 90% query: "100*(wmi_os_physical_memory_free_bytes) / wmi_cs_physical_memory_bytes > 90" severity: warning - - name: Disk Space Usage + - name: Windows Server disk Space Usage description: Disk Space on Drive is used more than 80% query: "100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{} / 1024 / 1024)) > 80" severity: error @@ -812,7 +813,7 @@ services: - name: OpenEBS exporters: - rules: - - name: Used pool capacity + - name: OpenEBS used pool capacity description: 'OpenEBS Pool use more than 80% of his capacity\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' query: "(openebs_used_pool_capacity_percent) > 80" severity: warning @@ -820,7 +821,7 @@ services: - name: Minio exporters: - rules: - - name: Disk down + - name: Minio disk down description: 'Minio Disk is down\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' query: "minio_offline_disks > 0" severity: error @@ -830,15 +831,15 @@ services: - name: czerwonk/junos_exporter doc_url: https://github.com/czerwonk/junos_exporter rules: - - name: Switch is down + - name: Juniper switch down description: The switch appears to be down query: junos_up == 0 severity: error - - name: High Bandwith Usage 1GiB + - name: Juniper high Bandwith Usage 1GiB description: Interface is highly saturated for at least 1 min. (> 0.90GiB/s) query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.90" severity: error - - name: High Bandwith Usage 1GiB + - name: Juniper high Bandwith Usage 1GiB description: Interface is getting saturated for at least 1 min. (> 0.80GiB/s) query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.80" severity: warning From b5469f2a591d15745f6ced77315bbb6a8b55d0da Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 17:39:49 +0100 Subject: [PATCH 055/126] Doc: organizing sections --- README.md | 3 ++- _data/rules.yml | 21 ++++++--------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index d6bbc5a..7aed6d5 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ - [HaProxy](https://awesome-prometheus-alerts.grep.to/rules#haproxy) - [Traefik](https://awesome-prometheus-alerts.grep.to/rules#traefik) - [PHP-FPM](https://awesome-prometheus-alerts.grep.to/rules#php-fpm) -- [Java-Client](https://awesome-prometheus-alerts.grep.to/rules#java-client) +- [JVM](https://awesome-prometheus-alerts.grep.to/rules#jvm) - [ZFS](https://awesome-prometheus-alerts.grep.to/rules#zfs) - [Kubernetes](https://awesome-prometheus-alerts.grep.to/rules#kubernetes) - [Nomad](https://awesome-prometheus-alerts.grep.to/rules#nomad) @@ -44,6 +44,7 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ - [Juniper](https://awesome-prometheus-alerts.grep.to/rules#juniper) - [OpenEBS](https://awesome-prometheus-alerts.grep.to/rules#openebs) - [Minio](https://awesome-prometheus-alerts.grep.to/rules#minio) +- [Juniper](https://awesome-prometheus-alerts.grep.to/rules#juniper) - [CoreDNS](https://awesome-prometheus-alerts.grep.to/rules#coredns) ## 🤝 Contributing diff --git a/_data/rules.yml b/_data/rules.yml index a559fb8..346c085 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -482,15 +482,12 @@ services: doc_url: https://github.com/Lusitaniae/apache_exporter rules: - - name: HaProxy v2.* + - name: HaProxy exporters: - - name: Embedded exporter + - name: Embedded exporter (HAProxy >= v2) doc_url: https://github.com/haproxy/haproxy/tree/master/contrib/prometheus-exporter rules: - - - name: HaProxy v1.* - exporters: - - name: prometheus/haproxy_exporter + - name: prometheus/haproxy_exporter (HAProxy < v2) doc_url: https://github.com/prometheus/haproxy_exporter rules: - name: HAProxy down @@ -558,10 +555,10 @@ services: query: 'increase(haproxy_server_check_failures_total) > 0' severity: warning - - name: Traefik v1.* + - name: Traefik exporters: - name: Embedded exporter - doc_url: https://docs.traefik.io/v1.7/configuration/metrics/ + doc_url: https://docs.traefik.io/observability/metrics/prometheus/ rules: - name: Traefik backend down description: All Traefik backends are down @@ -576,19 +573,13 @@ services: query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' severity: error - - name: Traefik v2.* - exporters: - - name: Embedded exporter - doc_url: https://docs.traefik.io/observability/metrics/prometheus/ - rules: - - name: PHP-FPM exporters: - name: bakins/php-fpm-exporter doc_url: https://github.com/bakins/php-fpm-exporter rules: - - name: Java + - name: JVM exporters: - name: java-client doc_url: https://github.com/prometheus/client_java From 542adc3ca7f5ed49baffdbf5df239344d366aa3a Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 18:55:53 +0100 Subject: [PATCH 056/126] Adding minio rules --- _data/rules.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 346c085..2875634 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -812,10 +812,14 @@ services: - name: Minio exporters: - rules: - - name: Minio disk down - description: 'Minio Disk is down\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' + - name: Minio disk offline + description: 'Minio disk is offline' query: "minio_offline_disks > 0" severity: error + - name: Minio storage space exhausted + description: 'Minio storage space is low (< 10 GB)' + query: "minio_disk_storage_free_bytes / 1024 / 1024 / 1024 < 10" + severity: warning - name: Juniper exporters: From 8f515ceae26dac6902e83accc78ff32ab244a0df Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 19:23:28 +0100 Subject: [PATCH 057/126] Improves repo intro --- README.md | 2 +- index.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 7aed6d5..0a7f607 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # 👋 Awesome Prometheus Alerts [![Awesome](https://awesome.re/badge-flat.svg)](https://awesome.re) -> Most alerting rules are common to any Prometheus setup. We need a place to find them all. 🤘 🚨 📊 +> Most alerting rules are common to every Prometheus setup. We need a place to find them all. 🤘 🚨 📊 Collection available here: **[https://awesome-prometheus-alerts.grep.to](https://awesome-prometheus-alerts.grep.to)** diff --git a/index.md b/index.md index 3a12729..79cd5e2 100644 --- a/index.md +++ b/index.md @@ -20,7 +20,7 @@

- Prometheus alerting rules + Out of the box prometheus alerting rules

    From 77eccab0e9826fb7a7ea48d4ebbe6432d960540b Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 20:30:22 +0100 Subject: [PATCH 058/126] some random changes on rules --- _data/rules.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 2875634..ff4f41d 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -6,7 +6,7 @@ services: description: Prometheus configuration reload error query: "prometheus_config_last_reload_successful != 1" severity: warning - - name: AlertManager configuration reload failure + - name: Prometheus AlertManager configuration reload failure description: AlertManager configuration reload error query: "alertmanager_config_last_reload_successful != 1" severity: warning @@ -14,7 +14,7 @@ services: description: Prometheus cannot connect the alertmanager query: "prometheus_notifications_alertmanagers_discovered < 1" severity: error - - name: Exporter down + - name: Prometheus Exporter down description: Prometheus exporter down query: "up == 0" severity: error @@ -228,7 +228,7 @@ services: - name: wrouesnel/postgres_exporter doc_url: https://github.com/wrouesnel/postgres_exporter/ rules: - - name: PostgreSQL down + - name: Postgresql down description: PostgreSQL instance is down query: "pg_up == 0" severity: error @@ -653,8 +653,12 @@ services: query: "consul_catalog_service_node_healthy == 0" severity: error - name: Consul missing master node - description: Numbers of consul raft peers less then expected - query: "consul_raft_peers < number_of_consul_master" + description: Numbers of consul raft peers should be 3, in order to preserve quorum. + query: "consul_raft_peers < 3" + severity: error + - name: Consul agent unhealthy + description: A Consul agent is down + query: 'consul_health_node_status{status="critical"} == 1' severity: error - name: Etcd @@ -724,7 +728,7 @@ services: - name: danielqsj/kafka_exporter doc_url: https://github.com/danielqsj/kafka_exporter rules: - - name: Kafka Topics + - name: Kafka topics replicas description: Kafka topic in-sync partition query: "sum(kafka_topic_partition_in_sync_replica) by (topic) < 3" severity: error From 99e3e64252db04d29f482f3d250dcaad21ee1b97 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 22:21:30 +0100 Subject: [PATCH 059/126] Insert Commit Message Here --- _data/rules.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index ff4f41d..518092d 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -639,8 +639,7 @@ services: - name: Nomad exporters: - - name: samber/prometheus-nomad-exporter - doc_url: https://github.com/samber/prometheus-nomad-exporter + - name: Embedded exporter rules: - name: Consul @@ -807,7 +806,8 @@ services: - name: OpenEBS exporters: - - rules: + - name: Embedded exporter + rules: - name: OpenEBS used pool capacity description: 'OpenEBS Pool use more than 80% of his capacity\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' query: "(openebs_used_pool_capacity_percent) > 80" @@ -815,7 +815,8 @@ services: - name: Minio exporters: - - rules: + - name: Embedded exporter + rules: - name: Minio disk offline description: 'Minio disk is offline' query: "minio_offline_disks > 0" @@ -845,7 +846,8 @@ services: - name: CoreDNS exporters: - - rules: + - name: Embedded exporter + rules: - name: CoreDNS Panic Count description: Number of CoreDNS panics encountered query: "increase(coredns_panic_count_total[10m]) > 0" From 3ad90152936f8dd0578c160b05a0ed32904a350c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 22:53:49 +0100 Subject: [PATCH 060/126] don't ask french people to write in english without error --- rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules.md b/rules.md index 9d7089a..b1057b8 100644 --- a/rules.md +++ b/rules.md @@ -5,7 +5,7 @@
    -

    ⚠️ Disclamer ⚠️

    +

    ⚠️ Disclaimer ⚠️

    Alert thresholds depend on nature of applications. From 6408af5ba30db3579e4c8201a13c40c9f6d1a059 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 23:00:01 +0100 Subject: [PATCH 061/126] don't ask french people to write in english without error --- rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules.md b/rules.md index b1057b8..15544f1 100644 --- a/rules.md +++ b/rules.md @@ -5,7 +5,7 @@

    -

    ⚠️ Disclaimer ⚠️

    +

    ⚠️ Caution ⚠️

    Alert thresholds depend on nature of applications. From 189a3129c3227400ebf8b9ba516dcbb5061852e4 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 8 Mar 2020 23:06:33 +0100 Subject: [PATCH 062/126] moving prom config to alertmanager page --- alertmanager.md | 38 +++++++++++++++++++++++++++++++++++++- rules.md | 40 ++++------------------------------------ 2 files changed, 41 insertions(+), 37 deletions(-) diff --git a/alertmanager.md b/alertmanager.md index d03bc4d..2d8595c 100644 --- a/alertmanager.md +++ b/alertmanager.md @@ -1,3 +1,39 @@ +

    + Prometheus configuration +

    + +{% highlight yaml %} +# prometheus.yml + +global: + scrape_interval: 15s + ... + +rule_files: + - 'alerts/*.yml' + +scrape_configs: + ... + +{% endhighlight %} + +{% highlight yaml %} +# alerts/example-redis.yml + +groups: + +- name: ExampleRedisGroup + rules: + - alert: ExampleRedisDown + expr: redis_up{} == 0 + for: 2m + labels: + severity: error + annotations: + summary: "Redis instance down" + description: "Whatever" + +{% endhighlight %}

    AlertManager configuration @@ -51,7 +87,7 @@ receivers: - name: "sms" webhook_config: - - url: http://a.b.c:8080/send/sms + - url: http://a.b.c.d:8080/send/sms send_resolved: true {% endraw %} diff --git a/rules.md b/rules.md index 15544f1..ef33915 100644 --- a/rules.md +++ b/rules.md @@ -10,46 +10,14 @@

    Alert thresholds depend on nature of applications.
    - Some queries may have arbitrary tolerance threshold. + Some queries in this page may have arbitrary tolerance threshold.

    - Building an efficient an battle-tested monitoring platform takes time. 😉 + Building an efficient and battle-tested monitoring platform takes time. 😉

    -

    0. Prometheus global configuration

    - -{% highlight yaml %} -# prometheus.yml - -global: - scrape_interval: 15s - ... - -rule_files: - - 'alerts/*.yml' - -scrape_configs: - ... - -{% endhighlight %} - -{% highlight yaml %} -# alerts/example-redis.yml - -groups: - -- name: ExampleRedisGroup - rules: - - alert: ExampleRedisDown - expr: redis_up{} == 0 - for: 2m - labels: - severity: error - annotations: - summary: "Redis instance ($instance) down" - description: "Whatever" - -{% endhighlight %} +
    +
      {% for service in site.data.rules.services %} From affacde49b18179abe5cda9131870578b690acc8 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Mon, 9 Mar 2020 00:16:17 +0100 Subject: [PATCH 063/126] adding prometheus internal alerts --- _data/rules.yml | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 518092d..b03d4ed 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -6,6 +6,10 @@ services: description: Prometheus configuration reload error query: "prometheus_config_last_reload_successful != 1" severity: warning + - name: Prometheus too many restarts + description: Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping. + query: "changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2" + severity: warning - name: Prometheus AlertManager configuration reload failure description: AlertManager configuration reload error query: "alertmanager_config_last_reload_successful != 1" @@ -29,7 +33,19 @@ services: - name: Prometheus rule evaluation slow description: 'Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query.' query: 'prometheus_rule_group_last_duration_seconds < prometheus_rule_group_interval_seconds' - severity: error + severity: warning + - name: Prometheus notifications backlog + description: The Prometheus notification queue has not been empty for 10 minutes + query: 'min_over_time(prometheus_notifications_queue_length[10m])' + severity: warning + - name: Prometheus target scraping slow + description: Prometheus is scraping exporters slowly + query: 'prometheus_target_interval_length_seconds{quantile="0.9"} > 60' + severity: warning + - name: Prometheus large scrape + description: Prometheus has many scapres that exceed the sample limit + query: 'increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10' + severity: warning - name: Prometheus TSDB checkpoint creation failures description: 'Prometheus encountered {{ $value }} checkpoint creation failures' query: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0' @@ -68,6 +84,10 @@ services: description: Node memory is filling up (< 10% left) query: "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10" severity: warning + - name: Host memory under memory pressure + description: The node is under heavy memory pressure. High rate of major page faults + query: "rate(node_vmstat_pgmajfault[1m]) > 1000" + severity: warning - name: Host unusual network throughput in description: Host network interfaces are probably receiving too much data (> 100 MB/s) query: "sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100" @@ -140,6 +160,10 @@ services: description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap' query: 'node_md_disks{state="fail"} > 0' severity: warning + - name: Kernel version deviations + description: Different kernel versions are running + query: 'count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1' + severity: warning - name: Docker containers exporters: From 0b89a764eed0e65863cad503a47a7a7695563f0c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Mon, 9 Mar 2020 21:13:55 +0100 Subject: [PATCH 064/126] Adding exporters: sidekiq, pgbouncer and thanos. Adding rules to: prometheus, kubernetes, redis, docker and postgresql. Arranging exporters into categories. Showing number of rules. Thanks to Gitlab for opensourcing alerting rules! --- README.md | 41 +- _data/rules.yml | 1945 ++++++++++++++++++++++++++--------------------- index.md | 26 +- rules.md | 133 ++-- 4 files changed, 1221 insertions(+), 924 deletions(-) diff --git a/README.md b/README.md index 0a7f607..6fa24b3 100644 --- a/README.md +++ b/README.md @@ -14,39 +14,60 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ ## 🚨 Rules -- [Prometheus internals](https://awesome-prometheus-alerts.grep.to/rules#prometheus-internals) +#### Basic resource monitoring + +- [Prometheus self-monitoring](https://awesome-prometheus-alerts.grep.to/rules#prometheus-internals) - [Host/Hardware](https://awesome-prometheus-alerts.grep.to/rules#host-and-hardware) - [Docker Containers](https://awesome-prometheus-alerts.grep.to/rules#docker-containers) -- [RabbitMQ](https://awesome-prometheus-alerts.grep.to/rules#rabbitmq) +- [Blackbox](https://awesome-prometheus-alerts.grep.to/rules#blackbox) +- [Windows](https://awesome-prometheus-alerts.grep.to/rules#windows-server) + +#### Databases and brokers + - [MySQL](https://awesome-prometheus-alerts.grep.to/rules#mysql) - [PostgreSQL](https://awesome-prometheus-alerts.grep.to/rules#postgresql) +- [PGBouncer](https://awesome-prometheus-alerts.grep.to/rules#pgbouncer) - [Redis](https://awesome-prometheus-alerts.grep.to/rules#redis) - [MongoDB](https://awesome-prometheus-alerts.grep.to/rules#mongodb) +- [RabbitMQ](https://awesome-prometheus-alerts.grep.to/rules#rabbitmq) - [Elasticsearch](https://awesome-prometheus-alerts.grep.to/rules#elasticsearch) - [Cassandra](https://awesome-prometheus-alerts.grep.to/rules#cassandra) +- [Zookeeper](https://awesome-prometheus-alerts.grep.to/rules#zookeeper) +- [Kafka](https://awesome-prometheus-alerts.grep.to/rules#kafka) + +#### Reverse proxies and load balancers + - [Nginx](https://awesome-prometheus-alerts.grep.to/rules#nginx) - [Apache](https://awesome-prometheus-alerts.grep.to/rules#apache) - [HaProxy](https://awesome-prometheus-alerts.grep.to/rules#haproxy) - [Traefik](https://awesome-prometheus-alerts.grep.to/rules#traefik) + +#### Runtimes + - [PHP-FPM](https://awesome-prometheus-alerts.grep.to/rules#php-fpm) - [JVM](https://awesome-prometheus-alerts.grep.to/rules#jvm) -- [ZFS](https://awesome-prometheus-alerts.grep.to/rules#zfs) +- [Sidekiq](https://awesome-prometheus-alerts.grep.to/rules#sidekiq) + +#### Orchestrators - [Kubernetes](https://awesome-prometheus-alerts.grep.to/rules#kubernetes) - [Nomad](https://awesome-prometheus-alerts.grep.to/rules#nomad) - [Consul](https://awesome-prometheus-alerts.grep.to/rules#consul) - [Etcd](https://awesome-prometheus-alerts.grep.to/rules#etcd) -- [Zookeeper](https://awesome-prometheus-alerts.grep.to/rules#zookeeper) -- [Kafka](https://awesome-prometheus-alerts.grep.to/rules#kafka) - [Linkerd](https://awesome-prometheus-alerts.grep.to/rules#linkerd) - [Istio](https://awesome-prometheus-alerts.grep.to/rules#istio) -- [Blackbox](https://awesome-prometheus-alerts.grep.to/rules#blackbox) -- [Windows](https://awesome-prometheus-alerts.grep.to/rules#windows-server) -- [Juniper](https://awesome-prometheus-alerts.grep.to/rules#juniper) + +#### Network and storage + +- [ZFS](https://awesome-prometheus-alerts.grep.to/rules#zfs) - [OpenEBS](https://awesome-prometheus-alerts.grep.to/rules#openebs) - [Minio](https://awesome-prometheus-alerts.grep.to/rules#minio) - [Juniper](https://awesome-prometheus-alerts.grep.to/rules#juniper) - [CoreDNS](https://awesome-prometheus-alerts.grep.to/rules#coredns) +#### Other + +- [Thanos](https://awesome-prometheus-alerts.grep.to/rules#thanos) + ## 🤝 Contributing Contributions from community (you!) are most welcome! @@ -66,6 +87,10 @@ Give a ⭐️ if this project helped you! [![support us](https://c5.patreon.com/external/logo/become_a_patron_button.png)](https://www.patreon.com/samber) +## 👏 Thanks + +Gratitude for the Gitlab operation team that provided 50+ rules. \o/ + ## 📝 License [![CC4](https://mirrors.creativecommons.org/presskit/cc.srr.primary.svg)](https://creativecommons.org/licenses/by/4.0/legalcode) diff --git a/_data/rules.yml b/_data/rules.yml index b03d4ed..9ad54d2 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1,878 +1,1129 @@ -services: - - name: Prometheus internals - exporters: - - rules: - - name: Prometheus configuration reload failure - description: Prometheus configuration reload error - query: "prometheus_config_last_reload_successful != 1" - severity: warning - - name: Prometheus too many restarts - description: Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping. - query: "changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2" - severity: warning - - name: Prometheus AlertManager configuration reload failure - description: AlertManager configuration reload error - query: "alertmanager_config_last_reload_successful != 1" - severity: warning - - name: Prometheus not connected to alertmanager - description: Prometheus cannot connect the alertmanager - query: "prometheus_notifications_alertmanagers_discovered < 1" - severity: error - - name: Prometheus Exporter down - description: Prometheus exporter down - query: "up == 0" - severity: error - - name: Prometheus rule evaluation failures - description: 'Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.' - query: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0' - severity: error - - name: Prometheus template text expansion failures - description: 'Prometheus encountered {{ $value }} template text expansion failures' - query: 'increase(prometheus_template_text_expansion_failures_total[3m]) > 0' - severity: error - - name: Prometheus rule evaluation slow - description: 'Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query.' - query: 'prometheus_rule_group_last_duration_seconds < prometheus_rule_group_interval_seconds' - severity: warning - - name: Prometheus notifications backlog - description: The Prometheus notification queue has not been empty for 10 minutes - query: 'min_over_time(prometheus_notifications_queue_length[10m])' - severity: warning - - name: Prometheus target scraping slow - description: Prometheus is scraping exporters slowly - query: 'prometheus_target_interval_length_seconds{quantile="0.9"} > 60' - severity: warning - - name: Prometheus large scrape - description: Prometheus has many scapres that exceed the sample limit - query: 'increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10' - severity: warning - - name: Prometheus TSDB checkpoint creation failures - description: 'Prometheus encountered {{ $value }} checkpoint creation failures' - query: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0' - severity: error - - name: Prometheus TSDB checkpoint deletion failures - description: 'Prometheus encountered {{ $value }} checkpoint deletion failures' - query: 'increase(prometheus_tsdb_checkpoint_deletions_failed_total[3m]) > 0' - severity: error - - name: Prometheus TSDB compactions failed - description: 'Prometheus encountered {{ $value }} TSDB compactions failures' - query: 'increase(prometheus_tsdb_compactions_failed_total[3m]) > 0' - severity: error - - name: Prometheus TSDB head truncations failed - description: 'Prometheus encountered {{ $value }} TSDB head truncation failures' - query: 'increase(prometheus_tsdb_head_truncations_failed_total[3m]) > 0' - severity: error - - name: Prometheus TSDB reload failures - description: 'Prometheus encountered {{ $value }} TSDB reload failures' - query: 'increase(prometheus_tsdb_reloads_failures_total[3m]) > 0' - severity: error - - name: Prometheus TSDB WAL corruptions - description: 'Prometheus encountered {{ $value }} TSDB WAL corruptions' - query: 'increase(prometheus_tsdb_wal_corruptions_total[3m]) > 0' - severity: error - - name: Prometheus TSDB WAL truncations failed - description: 'Prometheus encountered {{ $value }} TSDB WAL truncation failures' - query: 'increase(prometheus_tsdb_wal_truncations_failed_total[3m]) > 0' - severity: error +groups: + - name: Basic resource monitoring + services: + - name: Prometheus self-monitoring + exporters: + - rules: + - name: Prometheus configuration reload failure + description: Prometheus configuration reload error + query: 'prometheus_config_last_reload_successful != 1' + severity: warning + - name: Prometheus too many restarts + description: Prometheus has restarted more than twice in the last 15 minutes. It might be crashlooping. + query: 'changes(process_start_time_seconds{job=~"prometheus|pushgateway|alertmanager"}[15m]) > 2' + severity: warning + - name: Prometheus AlertManager configuration reload failure + description: AlertManager configuration reload error + query: 'alertmanager_config_last_reload_successful != 1' + severity: warning + - name: Prometheus AlertManager E2E dead man snitch + description: Prometheus DeadManSnitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager. + query: 'vector(1)' + severity: error + - name: Prometheus not connected to alertmanager + description: Prometheus cannot connect the alertmanager + query: "prometheus_notifications_alertmanagers_discovered < 1" + severity: error + - name: Prometheus Exporter down + description: Prometheus exporter down + query: "up == 0" + severity: error + - name: Prometheus rule evaluation failures + description: 'Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.' + query: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0' + severity: error + - name: Prometheus template text expansion failures + description: 'Prometheus encountered {{ $value }} template text expansion failures' + query: 'increase(prometheus_template_text_expansion_failures_total[3m]) > 0' + severity: error + - name: Prometheus rule evaluation slow + description: 'Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query.' + query: 'prometheus_rule_group_last_duration_seconds < prometheus_rule_group_interval_seconds' + severity: warning + - name: Prometheus notifications backlog + description: The Prometheus notification queue has not been empty for 10 minutes + query: 'min_over_time(prometheus_notifications_queue_length[10m]) > 0' + severity: warning + - name: Prometheus AlertManager notification failing + description: Alertmanager is failing sending notifications + query: 'rate(alertmanager_notifications_failed_total[1m]) > 0' + severity: error + - name: Prometheus target empty + description: Prometheus has no target in service discovery + query: 'prometheus_sd_discovered_targets == 0' + severity: error + - name: Prometheus target scraping slow + description: Prometheus is scraping exporters slowly + query: 'prometheus_target_interval_length_seconds{quantile="0.9"} > 60' + severity: warning + - name: Prometheus large scrape + description: Prometheus has many scrapes that exceed the sample limit + query: 'increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10' + severity: warning + - name: Prometheus TSDB checkpoint creation failures + description: 'Prometheus encountered {{ $value }} checkpoint creation failures' + query: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0' + severity: error + - name: Prometheus TSDB checkpoint deletion failures + description: 'Prometheus encountered {{ $value }} checkpoint deletion failures' + query: 'increase(prometheus_tsdb_checkpoint_deletions_failed_total[3m]) > 0' + severity: error + - name: Prometheus TSDB compactions failed + description: 'Prometheus encountered {{ $value }} TSDB compactions failures' + query: 'increase(prometheus_tsdb_compactions_failed_total[3m]) > 0' + severity: error + - name: Prometheus TSDB head truncations failed + description: 'Prometheus encountered {{ $value }} TSDB head truncation failures' + query: 'increase(prometheus_tsdb_head_truncations_failed_total[3m]) > 0' + severity: error + - name: Prometheus TSDB reload failures + description: 'Prometheus encountered {{ $value }} TSDB reload failures' + query: 'increase(prometheus_tsdb_reloads_failures_total[3m]) > 0' + severity: error + - name: Prometheus TSDB WAL corruptions + description: 'Prometheus encountered {{ $value }} TSDB WAL corruptions' + query: 'increase(prometheus_tsdb_wal_corruptions_total[3m]) > 0' + severity: error + - name: Prometheus TSDB WAL truncations failed + description: 'Prometheus encountered {{ $value }} TSDB WAL truncation failures' + query: 'increase(prometheus_tsdb_wal_truncations_failed_total[3m]) > 0' + severity: error - - name: Host and hardware - exporters: - - name: node-exporter - doc_url: https://github.com/prometheus/node_exporter - rules: - - name: Host out of memory - description: Node memory is filling up (< 10% left) - query: "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10" - severity: warning - - name: Host memory under memory pressure - description: The node is under heavy memory pressure. High rate of major page faults - query: "rate(node_vmstat_pgmajfault[1m]) > 1000" - severity: warning - - name: Host unusual network throughput in - description: Host network interfaces are probably receiving too much data (> 100 MB/s) - query: "sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100" - severity: warning - - name: Host unusual network throughput out - description: Host network interfaces are probably sending too much data (> 100 MB/s) - query: "sum by (instance) (irate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100" - severity: warning - - name: Host unusual disk read rate - description: Disk is probably reading too much data (> 50 MB/s) - query: "sum by (instance) (irate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50" - severity: warning - - name: Host unusual disk write rate - description: Disk is probably writing too much data (> 50 MB/s) - query: "sum by (instance) (irate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50" - severity: warning - - name: Host out of disk space - description: Disk is almost full (< 10% left) - query: '(node_filesystem_avail_bytes{mountpoint="/rootfs"} * 100) / node_filesystem_size_bytes{mountpoint="/rootfs"} < 10' - severity: warning - - name: Host disk will fill in 4 hours - description: Disk will fill in 4 hours at current write rate - query: 'predict_linear(node_filesystem_free_bytes{fstype!~"tmpfs"}[1h], 4 * 3600) < 0' - severity: warning - - name: Host out of inodes - description: Disk is almost running out of available inodes (< 10% left) - query: 'node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint ="/rootfs"} * 100 < 10' - severity: warning - - name: Host unusual disk read latency - description: Disk latency is growing (read operations > 100ms) - query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 100" - severity: warning - - name: Host unusual disk write latency - description: Disk latency is growing (write operations > 100ms) - query: "rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 100" - severity: warning - - name: Host high CPU load - description: CPU load is > 80% - query: '100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80' - severity: warning - - name: Host context switching - description: Context switching is growing on node (> 1000 / s) - query: "rate(node_context_switches_total[5m]) > 1000" - severity: warning - comments: | - 1000 context switches is an arbitrary number. - Alert threshold depends on nature of application. - Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58 - - name: Host swap is filling up - description: Swap is filling up (>80%) - query: "(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80" - severity: warning - - name: Host SystemD service crashed - description: "SystemD service crashed" - query: 'node_systemd_unit_state{state="failed"} == 1' - severity: warning - - name: Host physical component too hot - description: "Physical hardware component too hot" - query: "node_hwmon_temp_celsius > 75" - severity: warning - - name: Host node overtemperature alarm - description: "Physical node temperature alarm triggered" - query: "node_hwmon_temp_alarm == 1" - severity: error - - name: Host RAID array got inactive - description: 'RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.' - query: 'node_md_state{state="inactive"} > 0' - severity: error - - name: Host RAID disk failure - description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap' - query: 'node_md_disks{state="fail"} > 0' - severity: warning - - name: Kernel version deviations - description: Different kernel versions are running - query: 'count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1' - severity: warning + - name: Host and hardware + exporters: + - name: node-exporter + doc_url: https://github.com/prometheus/node_exporter + rules: + - name: Host out of memory + description: Node memory is filling up (< 10% left) + query: "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10" + severity: warning + - name: Host memory under memory pressure + description: The node is under heavy memory pressure. High rate of major page faults + query: "rate(node_vmstat_pgmajfault[1m]) > 1000" + severity: warning + - name: Host unusual network throughput in + description: Host network interfaces are probably receiving too much data (> 100 MB/s) + query: "sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100" + severity: warning + - name: Host unusual network throughput out + description: Host network interfaces are probably sending too much data (> 100 MB/s) + query: "sum by (instance) (irate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100" + severity: warning + - name: Host unusual disk read rate + description: Disk is probably reading too much data (> 50 MB/s) + query: "sum by (instance) (irate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50" + severity: warning + - name: Host unusual disk write rate + description: Disk is probably writing too much data (> 50 MB/s) + query: "sum by (instance) (irate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50" + severity: warning + - name: Host out of disk space + description: Disk is almost full (< 10% left) + query: '(node_filesystem_avail_bytes{mountpoint="/rootfs"} * 100) / node_filesystem_size_bytes{mountpoint="/rootfs"} < 10' + severity: warning + - name: Host disk will fill in 4 hours + description: Disk will fill in 4 hours at current write rate + query: 'predict_linear(node_filesystem_free_bytes{fstype!~"tmpfs"}[1h], 4 * 3600) < 0' + severity: warning + - name: Host out of inodes + description: Disk is almost running out of available inodes (< 10% left) + query: 'node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint ="/rootfs"} * 100 < 10' + severity: warning + - name: Host unusual disk read latency + description: Disk latency is growing (read operations > 100ms) + query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 100" + severity: warning + - name: Host unusual disk write latency + description: Disk latency is growing (write operations > 100ms) + query: "rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 100" + severity: warning + - name: Host high CPU load + description: CPU load is > 80% + query: '100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80' + severity: warning + - name: Host context switching + description: Context switching is growing on node (> 1000 / s) + query: "rate(node_context_switches_total[5m]) > 1000" + severity: warning + comments: | + 1000 context switches is an arbitrary number. + Alert threshold depends on nature of application. + Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58 + - name: Host swap is filling up + description: Swap is filling up (>80%) + query: "(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80" + severity: warning + - name: Host SystemD service crashed + description: "SystemD service crashed" + query: 'node_systemd_unit_state{state="failed"} == 1' + severity: warning + - name: Host physical component too hot + description: "Physical hardware component too hot" + query: "node_hwmon_temp_celsius > 75" + severity: warning + - name: Host node overtemperature alarm + description: "Physical node temperature alarm triggered" + query: "node_hwmon_temp_alarm == 1" + severity: error + - name: Host RAID array got inactive + description: 'RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.' + query: 'node_md_state{state="inactive"} > 0' + severity: error + - name: Host RAID disk failure + description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap' + query: 'node_md_disks{state="fail"} > 0' + severity: warning + - name: Host kernel version deviations + description: Different kernel versions are running + query: 'count(sum(label_replace(node_uname_info, "kernel", "$1", "release", "([0-9]+.[0-9]+.[0-9]+).*")) by (kernel)) > 1' + severity: warning + - name: Host OOM kill detected + description: OOM kill detected + query: 'increase(node_vmstat_oom_kill[30m]) > 1' + severity: warning - - name: Docker containers - exporters: - - name: cAdvisor - doc_url: https://github.com/google/cadvisor - rules: - - name: Container killed - description: A container has disappeared - query: "time() - container_last_seen > 60" - severity: warning - - name: Container CPU usage - description: Container CPU usage is above 80% - query: "(sum(rate(container_cpu_usage_seconds_total[3m])) BY (instance, name) * 100) > 80" - severity: warning - - name: Container Memory usage - description: Container Memory usage is above 80% - query: "(sum(container_memory_usage_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes) BY (instance, name) * 100) > 80" - severity: warning - - name: Container Volume usage - description: Container Volume usage is above 80% - query: "(1 - (sum(container_fs_inodes_free) BY (instance) / sum(container_fs_inodes_total) BY (instance)) * 100) > 80" - severity: warning - - name: Container Volume IO usage - description: Container Volume IO usage is above 80% - query: "(sum(container_fs_io_current) BY (instance, name) * 100) > 80" - severity: warning + - name: Docker containers + exporters: + - name: cAdvisor + doc_url: https://github.com/google/cadvisor + rules: + - name: Container killed + description: A container has disappeared + query: "time() - container_last_seen > 60" + severity: warning + - name: Container CPU usage + description: Container CPU usage is above 80% + query: "(sum(rate(container_cpu_usage_seconds_total[3m])) BY (instance, name) * 100) > 80" + severity: warning + - name: Container Memory usage + description: Container Memory usage is above 80% + query: "(sum(container_memory_usage_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes) BY (instance, name) * 100) > 80" + severity: warning + - name: Container Volume usage + description: Container Volume usage is above 80% + query: "(1 - (sum(container_fs_inodes_free) BY (instance) / sum(container_fs_inodes_total) BY (instance)) * 100) > 80" + severity: warning + - name: Container Volume IO usage + description: Container Volume IO usage is above 80% + query: "(sum(container_fs_io_current) BY (instance, name) * 100) > 80" + severity: warning + - name: Container high throttle rate + description: Container is being throttled + query: 'rate(container_cpu_cfs_throttled_seconds_total[3m]) > 1' + severity: warning - - name: RabbitMQ - exporters: - - name: kbudde/rabbitmq-exporter - doc_url: https://github.com/kbudde/rabbitmq_exporter - rules: - - name: Rabbitmq down - description: RabbitMQ node down - query: "rabbitmq_up == 0" - severity: error - - name: Rabbitmq cluster down - description: Less than 3 nodes running in RabbitMQ cluster - query: "sum(rabbitmq_running) < 3" - severity: error - - name: Rabbitmq cluster partition - description: Cluster partition - query: "rabbitmq_partitions > 0" - severity: error - - name: Rabbitmq out of memory - description: Memory available for RabbmitMQ is low (< 10%) - query: "rabbitmq_node_mem_used / rabbitmq_node_mem_limit * 100 > 90" - severity: warning - - name: Rabbitmq too many connections - description: RabbitMQ instance has too many connections (> 1000) - query: "rabbitmq_connectionsTotal > 1000" - severity: warning - - name: Rabbitmq dead letter queue filling up - description: Dead letter queue is filling up (> 10 msgs) - query: 'rabbitmq_queue_messages{queue="my-dead-letter-queue"} > 10' - severity: error - - name: Rabbitmq too many messages in queue - description: Queue is filling up (> 1000 msgs) - query: 'rabbitmq_queue_messages_ready{queue="my-queue"} > 1000' - severity: warning - - name: Rabbitmq slow queue consuming - description: Queue messages are consumed slowly (> 60s) - query: 'time() - rabbitmq_queue_head_message_timestamp{queue="my-queue"} > 60' - severity: warning - - name: Rabbitmq no consumer - description: Queue has no consumer - query: "rabbitmq_queue_consumers == 0" - severity: error - - name: Rabbitmq too many consumers - description: Queue should have only 1 consumer - query: "rabbitmq_queue_consumers > 1" - severity: error - - name: Rabbitmq unactive exchange - description: Exchange receive less than 5 msgs per second - query: 'rate(rabbitmq_exchange_messages_published_in_total{exchange="my-exchange"}[1m]) < 5' - severity: warning + - name: Blackbox + exporters: + - name: prometheus/blackbox_exporter + doc_url: https://github.com/prometheus/blackbox_exporter + rules: + - name: Blackbox probe failed + description: Probe failed + query: probe_success == 0 + severity: error + - name: Blackbox slow probe + description: Blackbox probe took more than 1s to complete + query: "avg_over_time(probe_duration_seconds[1m]) > 1" + severity: warning + - name: Blackbox probe HTTP failure + description: HTTP status code is not 200-399 + query: "probe_http_status_code <= 199 OR probe_http_status_code >= 400" + severity: error + - name: Blackbox SSL certificate will expire soon + description: SSL certificate expires in 30 days + query: "probe_ssl_earliest_cert_expiry - time() < 86400 * 30" + severity: warning + - name: Blackbox SSL certificate will expire soon + description: SSL certificate expires in 3 days + query: "probe_ssl_earliest_cert_expiry - time() < 86400 * 3" + severity: error + - name: Blackbox SSL certificate expired + description: SSL certificate has expired already + query: "probe_ssl_earliest_cert_expiry - time() <= 0" + severity: error + - name: Blackbox probe slow HTTP + description: HTTP request took more than 1s + query: "avg_over_time(probe_http_duration_seconds[1m]) > 1" + severity: warning + - name: Blackbox probe slow ping + description: Blackbox ping took more than 1s + query: "avg_over_time(probe_icmp_duration_seconds[1m]) > 1" + severity: warning - - name: MySQL - exporters: - - name: prometheus/mysqld_exporter - doc_url: https://github.com/prometheus/mysqld_exporter - rules: + - name: Windows Server + exporters: + - name: martinlindhe/wmi_exporter + doc_url: https://github.com/martinlindhe/wmi_exporter + rules: + - name: Windows Server collector Error + description: "Collector {{ $labels.collector }} was not successful" + query: "wmi_exporter_collector_success == 0" + severity: error + - name: Windows Server service Status + description: Windows Service state is not OK + query: 'wmi_service_status{status="ok"} != 1' + severity: error + - name: Windows Server CPU Usage + description: CPU Usage is more than 80% + query: '100 - (avg by (instance) (irate(wmi_cpu_time_total{mode="idle"}[2m])) * 100) > 80' + severity: warning + - name: Windows Server memory Usage + description: Memory Usage is more than 90% + query: "100*(wmi_os_physical_memory_free_bytes) / wmi_cs_physical_memory_bytes > 90" + severity: warning + - name: Windows Server disk Space Usage + description: Disk Space on Drive is used more than 80% + query: "100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{} / 1024 / 1024)) > 80" + severity: error - - name: PostgreSQL - exporters: - - name: wrouesnel/postgres_exporter - doc_url: https://github.com/wrouesnel/postgres_exporter/ - rules: - - name: Postgresql down - description: PostgreSQL instance is down - query: "pg_up == 0" - severity: error - - name: Postgresql replication lag - description: PostgreSQL replication lag is going up (> 10s) - query: "pg_replication_lag > 10" - severity: warning - comments: | - A label excluding master nodes should be added to this query, - in order to monitor lag on standby servers only. - Exporter does not guarantee a NaN value for pg_replication_log on promoted master nodes. - See https://github.com/samber/awesome-prometheus-alerts/issues/74 - - name: Postgresql table not vaccumed - description: Table has not been vaccum for 24 hours - query: "time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24" - severity: warning - - name: Postgresql table not analyzed - description: Table has not been analyzed for 24 hours - query: "time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24" - severity: warning - - name: Postgresql too many connections - description: PostgreSQL instance has too many connections - query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) > 100' - severity: warning - - name: Postgresql not enough connections - description: PostgreSQL instance should have more connections (> 5) - query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) < 5' - severity: warning - - name: Postgresql dead locks - description: PostgreSQL has dead-locks - query: 'rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0' - severity: warning - - name: Postgresql slow queries - description: PostgreSQL executes slow queries (> 1min) - query: 'avg(rate(pg_stat_activity_max_tx_duration{datname!~"template.*"}[1m])) BY (datname) > 60' - severity: warning - - name: Postgresql high rollback rate - description: Ratio of transactions being aborted compared to committed is > 2 % - query: 'rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02' - severity: warning - - name: Redis - exporters: - - name: oliver006/redis_exporter - doc_url: https://github.com/oliver006/redis_exporter - rules: - - name: Redis down - description: Redis instance is down - query: "redis_up == 0" - severity: error - - name: Redis missing backup - description: Redis has not been backuped for 24 hours - query: "time() - redis_rdb_last_save_timestamp_seconds > 60 * 60 * 24" - severity: error - - name: Redis out of memory - description: Redis is running out of memory (> 90%) - query: "redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90" - severity: warning - - name: Redis replication broken - description: Redis instance lost a slave - query: "delta(redis_connected_slaves[1m]) < 0" - severity: error - - name: Redis too many connections - description: Redis instance has too many connections - query: "redis_connected_clients > 100" - severity: warning - - name: Redis not enough connections - description: Redis instance should have more connections (> 5) - query: "redis_connected_clients < 5" - severity: warning - - name: Redis rejected connections - description: Some connections to Redis has been rejected - query: "increase(redis_rejected_connections_total[1m]) > 0" - severity: error + - name: Databases and brokers + services: + - name: MySQL + exporters: + - name: prometheus/mysqld_exporter + doc_url: https://github.com/prometheus/mysqld_exporter + rules: - - name: MongoDB - exporters: - - name: dcu/mongodb_exporter - doc_url: https://github.com/percona/mongodb_exporter - rules: - - name: MongoDB replication lag - description: Mongodb replication lag is more than 10s - query: 'avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}) > 10' - severity: error - - name: MongoDB replication headroom - description: MongoDB replication headroom is <= 0 - query: '(avg(mongodb_replset_oplog_tail_timestamp - mongodb_replset_oplog_head_timestamp) - (avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}))) <= 0' - severity: error - - name: MongoDB replication Status 3 - description: MongoDB Replication set member either perform startup self-checks, or transition from completing a rollback or resync - query: "mongodb_replset_member_state == 3" - severity: error - - name: MongoDB replication Status 6 - description: MongoDB Replication set member as seen from another member of the set, is not yet known - query: "mongodb_replset_member_state == 6" - severity: error - - name: MongoDB replication Status 8 - description: MongoDB Replication set member as seen from another member of the set, is unreachable - query: "mongodb_replset_member_state == 8" - severity: error - - name: MongoDB replication Status 9 - description: MongoDB Replication set member is actively performing a rollback. Data is not available for reads - query: "mongodb_replset_member_state == 9" - severity: error - - name: MongoDB replication Status 10 - description: MongoDB Replication set member was once in a replica set but was subsequently removed - query: "mongodb_replset_member_state == 10" - severity: error - - name: MongoDB number cursors open - description: Too many cursors opened by MongoDB for clients (> 10k) - query: 'mongodb_metrics_cursor_open{state="total_open"} > 10000' - severity: warning - - name: MongoDB cursors timeouts - description: Too many cursors are timing out - query: "increase(mongodb_metrics_cursor_timed_out_total[10m]) > 100" - severity: warning - - name: MongoDB too many connections - description: Too many connections - query: 'mongodb_connections{state="current"} > 500' - severity: warning - - name: MongoDB virtual memory usage - description: High memory usage - query: '(sum(mongodb_memory{type="virtual"}) BY (ip) / sum(mongodb_memory{type="mapped"}) BY (ip)) > 3' - severity: warning + - name: PostgreSQL + exporters: + - name: wrouesnel/postgres_exporter + doc_url: https://github.com/wrouesnel/postgres_exporter/ + rules: + - name: Postgresql down + description: Postgresql instance is down + query: "pg_up == 0" + severity: error + - name: Postgresql restarted + description: Postgresql restarted + query: "time() - pg_postmaster_start_time_seconds < 60" + severity: error + - name: Postgresql exporter error + description: Postgresql exporter is showing errors. A query may be buggy in query.yaml + query: 'pg_exporter_last_scrape_error > 0' + severity: warning + - name: Postgresql replication lag + description: PostgreSQL replication lag is going up (> 10s) + query: '(pg_replication_lag > 10 and ON(instance) (pg_replication_is_replica == 1)' + severity: warning + - name: Postgresql table not vaccumed + description: Table has not been vaccum for 24 hours + query: "time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24" + severity: warning + - name: Postgresql table not analyzed + description: Table has not been analyzed for 24 hours + query: "time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24" + severity: warning + - name: Postgresql too many connections + description: PostgreSQL instance has too many connections + query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) > pg_settings_max_connections * 0.9' + severity: warning + - name: Postgresql not enough connections + description: PostgreSQL instance should have more connections (> 5) + query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) < 5' + severity: warning + - name: Postgresql dead locks + description: PostgreSQL has dead-locks + query: 'rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0' + severity: warning + - name: Postgresql slow queries + description: PostgreSQL executes slow queries (> 1min) + query: 'rate(pg_slow_queries[1m]) * 60 > 10' + severity: warning + - name: Postgresql high rollback rate + description: Ratio of transactions being aborted compared to committed is > 2 % + query: 'rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02' + severity: warning + - name: Postgresql commit rate low + description: Postgres seems to be processing very few transactions + query: 'rate(pg_stat_database_xact_commit[1m]) < 10' + severity: error + - name: Postgresql low XID consumption + description: Postgresql seems to be consuming transaction IDs very slowly + query: 'rate(pg_txid_current[1m]) < 5' + severity: warning + - name: Postgresqllow XLOG consumption + description: Postgres seems to be consuming XLOG very slowly + query: 'rate(pg_xlog_position_bytes[1m]) < 100' + severity: warning + - name: Postgresql WALE replication stopped + description: WAL-E replication seems to be stopped + query: 'rate(pg_xlog_position_bytes[1m]) == 0' + severity: error + - name: Postgresql high rate statement timeout + description: Postgres transactions showing high rate of statement timeouts + query: 'rate(postgresql_errors_total{type="statement_timeout"}[5m]) > 3' + severity: error + - name: Postgresql high rate deadlock + description: Postgres detected deadlocks + query: 'rate(postgresql_errors_total{type="deadlock_detected"}[1m]) * 60 > 1' + severity: error + - name: Postgresql replication lab bytes + description: Postgres Replication lag (in bytes) is high + query: '(pg_xlog_position_bytes and pg_replication_is_replica == 0) - GROUP_RIGHT(instance) (pg_xlog_position_bytes and pg_replication_is_replica == 1) > 1e+09' + severity: error + - name: Postgresql unused replication slot + description: Unused Replication Slots + query: 'pg_replication_slots_active == 0' + severity: warning + - name: Postgresql too many dead tuples + description: PostgreSQL dead tuples is too large + query: '((pg_stat_user_tables_n_dead_tup > 10000) / (pg_stat_user_tables_n_live_tup + pg_stat_user_tables_n_dead_tup)) >= 0.1 unless ON(instance) (pg_replication_is_replica == 1)' + severity: warning + - name: Postgresql split brain + description: Split Brain, too many primary Postgresql databases in read-write mode + query: 'count(pg_replication_is_replica == 0) != 1' + severity: error + - name: Postgresql promoted node + description: Postgresql standby server has been promoted as primary node + query: 'pg_replication_is_replica and changes(pg_replication_is_replica[1m]) > 0' + severity: warning + - name: Postgresql configuration changed + description: Postgres Database configuration change has occurred + query: '{__name__=~"pg_settings_.*"} != ON(__name__) {__name__=~"pg_settings_([^t]|t[^r]|tr[^a]|tra[^n]|tran[^s]|trans[^a]|transa[^c]|transac[^t]|transact[^i]|transacti[^o]|transactio[^n]|transaction[^_]|transaction_[^r]|transaction_r[^e]|transaction_re[^a]|transaction_rea[^d]|transaction_read[^_]|transaction_read_[^o]|transaction_read_o[^n]|transaction_read_on[^l]|transaction_read_onl[^y]).*"} OFFSET 5m' + severity: warning + - name: Postgresql SSL compression active + description: Database connections with SSL compression enabled. This may add significant jitter in replication delay. Replicas should turn off SSL compression via `sslcompression=0` in `recovery.conf`. + query: 'sum(pg_stat_ssl_compression) > 0' + severity: error + - name: Postgresql too many locks acquired + description: Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction. + query: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20' + severity: error - - name: Elasticsearch - exporters: - - name: justwatchcom/elasticsearch_exporter - doc_url: https://github.com/justwatchcom/elasticsearch_exporter - rules: - - name: Elasticsearch Heap Usage Too High - description: "The heap usage is over 90% for 5m" - query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90' - severity: error - - name: Elasticsearch Heap Usage warning - description: "The heap usage is over 80% for 5m" - query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80' - severity: warning - - name: Elasticsearch Cluster Red - description: Elastic Cluster Red status - query: 'elasticsearch_cluster_health_status{color="red"} == 1' - severity: error - - name: Elasticsearch Cluster Yellow - description: Elastic Cluster Yellow status - query: 'elasticsearch_cluster_health_status{color="yellow"} == 1' - severity: warning - - name: Elasticsearch Healthy Nodes - description: "Number Healthy Nodes less then number_of_nodes" - query: "elasticsearch_cluster_health_number_of_nodes < number_of_nodes" - severity: error - - name: Elasticsearch Healthy Data Nodes - description: "Number Healthy Data Nodes less then number_of_data_nodes" - query: "elasticsearch_cluster_health_number_of_data_nodes < number_of_data_nodes" - severity: error - - name: Elasticsearch relocation shards - description: "Number of relocation shards for 20 min" - query: "elasticsearch_cluster_health_relocating_shards > 0" - severity: error - - name: Elasticsearch initializing shards - description: "Number of initializing shards for 10 min" - query: "elasticsearch_cluster_health_initializing_shards > 0" - severity: warning - - name: Elasticsearch unassigned shards - description: "Number of unassigned shards for 2 min" - query: "elasticsearch_cluster_health_unassigned_shards > 0" - severity: error - - name: Elasticsearch pending tasks - description: "Number of pending tasks for 10 min. Cluster works slowly." - query: "elasticsearch_cluster_health_number_of_pending_tasks > 0" - severity: warning - - name: Elasticsearch no new documents - description: No new documents for 10 min! - query: 'rate(elasticsearch_indices_docs{es_data_node="true"}[10m]) < 1' - severity: warning + - name: PGBouncer + exporters: + - name: spreaker/prometheus-pgbouncer-exporter + doc_url: https://github.com/spreaker/prometheus-pgbouncer-exporter + rules: + - name: PGBouncer active connectinos + description: PGBouncer pools are filling up + query: 'pgbouncer_pools_server_active_connections > 200' + severity: warning + - name: PGBouncer errors + description: PGBouncer is logging errors. This may be due to a a server restart or an admin typing commands at the pgbouncer console. + query: 'increase(pgbouncer_errors_count{errmsg!="server conn crashed?"}[5m]) > 10' + severity: warning + - name: PGBouncer max connections + description: The number of PGBouncer client connections has reached max_client_conn. + query: 'rate(pgbouncer_errors_count{errmsg="no more connections allowed (max_client_conn)"}[1m]) > 0' + severity: error - - name: Cassandra - exporters: - - name: instaclustr/cassandra-exporter - doc_url: https://github.com/instaclustr/cassandra-exporter - rules: + - name: Redis + exporters: + - name: oliver006/redis_exporter + doc_url: https://github.com/oliver006/redis_exporter + rules: + - name: Redis down + description: Redis instance is down + query: "redis_up == 0" + severity: error + - name: Redis missing master + description: Redis cluster has no node marked as master. + query: 'count(redis_instance_info{role="master"}) == 0' + severity: error + - name: Redis too many masters + description: Redis cluster has too many nodes marked as master. + query: 'count(redis_instance_info{role="master"}) > 1' + severity: error + - name: Redis disconnected slaves + description: Redis not replicating for all slaves. Consider reviewing the redis replication status. + query: 'count without (instance, job) (redis_connected_slaves) - sum without (instance, job) (redis_connected_slaves) - 1 > 1' + severity: error + - name: Redis replication broken + description: Redis instance lost a slave + query: "delta(redis_connected_slaves[1m]) < 0" + severity: error + - name: Redis cluster flapping + description: Changes have been detected in Redis replica connection. This can occur when replica nodes lose connection to the master and reconnect (a.k.a flapping). + query: 'changes(redis_connected_slaves[5m]) > 2' + severity: error + - name: Redis missing backup + description: Redis has not been backuped for 24 hours + query: "time() - redis_rdb_last_save_timestamp_seconds > 60 * 60 * 24" + severity: error + - name: Redis out of memory + description: Redis is running out of memory (> 90%) + query: "redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90" + severity: warning + - name: Redis too many connections + description: Redis instance has too many connections + query: "redis_connected_clients > 100" + severity: warning + - name: Redis not enough connections + description: Redis instance should have more connections (> 5) + query: "redis_connected_clients < 5" + severity: warning + - name: Redis rejected connections + description: Some connections to Redis has been rejected + query: "increase(redis_rejected_connections_total[1m]) > 0" + severity: error - - name: criteo/cassandra_exporter - doc_url: https://github.com/criteo/cassandra_exporter - rules: - - name: Cassandra hints count - description: Cassandra hints count has changed on {{ $labels.instance }} some nodes may go down - query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:totalhints:count"}[1m]) > 3' - severity: error - - name: Cassandra compaction task pending - description: Many Cassandra compaction tasks are pending. You might need to increase I/O capacity by adding nodes to the cluster. - query: 'avg_over_time(cassandra_stats{name="org:apache:cassandra:metrics:compaction:pendingtasks:value"}[30m]) > 100' - severity: warning - - name: Cassandra viewwrite latency - description: High viewwrite latency on {{ $labels.instance }} cassandra node - query: 'cassandra_stats{name="org:apache:cassandra:metrics:clientrequest:viewwrite:viewwritelatency:99thpercentile",service="cas"} > 100000' - severity: warning - - name: Cassandra cool hacker - description: Increase of Cassandra authentication failures - query: 'irate(cassandra_stats{name="org:apache:cassandra:metrics:client:authfailure:count"}[1m]) > 5' - severity: warning - - name: Cassandra node down - description: Cassandra node down - query: 'sum(cassandra_stats{name="org:apache:cassandra:net:failuredetector:downendpointcount"}) by (service,group,cluster,env) > 0' - severity: error - - name: Cassandra commitlog pending tasks - description: Unexpected number of Cassandra commitlog pending tasks - query: 'cassandra_stats{name="org:apache:cassandra:metrics:commitlog:pendingtasks:value"} > 15' - severity: warning - - name: Cassandra compaction executor blocked tasks - description: Some Cassandra compaction executor tasks are blocked - query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:compactionexecutor:currentlyblockedtasks:count"} > 0' - severity: warning - - name: Cassandra flush writer blocked tasks - description: Some Cassandra flush writer tasks are blocked - query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:memtableflushwriter:currentlyblockedtasks:count"} > 0' - severity: warning - - name: Cassandra repair pending tasks - description: Some Cassandra repair tasks are pending - query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:antientropystage:pendingtasks:value"} > 2' - severity: warning - - name: Cassandra repair blocked tasks - description: Some Cassandra repair tasks are blocked - query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:antientropystage:currentlyblockedtasks:count"} > 0' - severity: warning - - name: Cassandra connection timeouts total - description: Some connection between nodes are ending in timeout - query: 'rate(cassandra_stats{name="org:apache:cassandra:metrics:connection:totaltimeouts:count"}[1m]) > 5' - severity: error - - name: Cassandra storage exceptions - description: Something is going wrong with cassandra storage - query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:exceptions:count"}[1m]) > 1' - severity: error + - name: MongoDB + exporters: + - name: dcu/mongodb_exporter + doc_url: https://github.com/dcu/mongodb_exporter + rules: + - name: MongoDB replication lag + description: Mongodb replication lag is more than 10s + query: 'avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}) > 10' + severity: error + - name: MongoDB replication headroom + description: MongoDB replication headroom is <= 0 + query: '(avg(mongodb_replset_oplog_tail_timestamp - mongodb_replset_oplog_head_timestamp) - (avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}))) <= 0' + severity: error + - name: MongoDB replication Status 3 + description: MongoDB Replication set member either perform startup self-checks, or transition from completing a rollback or resync + query: "mongodb_replset_member_state == 3" + severity: error + - name: MongoDB replication Status 6 + description: MongoDB Replication set member as seen from another member of the set, is not yet known + query: "mongodb_replset_member_state == 6" + severity: error + - name: MongoDB replication Status 8 + description: MongoDB Replication set member as seen from another member of the set, is unreachable + query: "mongodb_replset_member_state == 8" + severity: error + - name: MongoDB replication Status 9 + description: MongoDB Replication set member is actively performing a rollback. Data is not available for reads + query: "mongodb_replset_member_state == 9" + severity: error + - name: MongoDB replication Status 10 + description: MongoDB Replication set member was once in a replica set but was subsequently removed + query: "mongodb_replset_member_state == 10" + severity: error + - name: MongoDB number cursors open + description: Too many cursors opened by MongoDB for clients (> 10k) + query: 'mongodb_metrics_cursor_open{state="total_open"} > 10000' + severity: warning + - name: MongoDB cursors timeouts + description: Too many cursors are timing out + query: "increase(mongodb_metrics_cursor_timed_out_total[10m]) > 100" + severity: warning + - name: MongoDB too many connections + description: Too many connections + query: 'mongodb_connections{state="current"} > 500' + severity: warning + - name: MongoDB virtual memory usage + description: High memory usage + query: '(sum(mongodb_memory{type="virtual"}) BY (ip) / sum(mongodb_memory{type="mapped"}) BY (ip)) > 3' + severity: warning - - name: Nginx - exporters: - - name: nginx-lua-prometheus - doc_url: https://github.com/knyar/nginx-lua-prometheus - rules: - - name: Nginx high HTTP 4xx error rate - description: Too many HTTP requests with status 4xx (> 5%) - query: 'sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' - severity: error - - name: Nginx high HTTP 5xx error rate - description: Too many HTTP requests with status 5xx (> 5%) - query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' - severity: error + - name: RabbitMQ + exporters: + - name: kbudde/rabbitmq-exporter + doc_url: https://github.com/kbudde/rabbitmq_exporter + rules: + - name: Rabbitmq down + description: RabbitMQ node down + query: "rabbitmq_up == 0" + severity: error + - name: Rabbitmq cluster down + description: Less than 3 nodes running in RabbitMQ cluster + query: "sum(rabbitmq_running) < 3" + severity: error + - name: Rabbitmq cluster partition + description: Cluster partition + query: "rabbitmq_partitions > 0" + severity: error + - name: Rabbitmq out of memory + description: Memory available for RabbmitMQ is low (< 10%) + query: "rabbitmq_node_mem_used / rabbitmq_node_mem_limit * 100 > 90" + severity: warning + - name: Rabbitmq too many connections + description: RabbitMQ instance has too many connections (> 1000) + query: "rabbitmq_connectionsTotal > 1000" + severity: warning + - name: Rabbitmq dead letter queue filling up + description: Dead letter queue is filling up (> 10 msgs) + query: 'rabbitmq_queue_messages{queue="my-dead-letter-queue"} > 10' + severity: error + - name: Rabbitmq too many messages in queue + description: Queue is filling up (> 1000 msgs) + query: 'rabbitmq_queue_messages_ready{queue="my-queue"} > 1000' + severity: warning + - name: Rabbitmq slow queue consuming + description: Queue messages are consumed slowly (> 60s) + query: 'time() - rabbitmq_queue_head_message_timestamp{queue="my-queue"} > 60' + severity: warning + - name: Rabbitmq no consumer + description: Queue has no consumer + query: "rabbitmq_queue_consumers == 0" + severity: error + - name: Rabbitmq too many consumers + description: Queue should have only 1 consumer + query: "rabbitmq_queue_consumers > 1" + severity: error + - name: Rabbitmq unactive exchange + description: Exchange receive less than 5 msgs per second + query: 'rate(rabbitmq_exchange_messages_published_in_total{exchange="my-exchange"}[1m]) < 5' + severity: warning - - name: Apache - exporters: - - name: Lusitaniae/apache_exporter - doc_url: https://github.com/Lusitaniae/apache_exporter - rules: + - name: Elasticsearch + exporters: + - name: justwatchcom/elasticsearch_exporter + doc_url: https://github.com/justwatchcom/elasticsearch_exporter + rules: + - name: Elasticsearch Heap Usage Too High + description: "The heap usage is over 90% for 5m" + query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90' + severity: error + - name: Elasticsearch Heap Usage warning + description: "The heap usage is over 80% for 5m" + query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80' + severity: warning + - name: Elasticsearch Cluster Red + description: Elastic Cluster Red status + query: 'elasticsearch_cluster_health_status{color="red"} == 1' + severity: error + - name: Elasticsearch Cluster Yellow + description: Elastic Cluster Yellow status + query: 'elasticsearch_cluster_health_status{color="yellow"} == 1' + severity: warning + - name: Elasticsearch Healthy Nodes + description: "Number Healthy Nodes less then number_of_nodes" + query: "elasticsearch_cluster_health_number_of_nodes < number_of_nodes" + severity: error + - name: Elasticsearch Healthy Data Nodes + description: "Number Healthy Data Nodes less then number_of_data_nodes" + query: "elasticsearch_cluster_health_number_of_data_nodes < number_of_data_nodes" + severity: error + - name: Elasticsearch relocation shards + description: "Number of relocation shards for 20 min" + query: "elasticsearch_cluster_health_relocating_shards > 0" + severity: error + - name: Elasticsearch initializing shards + description: "Number of initializing shards for 10 min" + query: "elasticsearch_cluster_health_initializing_shards > 0" + severity: warning + - name: Elasticsearch unassigned shards + description: "Number of unassigned shards for 2 min" + query: "elasticsearch_cluster_health_unassigned_shards > 0" + severity: error + - name: Elasticsearch pending tasks + description: "Number of pending tasks for 10 min. Cluster works slowly." + query: "elasticsearch_cluster_health_number_of_pending_tasks > 0" + severity: warning + - name: Elasticsearch no new documents + description: No new documents for 10 min! + query: 'rate(elasticsearch_indices_docs{es_data_node="true"}[10m]) < 1' + severity: warning - - name: HaProxy - exporters: - - name: Embedded exporter (HAProxy >= v2) - doc_url: https://github.com/haproxy/haproxy/tree/master/contrib/prometheus-exporter - rules: - - name: prometheus/haproxy_exporter (HAProxy < v2) - doc_url: https://github.com/prometheus/haproxy_exporter - rules: - - name: HAProxy down - description: HAProxy down - query: 'haproxy_up = 0' - severity: error - - name: HAProxy high HTTP 4xx error rate backend - description: Too many HTTP requests with status 4xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} - query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' - severity: error - - name: HAProxy high HTTP 4xx error rate backend - description: Too many HTTP requests with status 5xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} - query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' - severity: error - - name: HAProxy high HTTP 4xx error rate server - description: Too many HTTP requests with status 4xx (> 5%) on server {{ $labels.server }} - query: 'sum by (server) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' - severity: error - - name: HAProxy high HTTP 5xx error rate server - description: Too many HTTP requests with status 5xx (> 5%) on server {{ $labels.server }} - query: 'sum by (server) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' - severity: error - - name: HAProxy backend connection errors - description: Too many connection errors to {{ $labels.fqdn }}/{{ $labels.backend }} backend (> 5%). Request throughput may be to high. - query: 'sum by (backend) rate(haproxy_backend_connection_errors_total[1m]) * 100 > 5' - severity: error - - name: HAProxy server response errors - description: Too many response errors to {{ $labels.server }} server (> 5%). - query: 'sum by (server) rate(haproxy_server_response_errors_total[1m]) * 100 > 5' - severity: error - - name: HAProxy server connection errors - description: Too many connection errors to {{ $labels.server }} server (> 5%). Request throughput may be to high. - query: 'sum by (server) rate(haproxy_server_connection_errors_total[1m]) * 100 > 5' - severity: error - - name: HAProxy backend max active session - description: HAproxy backend {{ $labels.fqdn }}/{{ $labels.backend }} is reaching session limit (> 80%). - query: 'avg_over_time((sum by (backend) (haproxy_server_max_sessions) / sum by (backend) (haproxy_server_limit_sessions)) [2m]) * 100 > 80' - severity: warning - - name: HAProxy pending requests - description: Some HAProxy requests are pending on {{ $labels.fqdn }}/{{ $labels.backend }} backend - query: 'sum by (backend) haproxy_backend_current_queue > 0' - severity: warning - - name: HAProxy HTTP slowing down - description: Average request time is increasing - query: 'avg by (backend) (haproxy_backend_http_total_time_average_seconds) > 2' - severity: warning - - name: HAProxy retry high - description: High rate of retry on {{ $labels.fqdn }}/{{ $labels.backend }} backend - query: 'rate(sum by (backend) (haproxy_backend_retry_warnings_total)) > 10' - severity: warning - - name: HAProxy backend down - description: HAProxy backend is down - query: 'haproxy_backend_up = 0' - severity: error - - name: HAProxy server down - description: HAProxy server is down - query: 'haproxy_server_up = 0' - severity: error - - name: HAProxy frontend security blocked requests - description: HAProxy is blocking requests for security reason - query: 'rate(sum by (frontend) (haproxy_frontend_requests_denied_total)) > 10' - severity: warning - - name: HAProxy server healthcheck failure - description: Some server healthcheck are failing on {{ $labels.server }} - query: 'increase(haproxy_server_check_failures_total) > 0' - severity: warning + - name: Cassandra + exporters: + - name: instaclustr/cassandra-exporter + doc_url: https://github.com/instaclustr/cassandra-exporter + rules: - - name: Traefik - exporters: - - name: Embedded exporter - doc_url: https://docs.traefik.io/observability/metrics/prometheus/ - rules: - - name: Traefik backend down - description: All Traefik backends are down - query: "count(traefik_backend_server_up) by (backend) == 0" - severity: error - - name: Traefik high HTTP 4xx error rate backend - description: Traefik backend 4xx error rate is above 5% - query: 'sum(rate(traefik_backend_requests_total{code=~"4.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' - severity: error - - name: Traefik high HTTP 5xx error rate backend - description: Traefik backend 5xx error rate is above 5% - query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' - severity: error + - name: criteo/cassandra_exporter + doc_url: https://github.com/criteo/cassandra_exporter + rules: + - name: Cassandra hints count + description: Cassandra hints count has changed on {{ $labels.instance }} some nodes may go down + query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:totalhints:count"}[1m]) > 3' + severity: error + - name: Cassandra compaction task pending + description: Many Cassandra compaction tasks are pending. You might need to increase I/O capacity by adding nodes to the cluster. + query: 'avg_over_time(cassandra_stats{name="org:apache:cassandra:metrics:compaction:pendingtasks:value"}[30m]) > 100' + severity: warning + - name: Cassandra viewwrite latency + description: High viewwrite latency on {{ $labels.instance }} cassandra node + query: 'cassandra_stats{name="org:apache:cassandra:metrics:clientrequest:viewwrite:viewwritelatency:99thpercentile",service="cas"} > 100000' + severity: warning + - name: Cassandra cool hacker + description: Increase of Cassandra authentication failures + query: 'irate(cassandra_stats{name="org:apache:cassandra:metrics:client:authfailure:count"}[1m]) > 5' + severity: warning + - name: Cassandra node down + description: Cassandra node down + query: 'sum(cassandra_stats{name="org:apache:cassandra:net:failuredetector:downendpointcount"}) by (service,group,cluster,env) > 0' + severity: error + - name: Cassandra commitlog pending tasks + description: Unexpected number of Cassandra commitlog pending tasks + query: 'cassandra_stats{name="org:apache:cassandra:metrics:commitlog:pendingtasks:value"} > 15' + severity: warning + - name: Cassandra compaction executor blocked tasks + description: Some Cassandra compaction executor tasks are blocked + query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:compactionexecutor:currentlyblockedtasks:count"} > 0' + severity: warning + - name: Cassandra flush writer blocked tasks + description: Some Cassandra flush writer tasks are blocked + query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:memtableflushwriter:currentlyblockedtasks:count"} > 0' + severity: warning + - name: Cassandra repair pending tasks + description: Some Cassandra repair tasks are pending + query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:antientropystage:pendingtasks:value"} > 2' + severity: warning + - name: Cassandra repair blocked tasks + description: Some Cassandra repair tasks are blocked + query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:antientropystage:currentlyblockedtasks:count"} > 0' + severity: warning + - name: Cassandra connection timeouts total + description: Some connection between nodes are ending in timeout + query: 'rate(cassandra_stats{name="org:apache:cassandra:metrics:connection:totaltimeouts:count"}[1m]) > 5' + severity: error + - name: Cassandra storage exceptions + description: Something is going wrong with cassandra storage + query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:exceptions:count"}[1m]) > 1' + severity: error - - name: PHP-FPM - exporters: - - name: bakins/php-fpm-exporter - doc_url: https://github.com/bakins/php-fpm-exporter - rules: + - name: Zookeeper + exporters: + - name: cloudflare/kafka_zookeeper_exporter + doc_url: https://github.com/cloudflare/kafka_zookeeper_exporter + rules: - - name: JVM - exporters: - - name: java-client - doc_url: https://github.com/prometheus/client_java - rules: - - name: JVM memory filling up - description: JVM memory is filling up (> 80%) - query: 'jvm_memory_bytes_used / jvm_memory_bytes_max{area="heap"} > 0.8' - severity: warning + - name: Kafka + exporters: + - name: danielqsj/kafka_exporter + doc_url: https://github.com/danielqsj/kafka_exporter + rules: + - name: Kafka topics replicas + description: Kafka topic in-sync partition + query: "sum(kafka_topic_partition_in_sync_replica) by (topic) < 3" + severity: error + - name: Kafka consumers group + description: Kafka consumers group + query: "sum(kafka_consumergroup_lag) by (consumergroup) > 50" + severity: error - - name: ZFS - exporters: - - name: node-exporter - doc_url: https://github.com/prometheus/node_exporter - rules: - - name: Kubernetes - exporters: - - name: kube-state-metrics - doc_url: https://github.com/kubernetes/kube-state-metrics/tree/master/docs - rules: - - name: Kubernetes MemoryPressure - description: "{{ $labels.node }} has MemoryPressure condition" - query: 'kube_node_status_condition{condition="MemoryPressure",status="true"} == 1' - severity: error - - name: Kubernetes DiskPressure - description: "{{ $labels.node }} has DiskPressure condition" - query: 'kube_node_status_condition{condition="DiskPressure",status="true"} == 1' - severity: error - - name: Kubernetes OutOfDisk - description: "{{ $labels.node }} has OutOfDisk condition" - query: 'kube_node_status_condition{condition="OutOfDisk",status="true"} == 1' - severity: error - - name: Kubernetes Job failed - description: "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete" - query: "kube_job_status_failed > 0" - severity: warning - - name: Kubernetes CronJob suspended - description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended" - query: "kube_cronjob_spec_suspend != 0" - severity: info - - name: Kubernetes PersistentVolumeClaim pending - description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending" - query: 'kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1' - severity: warning - - name: Kubernetes Volume out of disk space - description: Volume is almost full (< 10% left) - query: "kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10" - severity: warning - - name: Kubernetes Volume full in four days - description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available." - query: "100 * (kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) < 15 and predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0" - severity: error - - name: Kubernetes StatefulSet down - description: A StatefulSet went down - query: "(kube_statefulset_status_replicas_ready / kube_statefulset_status_replicas_current) != 1" - severity: error + - name: Reverse proxies and load balancers + services: + - name: Nginx + exporters: + - name: nginx-lua-prometheus + doc_url: https://github.com/knyar/nginx-lua-prometheus + rules: + - name: Nginx high HTTP 4xx error rate + description: Too many HTTP requests with status 4xx (> 5%) + query: 'sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' + severity: error + - name: Nginx high HTTP 5xx error rate + description: Too many HTTP requests with status 5xx (> 5%) + query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' + severity: error - - name: Nomad - exporters: - - name: Embedded exporter - rules: + - name: Apache + exporters: + - name: Lusitaniae/apache_exporter + doc_url: https://github.com/Lusitaniae/apache_exporter + rules: - - name: Consul - exporters: - - name: prometheus/consul_exporter - doc_url: https://github.com/prometheus/consul_exporter - rules: - - name: Consul service healthcheck failed - description: "Service: `{{ $labels.service_name }}` Healthcheck: `{{ $labels.service_id }}`" - query: "consul_catalog_service_node_healthy == 0" - severity: error - - name: Consul missing master node - description: Numbers of consul raft peers should be 3, in order to preserve quorum. - query: "consul_raft_peers < 3" - severity: error - - name: Consul agent unhealthy - description: A Consul agent is down - query: 'consul_health_node_status{status="critical"} == 1' - severity: error + - name: HaProxy + exporters: + - name: Embedded exporter (HAProxy >= v2) + doc_url: https://github.com/haproxy/haproxy/tree/master/contrib/prometheus-exporter + rules: + - name: prometheus/haproxy_exporter (HAProxy < v2) + doc_url: https://github.com/prometheus/haproxy_exporter + rules: + - name: HAProxy down + description: HAProxy down + query: 'haproxy_up = 0' + severity: error + - name: HAProxy high HTTP 4xx error rate backend + description: Too many HTTP requests with status 4xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} + query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' + severity: error + - name: HAProxy high HTTP 4xx error rate backend + description: Too many HTTP requests with status 5xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} + query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' + severity: error + - name: HAProxy high HTTP 4xx error rate server + description: Too many HTTP requests with status 4xx (> 5%) on server {{ $labels.server }} + query: 'sum by (server) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' + severity: error + - name: HAProxy high HTTP 5xx error rate server + description: Too many HTTP requests with status 5xx (> 5%) on server {{ $labels.server }} + query: 'sum by (server) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' + severity: error + - name: HAProxy backend connection errors + description: Too many connection errors to {{ $labels.fqdn }}/{{ $labels.backend }} backend (> 5%). Request throughput may be to high. + query: 'sum by (backend) rate(haproxy_backend_connection_errors_total[1m]) * 100 > 5' + severity: error + - name: HAProxy server response errors + description: Too many response errors to {{ $labels.server }} server (> 5%). + query: 'sum by (server) rate(haproxy_server_response_errors_total[1m]) * 100 > 5' + severity: error + - name: HAProxy server connection errors + description: Too many connection errors to {{ $labels.server }} server (> 5%). Request throughput may be to high. + query: 'sum by (server) rate(haproxy_server_connection_errors_total[1m]) * 100 > 5' + severity: error + - name: HAProxy backend max active session + description: HAproxy backend {{ $labels.fqdn }}/{{ $labels.backend }} is reaching session limit (> 80%). + query: 'avg_over_time((sum by (backend) (haproxy_server_max_sessions) / sum by (backend) (haproxy_server_limit_sessions)) [2m]) * 100 > 80' + severity: warning + - name: HAProxy pending requests + description: Some HAProxy requests are pending on {{ $labels.fqdn }}/{{ $labels.backend }} backend + query: 'sum by (backend) haproxy_backend_current_queue > 0' + severity: warning + - name: HAProxy HTTP slowing down + description: Average request time is increasing + query: 'avg by (backend) (haproxy_backend_http_total_time_average_seconds) > 2' + severity: warning + - name: HAProxy retry high + description: High rate of retry on {{ $labels.fqdn }}/{{ $labels.backend }} backend + query: 'rate(sum by (backend) (haproxy_backend_retry_warnings_total)) > 10' + severity: warning + - name: HAProxy backend down + description: HAProxy backend is down + query: 'haproxy_backend_up = 0' + severity: error + - name: HAProxy server down + description: HAProxy server is down + query: 'haproxy_server_up = 0' + severity: error + - name: HAProxy frontend security blocked requests + description: HAProxy is blocking requests for security reason + query: 'rate(sum by (frontend) (haproxy_frontend_requests_denied_total)) > 10' + severity: warning + - name: HAProxy server healthcheck failure + description: Some server healthcheck are failing on {{ $labels.server }} + query: 'increase(haproxy_server_check_failures_total) > 0' + severity: warning - - name: Etcd - exporters: - - rules: - - name: Etcd insufficient Members - description: Etcd cluster should have an odd number of members - query: "count(etcd_server_id) % 2 == 0" - severity: error - - name: Etcd no Leader - description: Etcd cluster have no leader - query: "etcd_server_has_leader == 0" - severity: error - - name: Etcd high number of leader changes - description: Etcd leader changed more than 3 times during last hour - query: "increase(etcd_server_leader_changes_seen_total[1h]) > 3" - severity: warning - - name: Etcd high number of failed GRPC requests - description: More than 1% GRPC request failure detected in Etcd for 5 minutes - query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.01' - severity: warning - - name: Etcd high number of failed GRPC requests - description: More than 5% GRPC request failure detected in Etcd for 5 minutes - query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.05' - severity: error - - name: Etcd GRPC requests slow - description: GRPC requests slowing down, 99th percentil is over 0.15s for 5 minutes - query: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15' - severity: warning - - name: Etcd high number of failed HTTP requests - description: More than 1% HTTP failure detected in Etcd for 5 minutes - query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.01" - severity: warning - - name: Etcd high number of failed HTTP requests - description: More than 5% HTTP failure detected in Etcd for 5 minutes - query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.05" - severity: error - - name: Etcd HTTP requests slow - description: HTTP requests slowing down, 99th percentil is over 0.15s for 5 minutes - query: "histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15" - severity: warning - - name: Etcd member communication slow - description: Etcd member communication slowing down, 99th percentil is over 0.15s for 5 minutes - query: "histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) > 0.15" - severity: warning - - name: Etcd high number of failed proposals - description: Etcd server got more than 5 failed proposals past hour - query: "increase(etcd_server_proposals_failed_total[1h]) > 5" - severity: warning - - name: Etcd high fsync durations - description: Etcd WAL fsync duration increasing, 99th percentil is over 0.5s for 5 minutes - query: "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 0.5" - severity: warning - - name: Etcd high commit durations - description: Etcd commit duration increasing, 99th percentil is over 0.25s for 5 minutes - query: "histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) > 0.25" - severity: warning + - name: Traefik + exporters: + - name: Embedded exporter + doc_url: https://docs.traefik.io/observability/metrics/prometheus/ + rules: + - name: Traefik backend down + description: All Traefik backends are down + query: "count(traefik_backend_server_up) by (backend) == 0" + severity: error + - name: Traefik high HTTP 4xx error rate backend + description: Traefik backend 4xx error rate is above 5% + query: 'sum(rate(traefik_backend_requests_total{code=~"4.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' + severity: error + - name: Traefik high HTTP 5xx error rate backend + description: Traefik backend 5xx error rate is above 5% + query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' + severity: error - - name: Zookeeper - exporters: - - name: cloudflare/kafka_zookeeper_exporter - doc_url: https://github.com/cloudflare/kafka_zookeeper_exporter - rules: - - name: Kafka - exporters: - - name: danielqsj/kafka_exporter - doc_url: https://github.com/danielqsj/kafka_exporter - rules: - - name: Kafka topics replicas - description: Kafka topic in-sync partition - query: "sum(kafka_topic_partition_in_sync_replica) by (topic) < 3" - severity: error - - name: Kafka consumers group - description: Kafka consumers group - query: "sum(kafka_consumergroup_lag) by (consumergroup) > 50" - severity: error + - name: Runtimes + services: + - name: PHP-FPM + exporters: + - name: bakins/php-fpm-exporter + doc_url: https://github.com/bakins/php-fpm-exporter + rules: - - name: Linkerd - exporters: - - rules: + - name: JVM + exporters: + - name: java-client + doc_url: https://github.com/prometheus/client_java + rules: + - name: JVM memory filling up + description: JVM memory is filling up (> 80%) + query: 'jvm_memory_bytes_used / jvm_memory_bytes_max{area="heap"} > 0.8' + severity: warning - - name: Istio - exporters: - - rules: + - name: Sidekiq + exporters: + - name: Strech/sidekiq-prometheus-exporter + doc_url: https://github.com/Strech/sidekiq-prometheus-exporter + rules: + - name: Sidekiq queue size + description: Sidekiq queue {{ $labels.name }} is growing + query: 'sidekiq_queue_size{} > 100' + severity: warning + - name: Sidekiq scheduling latency too high + description: Sidekiq jobs are taking more than 2 minutes to be picked up. Users may be seeing delays in background processing. + query: 'max(sidekiq_queue_latency) > 120' + severity: error - - name: Blackbox - exporters: - - name: prometheus/blackbox_exporter - doc_url: https://github.com/prometheus/blackbox_exporter - rules: - - name: Blackbox probe failed - description: Probe failed - query: probe_success == 0 - severity: error - - name: Blackbox slow probe - description: Blackbox probe took more than 1s to complete - query: "avg_over_time(probe_duration_seconds[1m]) > 1" - severity: warning - - name: Blackbox HTTP Status Code - description: HTTP status code is not 200-399 - query: "probe_http_status_code <= 199 OR probe_http_status_code >= 400" - severity: error - - name: Blackbox SSL certificate will expire soon - description: SSL certificate expires in 30 days - query: "probe_ssl_earliest_cert_expiry - time() < 86400 * 30" - severity: warning - - name: Blackbox SSL certificate expired - description: SSL certificate has expired already - query: "probe_ssl_earliest_cert_expiry - time() <= 0" - severity: error - - name: Blackbox HTTP slow requests - description: HTTP request took more than 1s - query: "avg_over_time(probe_http_duration_seconds[1m]) > 1" - severity: warning - - name: Blackbox slow ping - description: Blackbox ping took more than 1s - query: "avg_over_time(probe_icmp_duration_seconds[1m]) > 1" - severity: warning - - name: Windows Server - exporters: - - name: martinlindhe/wmi_exporter - doc_url: https://github.com/martinlindhe/wmi_exporter - rules: - - name: Windows Server collector Error - description: "Collector {{ $labels.collector }} was not successful" - query: "wmi_exporter_collector_success == 0" - severity: error - - name: Windows Server service Status - description: Windows Service state is not OK - query: 'wmi_service_status{status="ok"} != 1' - severity: error - - name: Windows Server CPU Usage - description: CPU Usage is more than 80% - query: '100 - (avg by (instance) (irate(wmi_cpu_time_total{mode="idle"}[2m])) * 100) > 80' - severity: warning - - name: Windows Server memory Usage - description: Memory Usage is more than 90% - query: "100*(wmi_os_physical_memory_free_bytes) / wmi_cs_physical_memory_bytes > 90" - severity: warning - - name: Windows Server disk Space Usage - description: Disk Space on Drive is used more than 80% - query: "100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{} / 1024 / 1024)) > 80" - severity: error + - name: Orchestrators + services: + - name: Kubernetes + exporters: + - name: kube-state-metrics + doc_url: https://github.com/kubernetes/kube-state-metrics/tree/master/docs + rules: + - name: Kubernetes Node ready + description: Node {{ $labels.node }} has been unready for a long time + query: 'kube_node_status_condition{condition="Ready",status="true"} == 0' + severity: error + - name: Kubernetes memory pressure + description: "{{ $labels.node }} has MemoryPressure condition" + query: 'kube_node_status_condition{condition="MemoryPressure",status="true"} == 1' + severity: error + - name: Kubernetes disk pressure + description: "{{ $labels.node }} has DiskPressure condition" + query: 'kube_node_status_condition{condition="DiskPressure",status="true"} == 1' + severity: error + - name: Kubernetes out of disk + description: "{{ $labels.node }} has OutOfDisk condition" + query: 'kube_node_status_condition{condition="OutOfDisk",status="true"} == 1' + severity: error + - name: Kubernetes Job failed + description: "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete" + query: "kube_job_status_failed > 0" + severity: warning + - name: Kubernetes CronJob suspended + description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended" + query: "kube_cronjob_spec_suspend != 0" + severity: warning + - name: Kubernetes PersistentVolumeClaim pending + description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending" + query: 'kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1' + severity: warning + - name: Kubernetes Volume out of disk space + description: Volume is almost full (< 10% left) + query: "kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10" + severity: warning + - name: Kubernetes Volume full in four days + description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available." + query: 'predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0' + severity: error + - name: Kubernetes PersistentVolume error + description: "Persistent volume is in bad state" + query: 'kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0' + severity: error + - name: Kubernetes StatefulSet down + description: A StatefulSet went down + query: "(kube_statefulset_status_replicas_ready / kube_statefulset_status_replicas_current) != 1" + severity: error + - name: Kubernetes HPA scaling ability + description: Pod is unable to scale + query: 'kube_hpa_status_condition{condition="false", status="AbleToScale"} == 1' + severity: warning + - name: Kubernetes HPA metric availability + description: HPA is not able to colelct metrics + query: 'kube_hpa_status_condition{condition="false", status="ScalingActive"} == 1' + severity: warning + - name: Kubernetes HPA scale capability + description: The maximum number of desired Pods has been hit + query: 'kube_hpa_status_desired_replicas >= kube_hpa_spec_max_replicas' + severity: warning + - name: Kubernetes Pod not healthy + description: Pod has been in a non-ready state for longer than an hour. + query: 'min_over_time(sum by (namespace, pod, env, stage) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h]) > 0' + severity: error + - name: Kubernetes pod crash looping + description: Pod {{ $labels.pod }} is crash looping + query: 'rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5' + severity: warning + - name: Kubernetes ReplicasSet mismatch + description: Deployment Replicas mismatch + query: 'kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas' + severity: warning + - name: Kubernetes Deployment replicas mismatch + description: Deployment Replicas mismatch + query: 'kube_deployment_spec_replicas != kube_deployment_status_replicas_available' + severity: warning + - name: Kubernetes StatefulSet replicas mismatch + description: A StatefulSet has not matched the expected number of replicas for longer than 15 minutes. + query: 'kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas' + severity: warning + - name: Kubernetes Deployment generation mismatch + description: A Deployment has failed but has not been rolled back. + query: 'kube_deployment_status_observed_generation != kube_deployment_metadata_generation' + severity: error + - name: Kubernetes StatefulSet generation mismatch + description: A StatefulSet has failed but has not been rolled back. + query: 'kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation' + severity: error + - name: Kubernetes StatefulSet update not rolled out + description: StatefulSet update has not been rolled out. + query: 'max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)' + severity: error + - name: Kubernetes DaemonSet rollout stuck + description: Some Pods of DaemonSet are not scheduled or not ready + query: 'kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0' + severity: error + - name: Kubernetes DaemonSet misscheduled + description: Some DaemonSet Pods are running where they are not supposed to run + query: 'kube_daemonset_status_number_misscheduled > 0' + severity: error + - name: Kubernetes CronJob too long + description: CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete. + query: 'time() - kube_cronjob_next_schedule_time > 3600' + severity: warning + - name: Kubernetes job completion + description: Kubernetes Job failed to complete + query: 'kube_job_spec_completions - kube_job_status_succeeded > 0 or kube_job_status_failed > 0' + severity: error + - name: Kubernetes API server errors + description: Kubernetes API server is experiencing high error rate + query: 'sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[2m])) / sum(rate(apiserver_request_count{job="apiserver"}[2m])) * 100 > 3' + severity: error + - name: Kubernetes API client errors + description: Kubernetes API client is experiencing high error rate + query: '(sum(rate(rest_client_requests_total{code=~"(4|5).."}[2m])) by (instance, job) / sum(rate(rest_client_requests_total[2m])) by (instance, job)) * 100 > 1' + severity: error + - name: Kubernetes client certificate expires next week + description: A client certificate used to authenticate to the apiserver is expiring next week. + query: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60' + severity: warning + - name: Kubernetes client certificate expires soon + description: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. + query: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60' + severity: error - - name: OpenEBS - exporters: - - name: Embedded exporter - rules: - - name: OpenEBS used pool capacity - description: 'OpenEBS Pool use more than 80% of his capacity\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' - query: "(openebs_used_pool_capacity_percent) > 80" - severity: warning + - name: Nomad + exporters: + - name: Embedded exporter + rules: - - name: Minio - exporters: - - name: Embedded exporter - rules: - - name: Minio disk offline - description: 'Minio disk is offline' - query: "minio_offline_disks > 0" - severity: error - - name: Minio storage space exhausted - description: 'Minio storage space is low (< 10 GB)' - query: "minio_disk_storage_free_bytes / 1024 / 1024 / 1024 < 10" - severity: warning + - name: Consul + exporters: + - name: prometheus/consul_exporter + doc_url: https://github.com/prometheus/consul_exporter + rules: + - name: Consul service healthcheck failed + description: "Service: `{{ $labels.service_name }}` Healthcheck: `{{ $labels.service_id }}`" + query: "consul_catalog_service_node_healthy == 0" + severity: error + - name: Consul missing master node + description: Numbers of consul raft peers should be 3, in order to preserve quorum. + query: "consul_raft_peers < 3" + severity: error + - name: Consul agent unhealthy + description: A Consul agent is down + query: 'consul_health_node_status{status="critical"} == 1' + severity: error - - name: Juniper - exporters: - - name: czerwonk/junos_exporter - doc_url: https://github.com/czerwonk/junos_exporter - rules: - - name: Juniper switch down - description: The switch appears to be down - query: junos_up == 0 - severity: error - - name: Juniper high Bandwith Usage 1GiB - description: Interface is highly saturated for at least 1 min. (> 0.90GiB/s) - query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.90" - severity: error - - name: Juniper high Bandwith Usage 1GiB - description: Interface is getting saturated for at least 1 min. (> 0.80GiB/s) - query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.80" - severity: warning + - name: Etcd + exporters: + - rules: + - name: Etcd insufficient Members + description: Etcd cluster should have an odd number of members + query: "count(etcd_server_id) % 2 == 0" + severity: error + - name: Etcd no Leader + description: Etcd cluster have no leader + query: "etcd_server_has_leader == 0" + severity: error + - name: Etcd high number of leader changes + description: Etcd leader changed more than 3 times during last hour + query: "increase(etcd_server_leader_changes_seen_total[1h]) > 3" + severity: warning + - name: Etcd high number of failed GRPC requests + description: More than 1% GRPC request failure detected in Etcd for 5 minutes + query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.01' + severity: warning + - name: Etcd high number of failed GRPC requests + description: More than 5% GRPC request failure detected in Etcd for 5 minutes + query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.05' + severity: error + - name: Etcd GRPC requests slow + description: GRPC requests slowing down, 99th percentil is over 0.15s for 5 minutes + query: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15' + severity: warning + - name: Etcd high number of failed HTTP requests + description: More than 1% HTTP failure detected in Etcd for 5 minutes + query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.01" + severity: warning + - name: Etcd high number of failed HTTP requests + description: More than 5% HTTP failure detected in Etcd for 5 minutes + query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.05" + severity: error + - name: Etcd HTTP requests slow + description: HTTP requests slowing down, 99th percentil is over 0.15s for 5 minutes + query: "histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15" + severity: warning + - name: Etcd member communication slow + description: Etcd member communication slowing down, 99th percentil is over 0.15s for 5 minutes + query: "histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) > 0.15" + severity: warning + - name: Etcd high number of failed proposals + description: Etcd server got more than 5 failed proposals past hour + query: "increase(etcd_server_proposals_failed_total[1h]) > 5" + severity: warning + - name: Etcd high fsync durations + description: Etcd WAL fsync duration increasing, 99th percentil is over 0.5s for 5 minutes + query: "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 0.5" + severity: warning + - name: Etcd high commit durations + description: Etcd commit duration increasing, 99th percentil is over 0.25s for 5 minutes + query: "histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) > 0.25" + severity: warning - - name: CoreDNS - exporters: - - name: Embedded exporter - rules: - - name: CoreDNS Panic Count - description: Number of CoreDNS panics encountered - query: "increase(coredns_panic_count_total[10m]) > 0" - severity: error + - name: Linkerd + exporters: + - rules: + + - name: Istio + exporters: + - rules: + + + - name: Network and storage + services: + - name: ZFS + exporters: + - name: node-exporter + doc_url: https://github.com/prometheus/node_exporter + rules: + + - name: OpenEBS + exporters: + - name: Embedded exporter + rules: + - name: OpenEBS used pool capacity + description: 'OpenEBS Pool use more than 80% of his capacity\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' + query: "(openebs_used_pool_capacity_percent) > 80" + severity: warning + + - name: Minio + exporters: + - name: Embedded exporter + rules: + - name: Minio disk offline + description: 'Minio disk is offline' + query: "minio_offline_disks > 0" + severity: error + - name: Minio storage space exhausted + description: 'Minio storage space is low (< 10 GB)' + query: "minio_disk_storage_free_bytes / 1024 / 1024 / 1024 < 10" + severity: warning + + - name: Juniper + exporters: + - name: czerwonk/junos_exporter + doc_url: https://github.com/czerwonk/junos_exporter + rules: + - name: Juniper switch down + description: The switch appears to be down + query: junos_up == 0 + severity: error + - name: Juniper high Bandwith Usage 1GiB + description: Interface is highly saturated for at least 1 min. (> 0.90GiB/s) + query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.90" + severity: error + - name: Juniper high Bandwith Usage 1GiB + description: Interface is getting saturated for at least 1 min. (> 0.80GiB/s) + query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.80" + severity: warning + + - name: CoreDNS + exporters: + - name: Embedded exporter + rules: + - name: CoreDNS Panic Count + description: Number of CoreDNS panics encountered + query: "increase(coredns_panic_count_total[10m]) > 0" + severity: error + + + - name: Other + services: + - name: Thanos + exporters: + - rules: + - name: Thanos compaction halted + description: Thanos compaction has failed to run and is now halted. + query: 'thanos_compactor_halted == 1' + severity: error + - name: Thanos compact bucket operation failure + description: Thanos compaction has failing storage operations + query: 'rate(thanos_objstore_bucket_operation_failures_total[1m]) > 0' + severity: error + - name: Thanos compact not run + description: Thanos compaction has not run in 24 hours. + query: '(time() - thanos_objstore_bucket_last_successful_upload_time) > 24*60*60' + severity: error diff --git a/index.md b/index.md index 79cd5e2..4b9f137 100644 --- a/index.md +++ b/index.md @@ -24,11 +24,27 @@
        - {% for service in site.data.rules.services %} -
      • - - {{ service.name }} - + {% for group in site.data.rules.groups %} +
      • + {% assign nbrRules = 0 %} + {% for service in group.services %} + {% for exporter in service.exporters %} + {% for rule in exporter.rules %} + {% assign nbrRules = nbrRules | plus: 1 %} + {% endfor %} + {% endfor %} + {% endfor %} + +

        {{ group.name }} ({{ nbrRules }} rules)

        +
        • {% endfor %}
      \ No newline at end of file diff --git a/rules.md b/rules.md index ef33915..6225fd9 100644 --- a/rules.md +++ b/rules.md @@ -19,78 +19,83 @@

      +

      +
        - {% for service in site.data.rules.services %} - {% assign serviceIndex = forloop.index %} - {% for exporter in service.exporters %} - {% assign nbrRules = exporter.rules | size %} -
      • -

        - {{ serviceIndex }}. - {{ service.name }} - {% if exporter.name %} - : - {% if exporter.doc_url %} - + {% for group in site.data.rules.groups %} + {% for service in group.services %} + {% assign serviceIndex = forloop.index %} + {% for exporter in service.exporters %} + {% assign nbrRules = exporter.rules | size %} +
      • +

        + {{ serviceIndex }}. + {{ service.name }} + {% if exporter.name %}: + {% if exporter.doc_url %} + + {{ exporter.name }} + + {% else %} {{ exporter.name }} - - {% else %} - {{ exporter.name }} - {% endif %} + {% endif %} + {% endif %} + + {% if nbrRules > 0 %} + + ({{ nbrRules }} rules) + + [copy all] + {% endif %} +

        + + {% if nbrRules == 0 %} + {% highlight javascript %} + // @TODO: Please contribute => https://github.com/samber/awesome-prometheus-alerts 👋 + {% endhighlight %} {% endif %} - {% if nbrRules > 0 %} - [copy all] - {% endif %} -

        • +
          + {% for rule in exporter.rules %} + {% assign ruleIndex = forloop.index %} + {% assign comments = rule.comments | strip | newline_to_br | split: '
          ' %} +
        • +

          + {{ serviceIndex }}.{{ ruleIndex }}. + {{ rule.name }} +

          +
          + + {{ rule.description }} + [copy] + +

          + {% assign ruleName = rule.name | split: ' ' %} + {% capture ruleNameCamelcase %}{% for word in ruleName %}{{ word | capitalize }} {% endfor %}{% endcapture %} - {% if nbrRules == 0 %} -{% highlight javascript %} -// @TODO: Please contribute => https://github.com/samber/awesome-prometheus-alerts 👋 -{% endhighlight %} - {% endif %} - -

            - {% for rule in exporter.rules %} - {% assign ruleIndex = forloop.index %} - {% assign comments = rule.comments | strip | newline_to_br | split: '
            ' %} -
          • -

            - {{ serviceIndex }}.{{ ruleIndex }}. - {{ rule.name }} -

            -
            - - {{ rule.description }} - [copy] - -

            - {% assign ruleName = rule.name | split: ' ' %} - {% capture ruleNameCamelcase %}{% for word in ruleName %}{{ word | capitalize }} {% endfor %}{% endcapture %} - -{% highlight yaml %} -{% for comment in comments %}# {{ comment | strip }} -{% endfor %} -- alert: {{ ruleNameCamelcase | remove: ' ' }} - expr: {{ rule.query }} - for: 5m - labels: - severity: {{ rule.severity }} - annotations: - summary: "{{ rule.name }} (instance {% raw %}{{ $labels.instance }}{% endraw %})" - description: "{{ rule.description }}\n VALUE = {% raw %}{{ $value }}{% endraw %}\n LABELS: {% raw %}{{ $labels }}{% endraw %}" + {% highlight yaml %} + {% for comment in comments %}# {{ comment | strip }} + {% endfor %}- alert: {{ ruleNameCamelcase | remove: ' ' }} + expr: {{ rule.query }} + for: 5m + labels: + severity: {{ rule.severity }} + annotations: + summary: "{{ rule.name }} (instance {% raw %}{{ $labels.instance }}{% endraw %})" + description: "{{ rule.description }}\n VALUE = {% raw %}{{ $value }}{% endraw %}\n LABELS: {% raw %}{{ $labels }}{% endraw %}" {% endhighlight %} -

            -
            -
            -
            • - {% endfor %} -
          +

          +
          +
          +
          • + {% endfor %} +
        -
        -
        • - {% endfor %} +
        + + {% endfor %} + {% endfor %} {% endfor %}
      From f554b72671426d679dea31830847392553351540 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Mon, 9 Mar 2020 21:55:17 +0100 Subject: [PATCH 065/126] Add alert for kubernetes api latency --- _data/rules.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 9ad54d2..e2fe678 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -963,6 +963,11 @@ groups: description: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. query: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60' severity: error + - name: Kubernetes API server latency + description: 'Kubernetes API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.' + query: 'histogram_quantile(0.99, sum(apiserver_request_latencies_bucket{verb!~"CONNECT|WATCHLIST|WATCH|PROXY"}) WITHOUT (instance, resource)) / 1e+06 > 1' + severity: warning + - name: Nomad exporters: From 5b457b0e52e0c3214d54f68915237d0734e39f9c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Mon, 9 Mar 2020 23:31:27 +0100 Subject: [PATCH 066/126] adding github buttons to layout --- _data/rules.yml | 4 + _layouts/default.html | 184 ++++++++++++++++++++++++++++-------------- 2 files changed, 126 insertions(+), 62 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index e2fe678..681ecc6 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -462,6 +462,10 @@ groups: - name: MongoDB exporters: + - name: percona/mongodb_exporter + doc_url: https://github.com/percona/mongodb_exporter + rules: + - name: dcu/mongodb_exporter doc_url: https://github.com/dcu/mongodb_exporter rules: diff --git a/_layouts/default.html b/_layouts/default.html index d2cd43e..dae9e04 100644 --- a/_layouts/default.html +++ b/_layouts/default.html @@ -1,78 +1,138 @@ - - - {% seo %} - - - - - - - - - - - - - + + + - - - - + } - + + + -
      - {{ content }} - - -
      - - + \ No newline at end of file From f5bcac33fe80ab884d29acf38b8973512bcb6d43 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Tue, 10 Mar 2020 10:01:08 +0100 Subject: [PATCH 067/126] better contributing guidelines --- CONTRIBUTING.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 001d352..1fcb24b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -3,6 +3,8 @@ ## Adding alerting rule +If you don't have time to write a PR, just copy and paste some alerts into an issue. We will format it accordingly. + Rules are here: `_data/rules.yml`. ### Guidelines From c82df5d0058e156b1afa7250cce4d56bb2d3a9fb Mon Sep 17 00:00:00 2001 From: Alexander Knipping Date: Tue, 17 Mar 2020 15:14:40 +0100 Subject: [PATCH 068/126] Fix PrometheusRuleEvaluationSlow Fixes the rule PrometheusRuleEvaluationSlow as it should fire if prometheus_rule_group_last_duration_seconds takes longer than prometheus_rule_group_interval_seconds. prometheus_rule_group_last_duration_seconds: The duration of the last rule group evaluation. prometheus_rule_group_interval_seconds: The interval of a rule group. --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 681ecc6..e731e16 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -38,7 +38,7 @@ groups: severity: error - name: Prometheus rule evaluation slow description: 'Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query.' - query: 'prometheus_rule_group_last_duration_seconds < prometheus_rule_group_interval_seconds' + query: 'prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds' severity: warning - name: Prometheus notifications backlog description: The Prometheus notification queue has not been empty for 10 minutes From 5125c683c54013258abe93ca179035ea70be0445 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Tue, 17 Mar 2020 18:50:08 +0100 Subject: [PATCH 069/126] adding alerts for Ceph --- _data/rules.yml | 60 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 681ecc6..7ae35ce 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -185,7 +185,7 @@ groups: - name: Docker containers exporters: - - name: cAdvisor + - name: google/cAdvisor doc_url: https://github.com/google/cadvisor rules: - name: Container killed @@ -1063,6 +1063,64 @@ groups: - name: Network and storage services: + - name: Ceph + exporters: + - name: Embedded exporter + doc_url: https://docs.ceph.com/docs/luminous/mgr/prometheus/ + rules: + - name: Ceph State + description: Ceph instance unhealthy + query: 'ceph_health_status != 0' + severity: error + - name: Ceph monitor clock skew + description: Ceph monitor clock skew detected. Please check ntp and hardware clock settings + query: 'abs(ceph_monitor_clock_skew_seconds) > 0.2' + severity: warning + - name: Ceph monitor low space + description: Ceph monitor storage is low. + query: 'ceph_monitor_avail_percent < 10' + severity: warning + - name: Ceph OSD Down + description: Ceph Object Storage Daemon Down + query: 'ceph_osd_up == 0' + severity: error + - name: Ceph high OSD latency + description: "Ceph Object Storage Daemon latetncy is high. Please check if it doesn't stuck in weird state." + query: 'ceph_osd_perf_apply_latency_seconds > 10' + severity: warning + - name: Ceph OSD low space + description: Ceph Object Storage Daemon is going out of space. Please add more disks. + query: ceph_osd_utilization > 90 + severity: warning + - name: Ceph OSD reweighted + description: Ceph Object Storage Daemon take ttoo much time to resize. + query: 'ceph_osd_weight < 1' + severity: warning + - name: Ceph PG down + description: Some Ceph placement groups are down. Please ensure that all the data are available. + query: 'ceph_pg_down > 0' + severity: error + - name: Ceph PG incomplete + description: Some Ceph placement groups are incomplete. Please ensure that all the data are available. + query: 'ceph_pg_incomplete > 0' + severity: error + - name: Ceph PG inconsistant + description: Some Ceph placement groups are inconsitent. Data is available but inconsistent across nodes. + query: ceph_pg_inconsistent > 0 + severity: warning + - name: Ceph PG activation long + description: Some Ceph placement groups are too long to activate. + query: 'ceph_pg_activating > 0' + severity: warning + - name: Ceph PG backfill full + description: Some Ceph placement groups are located on full Object Storage Daemon on cluster. Those PGs can be unavailable shortly. Please check OSDs, change weight or reconfigure CRUSH rules. + query: 'ceph_pg_backfill_toofull > 0' + severity: warning + - name: Ceph PG unavailable + description: Some Ceph placement groups are unavailable. + query: 'ceph_pg_total - ceph_pg_active > 0 + severity: error + - name: ZFS exporters: - name: node-exporter From 778e1010307cf62824f442c0f22696773259ae4d Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Tue, 17 Mar 2020 18:50:36 +0100 Subject: [PATCH 070/126] adding alerts for Ceph --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 6fa24b3..b59736b 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ Collection available here: **[https://awesome-prometheus-alerts.grep.to](https:/ #### Network and storage +- [Ceph](https://awesome-prometheus-alerts.grep.to/rules#ceph) - [ZFS](https://awesome-prometheus-alerts.grep.to/rules#zfs) - [OpenEBS](https://awesome-prometheus-alerts.grep.to/rules#openebs) - [Minio](https://awesome-prometheus-alerts.grep.to/rules#minio) From c653b37e157ac57fe5427770e06a8177792ca3cc Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Tue, 17 Mar 2020 20:56:49 +0100 Subject: [PATCH 071/126] adding rules to prometheus self monitoring --- _data/rules.yml | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 3f80d7d..da449ea 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -4,6 +4,18 @@ groups: - name: Prometheus self-monitoring exporters: - rules: + - name: Prometheus job missing + description: A Prometheus job has disappeared + query: 'absent(up{job="my-job"})' + severity: warning + - name: Prometheus target missing + description: A Prometheus target has disappeared. An exporter might be crashed. + query: 'up == 0' + severity: error + - name: Prometheus all targets missing + description: A Prometheus job does not have living target anymore. + query: 'count by (job) (up) == 0' + severity: error - name: Prometheus configuration reload failure description: Prometheus configuration reload error query: 'prometheus_config_last_reload_successful != 1' @@ -16,6 +28,10 @@ groups: description: AlertManager configuration reload error query: 'alertmanager_config_last_reload_successful != 1' severity: warning + - name: Prometheus AlertManager config not synced + description: Configurations of AlertManager cluster instances are out of sync + query: 'count(count_values("config_hash", alertmanager_config_hash)) > 1' + severity: warning - name: Prometheus AlertManager E2E dead man snitch description: Prometheus DeadManSnitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager. query: 'vector(1)' @@ -24,10 +40,6 @@ groups: description: Prometheus cannot connect the alertmanager query: "prometheus_notifications_alertmanagers_discovered < 1" severity: error - - name: Prometheus Exporter down - description: Prometheus exporter down - query: "up == 0" - severity: error - name: Prometheus rule evaluation failures description: 'Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.' query: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0' @@ -60,6 +72,10 @@ groups: description: Prometheus has many scrapes that exceed the sample limit query: 'increase(prometheus_target_scrapes_exceeded_sample_limit_total[10m]) > 10' severity: warning + - name: Prometheus target scrape duplicate + description: Prometheus has many samples rejected due to duplicate timestamps but different values + query: 'increase(prometheus_target_scrapes_sample_duplicate_timestamp_total[5m]) > 0' + severity: warning - name: Prometheus TSDB checkpoint creation failures description: 'Prometheus encountered {{ $value }} checkpoint creation failures' query: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0' @@ -266,15 +282,15 @@ groups: severity: error - name: Windows Server CPU Usage description: CPU Usage is more than 80% - query: '100 - (avg by (instance) (irate(wmi_cpu_time_total{mode="idle"}[2m])) * 100) > 80' + query: '100 - (avg by (instance) (rate(wmi_cpu_time_total{mode="idle"}[2m])) * 100) > 80' severity: warning - name: Windows Server memory Usage - description: Memory Usage is more than 90% - query: "100*(wmi_os_physical_memory_free_bytes) / wmi_cs_physical_memory_bytes > 90" + description: Memory usage is more than 90% + query: "100 * (wmi_os_physical_memory_free_bytes) / wmi_cs_physical_memory_bytes > 90" severity: warning - name: Windows Server disk Space Usage - description: Disk Space on Drive is used more than 80% - query: "100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{} / 1024 / 1024)) > 80" + description: Disk usage is more than 80% + query: "100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{} / 1024 / 1024)) > 80" severity: error From 2ecdb636b2ab599aa9e101de1d89a688dcdbc734 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Tue, 17 Mar 2020 21:08:09 +0100 Subject: [PATCH 072/126] oops --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index da449ea..b2801b0 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1134,7 +1134,7 @@ groups: severity: warning - name: Ceph PG unavailable description: Some Ceph placement groups are unavailable. - query: 'ceph_pg_total - ceph_pg_active > 0 + query: 'ceph_pg_total - ceph_pg_active > 0' severity: error - name: ZFS From 07dde61116f8478ff388babd0e6a225c81d6949e Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Tue, 17 Mar 2020 21:19:58 +0100 Subject: [PATCH 073/126] elasticsearch: adding disk watermark alerts --- _data/rules.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index b2801b0..2deef6c 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -593,6 +593,14 @@ groups: description: "The heap usage is over 80% for 5m" query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80' severity: warning + - name: Elasticsearch disk space low + description: The disk usage is over 80% + query: 'elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 20' + severity: warning + - name: Elasticsearch disk out of space + description: The disk usage is over 90% + query: 'elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10' + severity: error - name: Elasticsearch Cluster Red description: Elastic Cluster Red status query: 'elasticsearch_cluster_health_status{color="red"} == 1' From 34e62cb3275cb7dd9dd851518013f5484b293f2b Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Tue, 17 Mar 2020 22:26:46 +0100 Subject: [PATCH 074/126] nginx: adding latency metric --- _data/rules.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 2deef6c..0c8dfd2 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -732,6 +732,10 @@ groups: description: Too many HTTP requests with status 5xx (> 5%) query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' severity: error + - name: Nginx latency high + description: Nginx p99 latency is higher than 10 seconds + query: 'histogram_quantile(0.99, sum(rate(nginx_http_request_duration_seconds_bucket[30m])) by (host, node)) > 10' + severity: warning - name: Apache exporters: From caaea2eeb7df37cfad9c4f4ef5333b78ea1d4773 Mon Sep 17 00:00:00 2001 From: Alexander Knipping Date: Wed, 18 Mar 2020 15:21:38 +0100 Subject: [PATCH 075/126] Fix typo in DeadManSwitch alert Rename it from snitch into switch. --- _data/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 0c8dfd2..812c630 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -32,8 +32,8 @@ groups: description: Configurations of AlertManager cluster instances are out of sync query: 'count(count_values("config_hash", alertmanager_config_hash)) > 1' severity: warning - - name: Prometheus AlertManager E2E dead man snitch - description: Prometheus DeadManSnitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager. + - name: Prometheus AlertManager E2E dead man switch + description: Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager. query: 'vector(1)' severity: error - name: Prometheus not connected to alertmanager From 3d41e2b3cada00ff9a13ed2f6d959f74db8c1324 Mon Sep 17 00:00:00 2001 From: luhellma Date: Fri, 20 Mar 2020 15:08:13 +0100 Subject: [PATCH 076/126] Add rules for apache --- _data/rules.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 812c630..664efc2 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -742,6 +742,18 @@ groups: - name: Lusitaniae/apache_exporter doc_url: https://github.com/Lusitaniae/apache_exporter rules: + - name: Apache down + description: Apache down + query: 'apache_up = 0' + severity: error + - name: Apache workers load + description: Apache workers in busy state approach the max workers count 80% workers busy on {{ $labels.instance }} + query: '(sum by (instance) (apache_workers{state="busy"}) / sum by (instance) (apache_scoreboard) ) * 100 > 80' + severity: error + - name: Apache restart + description: Apache has just been restarted, less than one minute ago. + query: 'apache_uptime_seconds_total / 60 < 1' + severity: warning - name: HaProxy exporters: From a4fc086b9aedbf2a6905e8961159a7e832918896 Mon Sep 17 00:00:00 2001 From: luhellma Date: Fri, 20 Mar 2020 15:22:20 +0100 Subject: [PATCH 077/126] fix wrong number of equal sign in query --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 664efc2..38dae95 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -744,7 +744,7 @@ groups: rules: - name: Apache down description: Apache down - query: 'apache_up = 0' + query: 'apache_up == 0' severity: error - name: Apache workers load description: Apache workers in busy state approach the max workers count 80% workers busy on {{ $labels.instance }} From 5d8f911d975c86dfaa3f73390c6939c46843f9f8 Mon Sep 17 00:00:00 2001 From: luhellma Date: Wed, 25 Mar 2020 11:57:29 +0100 Subject: [PATCH 078/126] feat: Add new rules for MySQLd_exporter from prometheus --- _data/rules.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 38dae95..764144c 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -301,6 +301,22 @@ groups: - name: prometheus/mysqld_exporter doc_url: https://github.com/prometheus/mysqld_exporter rules: + - name: MySQL down + description: MySQL down on {{ $labels.instance }} + query: 'mysql_up == 0' + severity: error + - name: MySQL connections number is high + description: MySQL connections numbers approach the maximum number of connexion possible : more than 80% of connection are in use on {{ $labels.instance }} + query: '(mysql_global_status_threads_connected / mysql_global_variables_max_connections) * 100 > 80' + severity: error + - name: MySQL slow query count + description: MySQL server is having some slow query. The count is higher than 0 on {{ $labels.instance }} + query: 'rate(mysql_global_status_slow_queries[5m]) > 0' + severity: warning + - name: MySQL restart + description: MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}. + query: 'mysql_global_status_uptime < 60' + severity: warning - name: PostgreSQL exporters: From 5559e0140b047c7afb3142307ea6442cb5de9b70 Mon Sep 17 00:00:00 2001 From: luhellma Date: Wed, 25 Mar 2020 16:34:04 +0100 Subject: [PATCH 079/126] fix: double usage in query and alert configuration --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 764144c..db81abb 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -311,7 +311,7 @@ groups: severity: error - name: MySQL slow query count description: MySQL server is having some slow query. The count is higher than 0 on {{ $labels.instance }} - query: 'rate(mysql_global_status_slow_queries[5m]) > 0' + query: '(mysql_global_status_slow_queries > 0' severity: warning - name: MySQL restart description: MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}. From 329583ac36f16675a0a32d5c05780fe98a36e45c Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Wed, 25 Mar 2020 16:44:49 +0100 Subject: [PATCH 080/126] Fix typo and make pg and mysql similar --- _data/rules.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index db81abb..ae80cfe 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -302,18 +302,18 @@ groups: doc_url: https://github.com/prometheus/mysqld_exporter rules: - name: MySQL down - description: MySQL down on {{ $labels.instance }} + description: MySQL instance is down on {{ $labels.instance }} query: 'mysql_up == 0' - severity: error - - name: MySQL connections number is high - description: MySQL connections numbers approach the maximum number of connexion possible : more than 80% of connection are in use on {{ $labels.instance }} - query: '(mysql_global_status_threads_connected / mysql_global_variables_max_connections) * 100 > 80' severity: error - - name: MySQL slow query count - description: MySQL server is having some slow query. The count is higher than 0 on {{ $labels.instance }} - query: '(mysql_global_status_slow_queries > 0' - severity: warning - - name: MySQL restart + - name: MySQL too many connections + description: 'More than 80% of MySQL connections are in use on {{ $labels.instance }}' + query: '(mysql_global_status_threads_connected / mysql_global_variables_max_connections) * 100 > 80' + severity: warning + - name: MySQL slow queries + description: MySQL server is having some slow queries. + query: 'mysql_global_status_slow_queries > 0' + severity: warning + - name: MySQL restarted description: MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}. query: 'mysql_global_status_uptime < 60' severity: warning @@ -360,8 +360,8 @@ groups: query: 'rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0' severity: warning - name: Postgresql slow queries - description: PostgreSQL executes slow queries (> 1min) - query: 'rate(pg_slow_queries[1m]) * 60 > 10' + description: PostgreSQL executes slow queries + query: 'pg_slow_queries > 0' severity: warning - name: Postgresql high rollback rate description: Ratio of transactions being aborted compared to committed is > 2 % @@ -761,7 +761,7 @@ groups: - name: Apache down description: Apache down query: 'apache_up == 0' - severity: error + severity: error - name: Apache workers load description: Apache workers in busy state approach the max workers count 80% workers busy on {{ $labels.instance }} query: '(sum by (instance) (apache_workers{state="busy"}) / sum by (instance) (apache_scoreboard) ) * 100 > 80' From 2cda73aa3acd4ccb17e5c919074ad3b70e8b3670 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Thu, 26 Mar 2020 16:19:26 +0100 Subject: [PATCH 081/126] fix(kubernetes): min_over_time takes a time range as paremeter --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index ae80cfe..4ec03cb 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -961,7 +961,7 @@ groups: severity: warning - name: Kubernetes Pod not healthy description: Pod has been in a non-ready state for longer than an hour. - query: 'min_over_time(sum by (namespace, pod, env, stage) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h]) > 0' + query: 'min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:])' severity: error - name: Kubernetes pod crash looping description: Pod {{ $labels.pod }} is crash looping From d9286f6c3969b42b07a87289e789880d648e7da7 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sat, 28 Mar 2020 15:12:21 +0100 Subject: [PATCH 082/126] doc: add instructions to rules yaml file --- _data/rules.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 4ec03cb..8ad6eb5 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1,3 +1,12 @@ + +# +# The following yaml cannot be copy-pasted to Prometheus configuration. +# Please navigate to https://awesome-prometheus-alerts.grep.to/rules instead. +# +# Contributing guidelines: +# https://github.com/samber/awesome-prometheus-alerts/blob/master/CONTRIBUTING.md +# + groups: - name: Basic resource monitoring services: From 486025036040c4a8bd5d87e82843021528343917 Mon Sep 17 00:00:00 2001 From: Matthias Crauwels Date: Mon, 30 Mar 2020 11:24:58 +0200 Subject: [PATCH 083/126] added some extra MySQL checks --- _data/rules.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 8ad6eb5..6de52eb 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -312,11 +312,27 @@ groups: rules: - name: MySQL down description: MySQL instance is down on {{ $labels.instance }} - query: 'mysql_up == 0' + query: 'avg by (instance) (mysql_up) == 0' severity: error - name: MySQL too many connections description: 'More than 80% of MySQL connections are in use on {{ $labels.instance }}' - query: '(mysql_global_status_threads_connected / mysql_global_variables_max_connections) * 100 > 80' + query: 'avg by(instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by(instance) (mysql_global_variables_max_connections) * 100 > 80' + severity: warning + - name: MySQL high threads running + description: 'More than 60% of MySQL connections are in running state on {{ $labels.instance }}' + query: 'avg by(instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by(instance) (mysql_global_variables_max_connections) * 100 > 60' + severity: warning + - name: MySQL Slave IO thread not running on {{ $instance.instance }} + description: 'MySQL Slave IO thread not running on {{ $labels.instance }}' + query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and avg by (instance) (mysql_slave_status_slave_io_running) == 0' + severity: error + - name: MySQL Slave SQL thread not running on {{ $instance.instance }} + description: 'MySQL Slave SQL thread not running on {{ $labels.instance }}' + query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and avg by (instance) (mysql_slave_status_slave_sql_running) == 0' + severity: error + - name: MySQL Slave replication lag {{ $instance.instance }} + description: 'MysqL replication lag on {{ $labels.instance }}' + query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and (avg by (instance) (max_over_time(mysql_slave_status_seconds_behind_master[1m])) - avg by (instance) (mysql_slave_status_sql_delay)) > 300' severity: warning - name: MySQL slow queries description: MySQL server is having some slow queries. From 79b5ad3b5d3a0921b614905a37c2aae775e9ba93 Mon Sep 17 00:00:00 2001 From: Matthias Crauwels Date: Tue, 31 Mar 2020 11:42:05 +0200 Subject: [PATCH 084/126] removed avg grouping where possible --- _data/rules.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 6de52eb..396b8e7 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -312,27 +312,27 @@ groups: rules: - name: MySQL down description: MySQL instance is down on {{ $labels.instance }} - query: 'avg by (instance) (mysql_up) == 0' + query: 'mysql_up == 0' severity: error - name: MySQL too many connections description: 'More than 80% of MySQL connections are in use on {{ $labels.instance }}' - query: 'avg by(instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by(instance) (mysql_global_variables_max_connections) * 100 > 80' + query: 'avg by (instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 80' severity: warning - name: MySQL high threads running description: 'More than 60% of MySQL connections are in running state on {{ $labels.instance }}' - query: 'avg by(instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by(instance) (mysql_global_variables_max_connections) * 100 > 60' + query: 'avg by (instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 60' severity: warning - name: MySQL Slave IO thread not running on {{ $instance.instance }} description: 'MySQL Slave IO thread not running on {{ $labels.instance }}' - query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and avg by (instance) (mysql_slave_status_slave_io_running) == 0' + query: 'mysql_slave_status_master_server_id > 0 and mysql_slave_status_slave_io_running == 0' severity: error - name: MySQL Slave SQL thread not running on {{ $instance.instance }} description: 'MySQL Slave SQL thread not running on {{ $labels.instance }}' - query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and avg by (instance) (mysql_slave_status_slave_sql_running) == 0' + query: 'mysql_slave_status_master_server_id > 0 and mysql_slave_status_slave_sql_running == 0' severity: error - name: MySQL Slave replication lag {{ $instance.instance }} description: 'MysqL replication lag on {{ $labels.instance }}' - query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and (avg by (instance) (max_over_time(mysql_slave_status_seconds_behind_master[1m])) - avg by (instance) (mysql_slave_status_sql_delay)) > 300' + query: 'mysql_slave_status_master_server_id > 0 and (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300' severity: warning - name: MySQL slow queries description: MySQL server is having some slow queries. From c20227b458f521c5b3b4cfb2c71d5a93428ef540 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Tue, 31 Mar 2020 16:02:28 +0200 Subject: [PATCH 085/126] oops: adding one-to-one vector matching to mysql subqueries --- _data/rules.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 396b8e7..7b3d427 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -324,15 +324,15 @@ groups: severity: warning - name: MySQL Slave IO thread not running on {{ $instance.instance }} description: 'MySQL Slave IO thread not running on {{ $labels.instance }}' - query: 'mysql_slave_status_master_server_id > 0 and mysql_slave_status_slave_io_running == 0' + query: 'mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0' severity: error - name: MySQL Slave SQL thread not running on {{ $instance.instance }} description: 'MySQL Slave SQL thread not running on {{ $labels.instance }}' - query: 'mysql_slave_status_master_server_id > 0 and mysql_slave_status_slave_sql_running == 0' + query: 'mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0' severity: error - name: MySQL Slave replication lag {{ $instance.instance }} description: 'MysqL replication lag on {{ $labels.instance }}' - query: 'mysql_slave_status_master_server_id > 0 and (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300' + query: 'mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300' severity: warning - name: MySQL slow queries description: MySQL server is having some slow queries. From c98a04784e47115475b884f638a67577daaa6ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sel=C3=A7uk=20Ar=C4=B1bal=C4=B1?= Date: Thu, 2 Apr 2020 21:01:04 +0300 Subject: [PATCH 086/126] FIX KubernetesPodnothealthy Alert Kube state metrics assigns value of current pod phase with 1, so according to that Kubernetes Pod not healthy fixed. --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 7b3d427..a5ff36e 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -986,7 +986,7 @@ groups: severity: warning - name: Kubernetes Pod not healthy description: Pod has been in a non-ready state for longer than an hour. - query: 'min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:])' + query: 'min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"} == 1)[1h:])' severity: error - name: Kubernetes pod crash looping description: Pod {{ $labels.pod }} is crash looping From e97023d2a446f2885f6b36bbeaa9eebbce2bbd87 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Mon, 6 Apr 2020 09:01:51 +0200 Subject: [PATCH 087/126] linkerd2: adding first rule --- _data/rules.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 7b3d427..25c4a2b 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1135,7 +1135,14 @@ groups: - name: Linkerd exporters: - - rules: + - name: Embedded exporter + doc_url: https://linkerd.io/2/tasks/exporting-metrics/ + rules: + - name: Linkerd high error rate + description: Linkerd error rate for {{ $labels.deployment | $labels.statefulset | $labels.daemonset }} is over 10% + query: 'sum(rate(request_errors_total[5m])) by (deployment, statefulset, daemonset) / sum(rate(request_total[5m])) by (deployment, statefulset, daemonset) * 100 > 10' + severity: warning + - name: Istio exporters: From c57a5e6e36d70ce54db39f149857d75ec4884c4a Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Thu, 30 Apr 2020 12:38:23 +0100 Subject: [PATCH 088/126] Add HostNetworkReceiveErrors and HostNetworkTransmitErrors rules --- _data/rules.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index d9003b9..b2af87a 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -207,7 +207,14 @@ groups: description: OOM kill detected query: 'increase(node_vmstat_oom_kill[30m]) > 1' severity: warning - + - name: Host Network Receive Errors + description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last five minutes.' + query: 'increase(node_network_receive_errs_total[5m])' + severity: warning + - name: Host Network Transmit Errors + description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last five minutes.' + query: 'increase(node_network_transmit_errs_total[5m])' + severity: warning - name: Docker containers exporters: - name: google/cAdvisor From f87e6d300d3aa07784640e4594208d46ccdf785b Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Thu, 30 Apr 2020 12:39:12 +0100 Subject: [PATCH 089/126] Added spacing as per standard --- _data/rules.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/_data/rules.yml b/_data/rules.yml index b2af87a..fdda241 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -215,6 +215,7 @@ groups: description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last five minutes.' query: 'increase(node_network_transmit_errs_total[5m])' severity: warning + - name: Docker containers exporters: - name: google/cAdvisor From 981e82d6490652943f87accb1525c0940adade7d Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Thu, 30 Apr 2020 13:27:30 +0100 Subject: [PATCH 090/126] Add HostEDACUncorrectableErrorsdetected and HostEDACCorrectableErrorsdetected rules --- _data/rules.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index d9003b9..2d715bb 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -207,6 +207,14 @@ groups: description: OOM kill detected query: 'increase(node_vmstat_oom_kill[30m]) > 1' severity: warning + - name: Host EDAC Correctable Errors detected + description: '{{ $labels.instance }} has had {{ printf "%.0f" $value }} correctable memory errors reported by EDAC in the last 5 minutes.' + query: 'increase(node_edac_correctable_errors_total[5m])' + severity: warning + - name: Host EDAC Uncorrectable Errors detected + description: '{{ $labels.instance }} has had {{ printf "%.0f" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.' + query: 'increase(node_edac_uncorrectable_errors_total[5m])' + severity: warning - name: Docker containers exporters: From d3d13946e60a33b19bbebc0d86f91957f478d253 Mon Sep 17 00:00:00 2001 From: Ondrej Zalesky Date: Thu, 30 Apr 2020 22:53:25 +0200 Subject: [PATCH 091/126] fix "Kubernetes Pod not healthy" query --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index d9003b9..b1f5847 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -986,7 +986,7 @@ groups: severity: warning - name: Kubernetes Pod not healthy description: Pod has been in a non-ready state for longer than an hour. - query: 'min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"} == 1)[1h:])' + query: 'min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:]) == 1' severity: error - name: Kubernetes pod crash looping description: Pod {{ $labels.pod }} is crash looping From 648b83250a94eeb4ccc3ccd1c8c0a36358dcc347 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 3 May 2020 18:01:25 +0200 Subject: [PATCH 092/126] improve accuracy "Kubernetes Pod not healthy" query --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index b1f5847..d491c7f 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -986,7 +986,7 @@ groups: severity: warning - name: Kubernetes Pod not healthy description: Pod has been in a non-ready state for longer than an hour. - query: 'min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:]) == 1' + query: 'min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:]) > 0' severity: error - name: Kubernetes pod crash looping description: Pod {{ $labels.pod }} is crash looping From 773b3456d254233e0ee854df19b75dc309b7837f Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 3 May 2020 21:40:45 +0200 Subject: [PATCH 093/126] renaming sms to pager --- alertmanager.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alertmanager.md b/alertmanager.md index 2d8595c..a4acb62 100644 --- a/alertmanager.md +++ b/alertmanager.md @@ -71,7 +71,7 @@ route: severity: error|warning continue: true - - receiver: "sms" + - receiver: "pager" group_wait: 10s match_re: severity: error @@ -85,7 +85,7 @@ receivers: channel: 'monitoring' text: "{{ range .Alerts }} {{ .Annotations.summary }}\n{{ .Annotations.description }}\n{{ end }}" - - name: "sms" + - name: "pager" webhook_config: - url: http://a.b.c.d:8080/send/sms send_resolved: true From 790139211ea3b1d9aeba54a3ab3fe3097ef04077 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 3 May 2020 23:23:21 +0200 Subject: [PATCH 094/126] fix typo: postgresql replication lag --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index d491c7f..c500425 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -362,7 +362,7 @@ groups: severity: warning - name: Postgresql replication lag description: PostgreSQL replication lag is going up (> 10s) - query: '(pg_replication_lag > 10 and ON(instance) (pg_replication_is_replica == 1)' + query: '(pg_replication_lag) > 10 and ON(instance) (pg_replication_is_replica == 1)' severity: warning - name: Postgresql table not vaccumed description: Table has not been vaccum for 24 hours From eb8dc736a3f136d4779621691966a51bc237bd85 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Mon, 4 May 2020 00:05:33 +0200 Subject: [PATCH 095/126] improve acuracy for context switching query --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index c500425..9313e0c 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -169,7 +169,7 @@ groups: severity: warning - name: Host context switching description: Context switching is growing on node (> 1000 / s) - query: "rate(node_context_switches_total[5m]) > 1000" + query: "(rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1000" severity: warning comments: | 1000 context switches is an arbitrary number. From 718cd2188c2e5c4f72f7e0025c1e898c739cfeff Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Mon, 4 May 2020 00:10:43 +0200 Subject: [PATCH 096/126] shame on me --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 9313e0c..a6b0254 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -169,7 +169,7 @@ groups: severity: warning - name: Host context switching description: Context switching is growing on node (> 1000 / s) - query: "(rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1000" + query: '(rate(node_context_switches_total[5m])) / (count without(cpu, mode) (node_cpu_seconds_total{mode="idle"})) > 1000' severity: warning comments: | 1000 context switches is an arbitrary number. From 4b22c078ea83c1ffc4bb91e3f8790b33bbb7df9e Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Mon, 4 May 2020 18:47:20 +0100 Subject: [PATCH 097/126] Align EDAC errors with comments --- _data/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 2d715bb..40262f5 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -210,10 +210,10 @@ groups: - name: Host EDAC Correctable Errors detected description: '{{ $labels.instance }} has had {{ printf "%.0f" $value }} correctable memory errors reported by EDAC in the last 5 minutes.' query: 'increase(node_edac_correctable_errors_total[5m])' - severity: warning + severity: info - name: Host EDAC Uncorrectable Errors detected description: '{{ $labels.instance }} has had {{ printf "%.0f" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.' - query: 'increase(node_edac_uncorrectable_errors_total[5m])' + query: 'node_edac_uncorrectable_errors_total > 1' severity: warning - name: Docker containers From 8912db93bc87772c5aba3d06ca5eb31959469d3f Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Mon, 4 May 2020 19:04:52 +0100 Subject: [PATCH 098/126] Fix "greater than" value --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 40262f5..6da8c7b 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -213,7 +213,7 @@ groups: severity: info - name: Host EDAC Uncorrectable Errors detected description: '{{ $labels.instance }} has had {{ printf "%.0f" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.' - query: 'node_edac_uncorrectable_errors_total > 1' + query: 'node_edac_uncorrectable_errors_total >= 1' severity: warning - name: Docker containers From 5d83e393cc2641188606555520b8148c15ee6c91 Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Fri, 8 May 2020 15:25:54 +0100 Subject: [PATCH 099/126] Add initial Speedtest Exporter rules --- _data/rules.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index d9003b9..bfc0b7d 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1209,6 +1209,18 @@ groups: query: 'ceph_pg_total - ceph_pg_active > 0' severity: error + - name: Speedtest exporter + doc_url: https://github.com/nlamirault/speedtest_exporter + rules: + - name: SpeedTest Slow Internet Download + description: Internet download speed is currently {{humanize $value}} Mbps. + query: 'avg_over_time(speedtest_download[30m]) < 75' + severity: warning + - name: SpeedTest Slow Internet Upload + description: Internet upload speed is currently {{humanize $value}} Mbps. + query: 'avg_over_time(speedtest_upload[30m]) < 20 ' + severity: warning + - name: ZFS exporters: - name: node-exporter From d5f6388899bd19fac884ac0153986cc33a11d625 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sat, 9 May 2020 02:11:18 +0200 Subject: [PATCH 100/126] renaming some mysql alerts --- _data/rules.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index a6b0254..571fbb7 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -322,15 +322,15 @@ groups: description: 'More than 60% of MySQL connections are in running state on {{ $labels.instance }}' query: 'avg by (instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 60' severity: warning - - name: MySQL Slave IO thread not running on {{ $instance.instance }} + - name: MySQL Slave IO thread not running description: 'MySQL Slave IO thread not running on {{ $labels.instance }}' query: 'mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0' severity: error - - name: MySQL Slave SQL thread not running on {{ $instance.instance }} + - name: MySQL Slave SQL thread not running description: 'MySQL Slave SQL thread not running on {{ $labels.instance }}' query: 'mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0' severity: error - - name: MySQL Slave replication lag {{ $instance.instance }} + - name: MySQL Slave replication lag description: 'MysqL replication lag on {{ $labels.instance }}' query: 'mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300' severity: warning From ee4e046c6613dd7241e0ec435ebc547f06583bb2 Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Sat, 9 May 2020 10:18:21 +0100 Subject: [PATCH 101/126] Add "> 0" at the end of NetworkTransmitErrors queries --- _data/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index fdda241..12a2eea 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -209,11 +209,11 @@ groups: severity: warning - name: Host Network Receive Errors description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} receive errors in the last five minutes.' - query: 'increase(node_network_receive_errs_total[5m])' + query: 'increase(node_network_receive_errs_total[5m]) > 0' severity: warning - name: Host Network Transmit Errors description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last five minutes.' - query: 'increase(node_network_transmit_errs_total[5m])' + query: 'increase(node_network_transmit_errs_total[5m]) > 0' severity: warning - name: Docker containers From 8faa2957454396ed3680962af75841902afbccfc Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Sat, 9 May 2020 10:20:55 +0100 Subject: [PATCH 102/126] Add SpeedTest stanza --- _data/rules.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index bfc0b7d..c7f708b 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1209,6 +1209,8 @@ groups: query: 'ceph_pg_total - ceph_pg_active > 0' severity: error + - name: SpeedTest + exporters: - name: Speedtest exporter doc_url: https://github.com/nlamirault/speedtest_exporter rules: From 660312d0ead07066db231231a635cf14a1c54790 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sat, 9 May 2020 21:25:13 +0200 Subject: [PATCH 103/126] fix OOM killer threshold --- _data/rules.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index bc5b237..c696bf8 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -205,15 +205,15 @@ groups: severity: warning - name: Host OOM kill detected description: OOM kill detected - query: 'increase(node_vmstat_oom_kill[30m]) > 1' + query: 'increase(node_vmstat_oom_kill[5m]) > 0' severity: warning - name: Host EDAC Correctable Errors detected description: '{{ $labels.instance }} has had {{ printf "%.0f" $value }} correctable memory errors reported by EDAC in the last 5 minutes.' - query: 'increase(node_edac_correctable_errors_total[5m])' + query: 'increase(node_edac_correctable_errors_total[5m]) > 0' severity: info - name: Host EDAC Uncorrectable Errors detected description: '{{ $labels.instance }} has had {{ printf "%.0f" $value }} uncorrectable memory errors reported by EDAC in the last 5 minutes.' - query: 'node_edac_uncorrectable_errors_total >= 1' + query: 'node_edac_uncorrectable_errors_total > 0' severity: warning - name: Docker containers From da1e4f63015442ce2ad9ea8b72a1dcfbabd1fdf4 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Thu, 14 May 2020 17:20:19 +0200 Subject: [PATCH 104/126] :lipstick: replacing "error" severity by "critical", repo wide --- _data/rules.yml | 254 ++++++++++++++++++++++++------------------------ alertmanager.md | 6 +- 2 files changed, 130 insertions(+), 130 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 0c1de75..c60ab90 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -20,11 +20,11 @@ groups: - name: Prometheus target missing description: A Prometheus target has disappeared. An exporter might be crashed. query: 'up == 0' - severity: error + severity: critical - name: Prometheus all targets missing description: A Prometheus job does not have living target anymore. query: 'count by (job) (up) == 0' - severity: error + severity: critical - name: Prometheus configuration reload failure description: Prometheus configuration reload error query: 'prometheus_config_last_reload_successful != 1' @@ -44,19 +44,19 @@ groups: - name: Prometheus AlertManager E2E dead man switch description: Prometheus DeadManSwitch is an always-firing alert. It's used as an end-to-end test of Prometheus through the Alertmanager. query: 'vector(1)' - severity: error + severity: critical - name: Prometheus not connected to alertmanager description: Prometheus cannot connect the alertmanager query: "prometheus_notifications_alertmanagers_discovered < 1" - severity: error + severity: critical - name: Prometheus rule evaluation failures description: 'Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.' query: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0' - severity: error + severity: critical - name: Prometheus template text expansion failures description: 'Prometheus encountered {{ $value }} template text expansion failures' query: 'increase(prometheus_template_text_expansion_failures_total[3m]) > 0' - severity: error + severity: critical - name: Prometheus rule evaluation slow description: 'Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query.' query: 'prometheus_rule_group_last_duration_seconds > prometheus_rule_group_interval_seconds' @@ -68,11 +68,11 @@ groups: - name: Prometheus AlertManager notification failing description: Alertmanager is failing sending notifications query: 'rate(alertmanager_notifications_failed_total[1m]) > 0' - severity: error + severity: critical - name: Prometheus target empty description: Prometheus has no target in service discovery query: 'prometheus_sd_discovered_targets == 0' - severity: error + severity: critical - name: Prometheus target scraping slow description: Prometheus is scraping exporters slowly query: 'prometheus_target_interval_length_seconds{quantile="0.9"} > 60' @@ -88,31 +88,31 @@ groups: - name: Prometheus TSDB checkpoint creation failures description: 'Prometheus encountered {{ $value }} checkpoint creation failures' query: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0' - severity: error + severity: critical - name: Prometheus TSDB checkpoint deletion failures description: 'Prometheus encountered {{ $value }} checkpoint deletion failures' query: 'increase(prometheus_tsdb_checkpoint_deletions_failed_total[3m]) > 0' - severity: error + severity: critical - name: Prometheus TSDB compactions failed description: 'Prometheus encountered {{ $value }} TSDB compactions failures' query: 'increase(prometheus_tsdb_compactions_failed_total[3m]) > 0' - severity: error + severity: critical - name: Prometheus TSDB head truncations failed description: 'Prometheus encountered {{ $value }} TSDB head truncation failures' query: 'increase(prometheus_tsdb_head_truncations_failed_total[3m]) > 0' - severity: error + severity: critical - name: Prometheus TSDB reload failures description: 'Prometheus encountered {{ $value }} TSDB reload failures' query: 'increase(prometheus_tsdb_reloads_failures_total[3m]) > 0' - severity: error + severity: critical - name: Prometheus TSDB WAL corruptions description: 'Prometheus encountered {{ $value }} TSDB WAL corruptions' query: 'increase(prometheus_tsdb_wal_corruptions_total[3m]) > 0' - severity: error + severity: critical - name: Prometheus TSDB WAL truncations failed description: 'Prometheus encountered {{ $value }} TSDB WAL truncation failures' query: 'increase(prometheus_tsdb_wal_truncations_failed_total[3m]) > 0' - severity: error + severity: critical - name: Host and hardware exporters: @@ -190,11 +190,11 @@ groups: - name: Host node overtemperature alarm description: "Physical node temperature alarm triggered" query: "node_hwmon_temp_alarm == 1" - severity: error + severity: critical - name: Host RAID array got inactive description: 'RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.' query: 'node_md_state{state="inactive"} > 0' - severity: error + severity: critical - name: Host RAID disk failure description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap' query: 'node_md_disks{state="fail"} > 0' @@ -223,7 +223,7 @@ groups: description: '{{ $labels.instance }} interface {{ $labels.device }} has encountered {{ printf "%.0f" $value }} transmit errors in the last five minutes.' query: 'increase(node_network_transmit_errs_total[5m]) > 0' severity: warning - + - name: Docker containers exporters: - name: google/cAdvisor @@ -262,7 +262,7 @@ groups: - name: Blackbox probe failed description: Probe failed query: probe_success == 0 - severity: error + severity: critical - name: Blackbox slow probe description: Blackbox probe took more than 1s to complete query: "avg_over_time(probe_duration_seconds[1m]) > 1" @@ -270,7 +270,7 @@ groups: - name: Blackbox probe HTTP failure description: HTTP status code is not 200-399 query: "probe_http_status_code <= 199 OR probe_http_status_code >= 400" - severity: error + severity: critical - name: Blackbox SSL certificate will expire soon description: SSL certificate expires in 30 days query: "probe_ssl_earliest_cert_expiry - time() < 86400 * 30" @@ -278,11 +278,11 @@ groups: - name: Blackbox SSL certificate will expire soon description: SSL certificate expires in 3 days query: "probe_ssl_earliest_cert_expiry - time() < 86400 * 3" - severity: error + severity: critical - name: Blackbox SSL certificate expired description: SSL certificate has expired already query: "probe_ssl_earliest_cert_expiry - time() <= 0" - severity: error + severity: critical - name: Blackbox probe slow HTTP description: HTTP request took more than 1s query: "avg_over_time(probe_http_duration_seconds[1m]) > 1" @@ -300,11 +300,11 @@ groups: - name: Windows Server collector Error description: "Collector {{ $labels.collector }} was not successful" query: "wmi_exporter_collector_success == 0" - severity: error + severity: critical - name: Windows Server service Status description: Windows Service state is not OK query: 'wmi_service_status{status="ok"} != 1' - severity: error + severity: critical - name: Windows Server CPU Usage description: CPU Usage is more than 80% query: '100 - (avg by (instance) (rate(wmi_cpu_time_total{mode="idle"}[2m])) * 100) > 80' @@ -316,7 +316,7 @@ groups: - name: Windows Server disk Space Usage description: Disk usage is more than 80% query: "100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{} / 1024 / 1024)) > 80" - severity: error + severity: critical - name: Databases and brokers @@ -329,7 +329,7 @@ groups: - name: MySQL down description: MySQL instance is down on {{ $labels.instance }} query: 'mysql_up == 0' - severity: error + severity: critical - name: MySQL too many connections description: 'More than 80% of MySQL connections are in use on {{ $labels.instance }}' query: 'avg by (instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 80' @@ -341,11 +341,11 @@ groups: - name: MySQL Slave IO thread not running description: 'MySQL Slave IO thread not running on {{ $labels.instance }}' query: 'mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_io_running == 0' - severity: error + severity: critical - name: MySQL Slave SQL thread not running description: 'MySQL Slave SQL thread not running on {{ $labels.instance }}' query: 'mysql_slave_status_master_server_id > 0 and ON (instance) mysql_slave_status_slave_sql_running == 0' - severity: error + severity: critical - name: MySQL Slave replication lag description: 'MysqL replication lag on {{ $labels.instance }}' query: 'mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300' @@ -367,11 +367,11 @@ groups: - name: Postgresql down description: Postgresql instance is down query: "pg_up == 0" - severity: error + severity: critical - name: Postgresql restarted description: Postgresql restarted query: "time() - pg_postmaster_start_time_seconds < 60" - severity: error + severity: critical - name: Postgresql exporter error description: Postgresql exporter is showing errors. A query may be buggy in query.yaml query: 'pg_exporter_last_scrape_error > 0' @@ -411,7 +411,7 @@ groups: - name: Postgresql commit rate low description: Postgres seems to be processing very few transactions query: 'rate(pg_stat_database_xact_commit[1m]) < 10' - severity: error + severity: critical - name: Postgresql low XID consumption description: Postgresql seems to be consuming transaction IDs very slowly query: 'rate(pg_txid_current[1m]) < 5' @@ -423,19 +423,19 @@ groups: - name: Postgresql WALE replication stopped description: WAL-E replication seems to be stopped query: 'rate(pg_xlog_position_bytes[1m]) == 0' - severity: error + severity: critical - name: Postgresql high rate statement timeout description: Postgres transactions showing high rate of statement timeouts query: 'rate(postgresql_errors_total{type="statement_timeout"}[5m]) > 3' - severity: error + severity: critical - name: Postgresql high rate deadlock description: Postgres detected deadlocks query: 'rate(postgresql_errors_total{type="deadlock_detected"}[1m]) * 60 > 1' - severity: error + severity: critical - name: Postgresql replication lab bytes description: Postgres Replication lag (in bytes) is high query: '(pg_xlog_position_bytes and pg_replication_is_replica == 0) - GROUP_RIGHT(instance) (pg_xlog_position_bytes and pg_replication_is_replica == 1) > 1e+09' - severity: error + severity: critical - name: Postgresql unused replication slot description: Unused Replication Slots query: 'pg_replication_slots_active == 0' @@ -447,7 +447,7 @@ groups: - name: Postgresql split brain description: Split Brain, too many primary Postgresql databases in read-write mode query: 'count(pg_replication_is_replica == 0) != 1' - severity: error + severity: critical - name: Postgresql promoted node description: Postgresql standby server has been promoted as primary node query: 'pg_replication_is_replica and changes(pg_replication_is_replica[1m]) > 0' @@ -459,11 +459,11 @@ groups: - name: Postgresql SSL compression active description: Database connections with SSL compression enabled. This may add significant jitter in replication delay. Replicas should turn off SSL compression via `sslcompression=0` in `recovery.conf`. query: 'sum(pg_stat_ssl_compression) > 0' - severity: error + severity: critical - name: Postgresql too many locks acquired description: Too many locks acquired on the database. If this alert happens frequently, we may need to increase the postgres setting max_locks_per_transaction. query: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20' - severity: error + severity: critical - name: PGBouncer exporters: @@ -481,7 +481,7 @@ groups: - name: PGBouncer max connections description: The number of PGBouncer client connections has reached max_client_conn. query: 'rate(pgbouncer_errors_count{errmsg="no more connections allowed (max_client_conn)"}[1m]) > 0' - severity: error + severity: critical - name: Redis exporters: @@ -491,31 +491,31 @@ groups: - name: Redis down description: Redis instance is down query: "redis_up == 0" - severity: error + severity: critical - name: Redis missing master description: Redis cluster has no node marked as master. query: 'count(redis_instance_info{role="master"}) == 0' - severity: error + severity: critical - name: Redis too many masters description: Redis cluster has too many nodes marked as master. query: 'count(redis_instance_info{role="master"}) > 1' - severity: error + severity: critical - name: Redis disconnected slaves description: Redis not replicating for all slaves. Consider reviewing the redis replication status. query: 'count without (instance, job) (redis_connected_slaves) - sum without (instance, job) (redis_connected_slaves) - 1 > 1' - severity: error + severity: critical - name: Redis replication broken description: Redis instance lost a slave query: "delta(redis_connected_slaves[1m]) < 0" - severity: error + severity: critical - name: Redis cluster flapping description: Changes have been detected in Redis replica connection. This can occur when replica nodes lose connection to the master and reconnect (a.k.a flapping). query: 'changes(redis_connected_slaves[5m]) > 2' - severity: error + severity: critical - name: Redis missing backup description: Redis has not been backuped for 24 hours query: "time() - redis_rdb_last_save_timestamp_seconds > 60 * 60 * 24" - severity: error + severity: critical - name: Redis out of memory description: Redis is running out of memory (> 90%) query: "redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90" @@ -531,7 +531,7 @@ groups: - name: Redis rejected connections description: Some connections to Redis has been rejected query: "increase(redis_rejected_connections_total[1m]) > 0" - severity: error + severity: critical - name: MongoDB exporters: @@ -545,31 +545,31 @@ groups: - name: MongoDB replication lag description: Mongodb replication lag is more than 10s query: 'avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}) > 10' - severity: error + severity: critical - name: MongoDB replication headroom description: MongoDB replication headroom is <= 0 query: '(avg(mongodb_replset_oplog_tail_timestamp - mongodb_replset_oplog_head_timestamp) - (avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}))) <= 0' - severity: error + severity: critical - name: MongoDB replication Status 3 description: MongoDB Replication set member either perform startup self-checks, or transition from completing a rollback or resync query: "mongodb_replset_member_state == 3" - severity: error + severity: critical - name: MongoDB replication Status 6 description: MongoDB Replication set member as seen from another member of the set, is not yet known query: "mongodb_replset_member_state == 6" - severity: error + severity: critical - name: MongoDB replication Status 8 description: MongoDB Replication set member as seen from another member of the set, is unreachable query: "mongodb_replset_member_state == 8" - severity: error + severity: critical - name: MongoDB replication Status 9 description: MongoDB Replication set member is actively performing a rollback. Data is not available for reads query: "mongodb_replset_member_state == 9" - severity: error + severity: critical - name: MongoDB replication Status 10 description: MongoDB Replication set member was once in a replica set but was subsequently removed query: "mongodb_replset_member_state == 10" - severity: error + severity: critical - name: MongoDB number cursors open description: Too many cursors opened by MongoDB for clients (> 10k) query: 'mongodb_metrics_cursor_open{state="total_open"} > 10000' @@ -595,15 +595,15 @@ groups: - name: Rabbitmq down description: RabbitMQ node down query: "rabbitmq_up == 0" - severity: error + severity: critical - name: Rabbitmq cluster down description: Less than 3 nodes running in RabbitMQ cluster query: "sum(rabbitmq_running) < 3" - severity: error + severity: critical - name: Rabbitmq cluster partition description: Cluster partition query: "rabbitmq_partitions > 0" - severity: error + severity: critical - name: Rabbitmq out of memory description: Memory available for RabbmitMQ is low (< 10%) query: "rabbitmq_node_mem_used / rabbitmq_node_mem_limit * 100 > 90" @@ -615,7 +615,7 @@ groups: - name: Rabbitmq dead letter queue filling up description: Dead letter queue is filling up (> 10 msgs) query: 'rabbitmq_queue_messages{queue="my-dead-letter-queue"} > 10' - severity: error + severity: critical - name: Rabbitmq too many messages in queue description: Queue is filling up (> 1000 msgs) query: 'rabbitmq_queue_messages_ready{queue="my-queue"} > 1000' @@ -627,11 +627,11 @@ groups: - name: Rabbitmq no consumer description: Queue has no consumer query: "rabbitmq_queue_consumers == 0" - severity: error + severity: critical - name: Rabbitmq too many consumers description: Queue should have only 1 consumer query: "rabbitmq_queue_consumers > 1" - severity: error + severity: critical - name: Rabbitmq unactive exchange description: Exchange receive less than 5 msgs per second query: 'rate(rabbitmq_exchange_messages_published_in_total{exchange="my-exchange"}[1m]) < 5' @@ -645,7 +645,7 @@ groups: - name: Elasticsearch Heap Usage Too High description: "The heap usage is over 90% for 5m" query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90' - severity: error + severity: critical - name: Elasticsearch Heap Usage warning description: "The heap usage is over 80% for 5m" query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80' @@ -657,11 +657,11 @@ groups: - name: Elasticsearch disk out of space description: The disk usage is over 90% query: 'elasticsearch_filesystem_data_available_bytes / elasticsearch_filesystem_data_size_bytes * 100 < 10' - severity: error + severity: critical - name: Elasticsearch Cluster Red description: Elastic Cluster Red status query: 'elasticsearch_cluster_health_status{color="red"} == 1' - severity: error + severity: critical - name: Elasticsearch Cluster Yellow description: Elastic Cluster Yellow status query: 'elasticsearch_cluster_health_status{color="yellow"} == 1' @@ -669,15 +669,15 @@ groups: - name: Elasticsearch Healthy Nodes description: "Number Healthy Nodes less then number_of_nodes" query: "elasticsearch_cluster_health_number_of_nodes < number_of_nodes" - severity: error + severity: critical - name: Elasticsearch Healthy Data Nodes description: "Number Healthy Data Nodes less then number_of_data_nodes" query: "elasticsearch_cluster_health_number_of_data_nodes < number_of_data_nodes" - severity: error + severity: critical - name: Elasticsearch relocation shards description: "Number of relocation shards for 20 min" query: "elasticsearch_cluster_health_relocating_shards > 0" - severity: error + severity: critical - name: Elasticsearch initializing shards description: "Number of initializing shards for 10 min" query: "elasticsearch_cluster_health_initializing_shards > 0" @@ -685,7 +685,7 @@ groups: - name: Elasticsearch unassigned shards description: "Number of unassigned shards for 2 min" query: "elasticsearch_cluster_health_unassigned_shards > 0" - severity: error + severity: critical - name: Elasticsearch pending tasks description: "Number of pending tasks for 10 min. Cluster works slowly." query: "elasticsearch_cluster_health_number_of_pending_tasks > 0" @@ -707,7 +707,7 @@ groups: - name: Cassandra hints count description: Cassandra hints count has changed on {{ $labels.instance }} some nodes may go down query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:totalhints:count"}[1m]) > 3' - severity: error + severity: critical - name: Cassandra compaction task pending description: Many Cassandra compaction tasks are pending. You might need to increase I/O capacity by adding nodes to the cluster. query: 'avg_over_time(cassandra_stats{name="org:apache:cassandra:metrics:compaction:pendingtasks:value"}[30m]) > 100' @@ -723,7 +723,7 @@ groups: - name: Cassandra node down description: Cassandra node down query: 'sum(cassandra_stats{name="org:apache:cassandra:net:failuredetector:downendpointcount"}) by (service,group,cluster,env) > 0' - severity: error + severity: critical - name: Cassandra commitlog pending tasks description: Unexpected number of Cassandra commitlog pending tasks query: 'cassandra_stats{name="org:apache:cassandra:metrics:commitlog:pendingtasks:value"} > 15' @@ -747,11 +747,11 @@ groups: - name: Cassandra connection timeouts total description: Some connection between nodes are ending in timeout query: 'rate(cassandra_stats{name="org:apache:cassandra:metrics:connection:totaltimeouts:count"}[1m]) > 5' - severity: error + severity: critical - name: Cassandra storage exceptions description: Something is going wrong with cassandra storage query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:exceptions:count"}[1m]) > 1' - severity: error + severity: critical - name: Zookeeper exporters: @@ -767,11 +767,11 @@ groups: - name: Kafka topics replicas description: Kafka topic in-sync partition query: "sum(kafka_topic_partition_in_sync_replica) by (topic) < 3" - severity: error + severity: critical - name: Kafka consumers group description: Kafka consumers group query: "sum(kafka_consumergroup_lag) by (consumergroup) > 50" - severity: error + severity: critical - name: Reverse proxies and load balancers @@ -784,11 +784,11 @@ groups: - name: Nginx high HTTP 4xx error rate description: Too many HTTP requests with status 4xx (> 5%) query: 'sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' - severity: error + severity: critical - name: Nginx high HTTP 5xx error rate description: Too many HTTP requests with status 5xx (> 5%) query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' - severity: error + severity: critical - name: Nginx latency high description: Nginx p99 latency is higher than 10 seconds query: 'histogram_quantile(0.99, sum(rate(nginx_http_request_duration_seconds_bucket[30m])) by (host, node)) > 10' @@ -802,11 +802,11 @@ groups: - name: Apache down description: Apache down query: 'apache_up == 0' - severity: error + severity: critical - name: Apache workers load description: Apache workers in busy state approach the max workers count 80% workers busy on {{ $labels.instance }} query: '(sum by (instance) (apache_workers{state="busy"}) / sum by (instance) (apache_scoreboard) ) * 100 > 80' - severity: error + severity: critical - name: Apache restart description: Apache has just been restarted, less than one minute ago. query: 'apache_uptime_seconds_total / 60 < 1' @@ -823,35 +823,35 @@ groups: - name: HAProxy down description: HAProxy down query: 'haproxy_up = 0' - severity: error + severity: critical - name: HAProxy high HTTP 4xx error rate backend description: Too many HTTP requests with status 4xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' - severity: error + severity: critical - name: HAProxy high HTTP 4xx error rate backend description: Too many HTTP requests with status 5xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' - severity: error + severity: critical - name: HAProxy high HTTP 4xx error rate server description: Too many HTTP requests with status 4xx (> 5%) on server {{ $labels.server }} query: 'sum by (server) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' - severity: error + severity: critical - name: HAProxy high HTTP 5xx error rate server description: Too many HTTP requests with status 5xx (> 5%) on server {{ $labels.server }} query: 'sum by (server) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5' - severity: error + severity: critical - name: HAProxy backend connection errors description: Too many connection errors to {{ $labels.fqdn }}/{{ $labels.backend }} backend (> 5%). Request throughput may be to high. query: 'sum by (backend) rate(haproxy_backend_connection_errors_total[1m]) * 100 > 5' - severity: error + severity: critical - name: HAProxy server response errors description: Too many response errors to {{ $labels.server }} server (> 5%). query: 'sum by (server) rate(haproxy_server_response_errors_total[1m]) * 100 > 5' - severity: error + severity: critical - name: HAProxy server connection errors description: Too many connection errors to {{ $labels.server }} server (> 5%). Request throughput may be to high. query: 'sum by (server) rate(haproxy_server_connection_errors_total[1m]) * 100 > 5' - severity: error + severity: critical - name: HAProxy backend max active session description: HAproxy backend {{ $labels.fqdn }}/{{ $labels.backend }} is reaching session limit (> 80%). query: 'avg_over_time((sum by (backend) (haproxy_server_max_sessions) / sum by (backend) (haproxy_server_limit_sessions)) [2m]) * 100 > 80' @@ -871,11 +871,11 @@ groups: - name: HAProxy backend down description: HAProxy backend is down query: 'haproxy_backend_up = 0' - severity: error + severity: critical - name: HAProxy server down description: HAProxy server is down query: 'haproxy_server_up = 0' - severity: error + severity: critical - name: HAProxy frontend security blocked requests description: HAProxy is blocking requests for security reason query: 'rate(sum by (frontend) (haproxy_frontend_requests_denied_total)) > 10' @@ -893,15 +893,15 @@ groups: - name: Traefik backend down description: All Traefik backends are down query: "count(traefik_backend_server_up) by (backend) == 0" - severity: error + severity: critical - name: Traefik high HTTP 4xx error rate backend description: Traefik backend 4xx error rate is above 5% query: 'sum(rate(traefik_backend_requests_total{code=~"4.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' - severity: error + severity: critical - name: Traefik high HTTP 5xx error rate backend description: Traefik backend 5xx error rate is above 5% query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' - severity: error + severity: critical - name: Runtimes @@ -934,7 +934,7 @@ groups: - name: Sidekiq scheduling latency too high description: Sidekiq jobs are taking more than 2 minutes to be picked up. Users may be seeing delays in background processing. query: 'max(sidekiq_queue_latency) > 120' - severity: error + severity: critical - name: Orchestrators @@ -947,19 +947,19 @@ groups: - name: Kubernetes Node ready description: Node {{ $labels.node }} has been unready for a long time query: 'kube_node_status_condition{condition="Ready",status="true"} == 0' - severity: error + severity: critical - name: Kubernetes memory pressure description: "{{ $labels.node }} has MemoryPressure condition" query: 'kube_node_status_condition{condition="MemoryPressure",status="true"} == 1' - severity: error + severity: critical - name: Kubernetes disk pressure description: "{{ $labels.node }} has DiskPressure condition" query: 'kube_node_status_condition{condition="DiskPressure",status="true"} == 1' - severity: error + severity: critical - name: Kubernetes out of disk description: "{{ $labels.node }} has OutOfDisk condition" query: 'kube_node_status_condition{condition="OutOfDisk",status="true"} == 1' - severity: error + severity: critical - name: Kubernetes Job failed description: "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete" query: "kube_job_status_failed > 0" @@ -979,15 +979,15 @@ groups: - name: Kubernetes Volume full in four days description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available." query: 'predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0' - severity: error + severity: critical - name: Kubernetes PersistentVolume error description: "Persistent volume is in bad state" query: 'kube_persistentvolume_status_phase{phase=~"Failed|Pending",job="kube-state-metrics"} > 0' - severity: error + severity: critical - name: Kubernetes StatefulSet down description: A StatefulSet went down query: "(kube_statefulset_status_replicas_ready / kube_statefulset_status_replicas_current) != 1" - severity: error + severity: critical - name: Kubernetes HPA scaling ability description: Pod is unable to scale query: 'kube_hpa_status_condition{condition="false", status="AbleToScale"} == 1' @@ -1003,7 +1003,7 @@ groups: - name: Kubernetes Pod not healthy description: Pod has been in a non-ready state for longer than an hour. query: 'min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[1h:]) > 0' - severity: error + severity: critical - name: Kubernetes pod crash looping description: Pod {{ $labels.pod }} is crash looping query: 'rate(kube_pod_container_status_restarts_total[15m]) * 60 * 5 > 5' @@ -1023,23 +1023,23 @@ groups: - name: Kubernetes Deployment generation mismatch description: A Deployment has failed but has not been rolled back. query: 'kube_deployment_status_observed_generation != kube_deployment_metadata_generation' - severity: error + severity: critical - name: Kubernetes StatefulSet generation mismatch description: A StatefulSet has failed but has not been rolled back. query: 'kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation' - severity: error + severity: critical - name: Kubernetes StatefulSet update not rolled out description: StatefulSet update has not been rolled out. query: 'max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)' - severity: error + severity: critical - name: Kubernetes DaemonSet rollout stuck description: Some Pods of DaemonSet are not scheduled or not ready query: 'kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0' - severity: error + severity: critical - name: Kubernetes DaemonSet misscheduled description: Some DaemonSet Pods are running where they are not supposed to run query: 'kube_daemonset_status_number_misscheduled > 0' - severity: error + severity: critical - name: Kubernetes CronJob too long description: CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete. query: 'time() - kube_cronjob_next_schedule_time > 3600' @@ -1047,15 +1047,15 @@ groups: - name: Kubernetes job completion description: Kubernetes Job failed to complete query: 'kube_job_spec_completions - kube_job_status_succeeded > 0 or kube_job_status_failed > 0' - severity: error + severity: critical - name: Kubernetes API server errors description: Kubernetes API server is experiencing high error rate query: 'sum(rate(apiserver_request_count{job="apiserver",code=~"^(?:5..)$"}[2m])) / sum(rate(apiserver_request_count{job="apiserver"}[2m])) * 100 > 3' - severity: error + severity: critical - name: Kubernetes API client errors description: Kubernetes API client is experiencing high error rate query: '(sum(rate(rest_client_requests_total{code=~"(4|5).."}[2m])) by (instance, job) / sum(rate(rest_client_requests_total[2m])) by (instance, job)) * 100 > 1' - severity: error + severity: critical - name: Kubernetes client certificate expires next week description: A client certificate used to authenticate to the apiserver is expiring next week. query: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 7*24*60*60' @@ -1063,7 +1063,7 @@ groups: - name: Kubernetes client certificate expires soon description: A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours. query: 'apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60' - severity: error + severity: critical - name: Kubernetes API server latency description: 'Kubernetes API server has a 99th percentile latency of {{ $value }} seconds for {{ $labels.verb }} {{ $labels.resource }}.' query: 'histogram_quantile(0.99, sum(apiserver_request_latencies_bucket{verb!~"CONNECT|WATCHLIST|WATCH|PROXY"}) WITHOUT (instance, resource)) / 1e+06 > 1' @@ -1083,15 +1083,15 @@ groups: - name: Consul service healthcheck failed description: "Service: `{{ $labels.service_name }}` Healthcheck: `{{ $labels.service_id }}`" query: "consul_catalog_service_node_healthy == 0" - severity: error + severity: critical - name: Consul missing master node description: Numbers of consul raft peers should be 3, in order to preserve quorum. query: "consul_raft_peers < 3" - severity: error + severity: critical - name: Consul agent unhealthy description: A Consul agent is down query: 'consul_health_node_status{status="critical"} == 1' - severity: error + severity: critical - name: Etcd exporters: @@ -1099,11 +1099,11 @@ groups: - name: Etcd insufficient Members description: Etcd cluster should have an odd number of members query: "count(etcd_server_id) % 2 == 0" - severity: error + severity: critical - name: Etcd no Leader description: Etcd cluster have no leader query: "etcd_server_has_leader == 0" - severity: error + severity: critical - name: Etcd high number of leader changes description: Etcd leader changed more than 3 times during last hour query: "increase(etcd_server_leader_changes_seen_total[1h]) > 3" @@ -1115,7 +1115,7 @@ groups: - name: Etcd high number of failed GRPC requests description: More than 5% GRPC request failure detected in Etcd for 5 minutes query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.05' - severity: error + severity: critical - name: Etcd GRPC requests slow description: GRPC requests slowing down, 99th percentil is over 0.15s for 5 minutes query: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15' @@ -1127,7 +1127,7 @@ groups: - name: Etcd high number of failed HTTP requests description: More than 5% HTTP failure detected in Etcd for 5 minutes query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.05" - severity: error + severity: critical - name: Etcd HTTP requests slow description: HTTP requests slowing down, 99th percentil is over 0.15s for 5 minutes query: "histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15" @@ -1175,7 +1175,7 @@ groups: - name: Ceph State description: Ceph instance unhealthy query: 'ceph_health_status != 0' - severity: error + severity: critical - name: Ceph monitor clock skew description: Ceph monitor clock skew detected. Please check ntp and hardware clock settings query: 'abs(ceph_monitor_clock_skew_seconds) > 0.2' @@ -1187,7 +1187,7 @@ groups: - name: Ceph OSD Down description: Ceph Object Storage Daemon Down query: 'ceph_osd_up == 0' - severity: error + severity: critical - name: Ceph high OSD latency description: "Ceph Object Storage Daemon latetncy is high. Please check if it doesn't stuck in weird state." query: 'ceph_osd_perf_apply_latency_seconds > 10' @@ -1203,11 +1203,11 @@ groups: - name: Ceph PG down description: Some Ceph placement groups are down. Please ensure that all the data are available. query: 'ceph_pg_down > 0' - severity: error + severity: critical - name: Ceph PG incomplete description: Some Ceph placement groups are incomplete. Please ensure that all the data are available. query: 'ceph_pg_incomplete > 0' - severity: error + severity: critical - name: Ceph PG inconsistant description: Some Ceph placement groups are inconsitent. Data is available but inconsistent across nodes. query: ceph_pg_inconsistent > 0 @@ -1223,7 +1223,7 @@ groups: - name: Ceph PG unavailable description: Some Ceph placement groups are unavailable. query: 'ceph_pg_total - ceph_pg_active > 0' - severity: error + severity: critical - name: SpeedTest exporters: @@ -1261,7 +1261,7 @@ groups: - name: Minio disk offline description: 'Minio disk is offline' query: "minio_offline_disks > 0" - severity: error + severity: critical - name: Minio storage space exhausted description: 'Minio storage space is low (< 10 GB)' query: "minio_disk_storage_free_bytes / 1024 / 1024 / 1024 < 10" @@ -1275,11 +1275,11 @@ groups: - name: Juniper switch down description: The switch appears to be down query: junos_up == 0 - severity: error + severity: critical - name: Juniper high Bandwith Usage 1GiB description: Interface is highly saturated for at least 1 min. (> 0.90GiB/s) query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.90" - severity: error + severity: critical - name: Juniper high Bandwith Usage 1GiB description: Interface is getting saturated for at least 1 min. (> 0.80GiB/s) query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.80" @@ -1292,7 +1292,7 @@ groups: - name: CoreDNS Panic Count description: Number of CoreDNS panics encountered query: "increase(coredns_panic_count_total[10m]) > 0" - severity: error + severity: critical - name: Other @@ -1303,12 +1303,12 @@ groups: - name: Thanos compaction halted description: Thanos compaction has failed to run and is now halted. query: 'thanos_compactor_halted == 1' - severity: error + severity: critical - name: Thanos compact bucket operation failure description: Thanos compaction has failing storage operations query: 'rate(thanos_objstore_bucket_operation_failures_total[1m]) > 0' - severity: error + severity: critical - name: Thanos compact not run description: Thanos compaction has not run in 24 hours. query: '(time() - thanos_objstore_bucket_last_successful_upload_time) > 24*60*60' - severity: error + severity: critical diff --git a/alertmanager.md b/alertmanager.md index a4acb62..7e7375c 100644 --- a/alertmanager.md +++ b/alertmanager.md @@ -28,7 +28,7 @@ groups: expr: redis_up{} == 0 for: 2m labels: - severity: error + severity: critical annotations: summary: "Redis instance down" description: "Whatever" @@ -68,13 +68,13 @@ route: - receiver: "slack" group_wait: 10s match_re: - severity: error|warning + severity: critical|warning continue: true - receiver: "pager" group_wait: 10s match_re: - severity: error + severity: critial continue: true receivers: From e6de4131467c07406514886d62047ed60e252066 Mon Sep 17 00:00:00 2001 From: Fernando Carletti Date: Mon, 18 May 2020 17:38:05 -0500 Subject: [PATCH 105/126] fix: container ContainerMemoryUsage alert --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..ac9ed97 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -239,7 +239,7 @@ groups: severity: warning - name: Container Memory usage description: Container Memory usage is above 80% - query: "(sum(container_memory_usage_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes) BY (instance, name) * 100) > 80" + query: "(sum(container_memory_working_set_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes) BY (instance, name) * 100) > 80" severity: warning - name: Container Volume usage description: Container Volume usage is above 80% From 663b0e94da313e6d6414992b28fbfa2b934fae08 Mon Sep 17 00:00:00 2001 From: Ilya Kisleyko Date: Wed, 20 May 2020 20:04:32 +0300 Subject: [PATCH 106/126] check free space for all mountpoints --- _data/rules.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..a98290e 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -145,8 +145,11 @@ groups: severity: warning - name: Host out of disk space description: Disk is almost full (< 10% left) - query: '(node_filesystem_avail_bytes{mountpoint="/rootfs"} * 100) / node_filesystem_size_bytes{mountpoint="/rootfs"} < 10' + query: '(node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10' severity: warning + comments: | + please add ignored mountpoints in node_exporter parameters like + "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)" - name: Host disk will fill in 4 hours description: Disk will fill in 4 hours at current write rate query: 'predict_linear(node_filesystem_free_bytes{fstype!~"tmpfs"}[1h], 4 * 3600) < 0' From 2f1a1b4670a5f658cc299727eb3e3cfb30416926 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 May 2020 16:18:47 +0000 Subject: [PATCH 107/126] Bump activesupport from 6.0.2.1 to 6.0.3.1 Bumps [activesupport](https://github.com/rails/rails) from 6.0.2.1 to 6.0.3.1. - [Release notes](https://github.com/rails/rails/releases) - [Changelog](https://github.com/rails/rails/blob/v6.0.3.1/activesupport/CHANGELOG.md) - [Commits](https://github.com/rails/rails/compare/v6.0.2.1...v6.0.3.1) Signed-off-by: dependabot[bot] --- Gemfile.lock | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8fd8867..27081de 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,12 +1,12 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.0.2.1) + activesupport (6.0.3.1) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) tzinfo (~> 1.1) - zeitwerk (~> 2.2) + zeitwerk (~> 2.2, >= 2.2.2) addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) coffee-script (2.4.1) @@ -202,7 +202,7 @@ GEM jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) - minitest (5.14.0) + minitest (5.14.1) multipart-post (2.1.1) nokogiri (1.10.9) mini_portile2 (~> 2.4.0) @@ -233,7 +233,7 @@ GEM thread_safe (0.3.6) typhoeus (1.3.1) ethon (>= 0.9.0) - tzinfo (1.2.6) + tzinfo (1.2.7) thread_safe (~> 0.1) unicode-display_width (1.6.1) zeitwerk (2.3.0) From 24f7095cd59ce2d2b46c9d0090f1fc27b58bbe8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micka=C3=ABl=20Can=C3=A9vet?= Date: Fri, 29 May 2020 10:11:54 +0200 Subject: [PATCH 108/126] Fix HAProxy rules --- _data/rules.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..505931c 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -822,7 +822,7 @@ groups: rules: - name: HAProxy down description: HAProxy down - query: 'haproxy_up = 0' + query: 'haproxy_up == 0' severity: critical - name: HAProxy high HTTP 4xx error rate backend description: Too many HTTP requests with status 4xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }} @@ -870,11 +870,11 @@ groups: severity: warning - name: HAProxy backend down description: HAProxy backend is down - query: 'haproxy_backend_up = 0' + query: 'haproxy_backend_up == 0' severity: critical - name: HAProxy server down description: HAProxy server is down - query: 'haproxy_server_up = 0' + query: 'haproxy_server_up == 0' severity: critical - name: HAProxy frontend security blocked requests description: HAProxy is blocking requests for security reason From 5e51c3daef8550cbe040d28b012593ac0db43fc7 Mon Sep 17 00:00:00 2001 From: Nabil BENDAFI Date: Wed, 17 Jun 2020 14:45:39 +0200 Subject: [PATCH 109/126] Fix data-clipboard-target-id unicity --- rules.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/rules.md b/rules.md index 6225fd9..9b2a86f 100644 --- a/rules.md +++ b/rules.md @@ -23,12 +23,14 @@
        {% for group in site.data.rules.groups %} + {% assign groupIndex = forloop.index %} {% for service in group.services %} {% assign serviceIndex = forloop.index %} {% for exporter in service.exporters %} {% assign nbrRules = exporter.rules | size %}
      • -

        + {% assign serviceId = service.name | replace: " ", "-" | downcase %} +

        {{ serviceIndex }}. {{ service.name }} {% if exporter.name %}: @@ -45,7 +47,7 @@ ({{ nbrRules }} rules) - [copy all] + [copy all] {% endif %}

        @@ -64,10 +66,10 @@ {{ serviceIndex }}.{{ ruleIndex }}. {{ rule.name }} -
        +
        {{ rule.description }} - [copy] + [copy]

        {% assign ruleName = rule.name | split: ' ' %} From b324c6f32f71c7ce9897fff61cc0e44394692b77 Mon Sep 17 00:00:00 2001 From: Nabil BENDAFI Date: Tue, 23 Jun 2020 13:40:01 +0200 Subject: [PATCH 110/126] feat(traefik): add rules for Traefik v2 Fixes #7 --- _data/rules.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..06a344d 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -902,7 +902,21 @@ groups: description: Traefik backend 5xx error rate is above 5% query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5' severity: critical - + - name: Embedded exporter v2 + doc_url: https://docs.traefik.io/observability/metrics/prometheus/ + rules: + - name: Traefik service down + description: All Traefik services are down + query: "count(traefik_service_server_up) by (service) == 0" + severity: critical + - name: Traefik high HTTP 4xx error rate service + description: Traefik service 4xx error rate is above 5% + query: 'sum(rate(traefik_service_requests_total{code=~"4.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5' + severity: critical + - name: Traefik high HTTP 5xx error rate service + description: Traefik service 5xx error rate is above 5% + query: 'sum(rate(traefik_service_requests_total{code=~"5.*"}[3m])) by (service) / sum(rate(traefik_service_requests_total[3m])) by (service) * 100 > 5' + severity: critical - name: Runtimes services: From 42b2dc07a6ad0786bf7a29fa593e23bdb3628eae Mon Sep 17 00:00:00 2001 From: Nabil BENDAFI Date: Wed, 24 Jun 2020 14:22:31 +0200 Subject: [PATCH 111/126] Fix numbering --- rules.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rules.md b/rules.md index 9b2a86f..41aeec6 100644 --- a/rules.md +++ b/rules.md @@ -31,6 +31,7 @@

      • {% assign serviceId = service.name | replace: " ", "-" | downcase %}

        + {{ groupIndex }}. {{ serviceIndex }}. {{ service.name }} {% if exporter.name %}: @@ -63,7 +64,7 @@ {% assign comments = rule.comments | strip | newline_to_br | split: '
        ' %}

      • - {{ serviceIndex }}.{{ ruleIndex }}. + {{ groupIndex}}.{{ serviceIndex }}.{{ ruleIndex }}. {{ rule.name }}

        From edbc9cac2bccc578be048b4dcc869310966971bc Mon Sep 17 00:00:00 2001 From: Nabil BENDAFI Date: Wed, 24 Jun 2020 14:49:56 +0200 Subject: [PATCH 112/126] fix: remove unnecessary test --- rules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules.md b/rules.md index 6225fd9..c0f15c2 100644 --- a/rules.md +++ b/rules.md @@ -64,7 +64,7 @@ {{ serviceIndex }}.{{ ruleIndex }}. {{ rule.name }}

        • -
        +
        {{ rule.description }} [copy] From add6d9c2f3bd57acf5eaf2b18b9ee97b78363cf8 Mon Sep 17 00:00:00 2001 From: tux Date: Tue, 30 Jun 2020 15:48:42 +0200 Subject: [PATCH 113/126] Add official rabbitmq exporter rules --- _data/rules.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..bffa881 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -587,6 +587,32 @@ groups: query: '(sum(mongodb_memory{type="virtual"}) BY (ip) / sum(mongodb_memory{type="mapped"}) BY (ip)) > 3' severity: warning + - name: RabbitMQ (official exporter) + exporters: + - name: rabbitmq/rabbitmq-prometheus + doc_url: https://github.com/rabbitmq/rabbitmq-prometheus + rules: + - name: Rabbitmq node down + description: Less than 3 nodes running in RabbitMQ cluster + query: "sum(rabbitmq_build_info) < 3" + severity: critical + - name: Rabbitmq memory high + description: A node use more than 90% of allocated RAM + query: "rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes * 100 > 90" + severity: warning + - name: Rabbitmq too much unack + description: Total unacknowledged messages are too high + query: "sum(rabbitmq_queue_messages_unacked) > 1000" + severity: warning + - name: Rabbitmq too much connections + description: The total connections of a node is too high + query: "rabbitmq_connections > 1000" + severity: warning + - name: Rabbitmq no queue consumer + description: A queue has less than 1 consumer + query: "rabbitmq_queue_consumers < 1" + severity: warning + - name: RabbitMQ exporters: - name: kbudde/rabbitmq-exporter From 05e521c0a81f5384567a45147607584a98c313da Mon Sep 17 00:00:00 2001 From: Mansur Marvanov Date: Thu, 9 Jul 2020 16:30:50 +0900 Subject: [PATCH 114/126] Fix PrometheusJobMissing alert --- Gemfile.lock | 2 +- _data/rules.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 8fd8867..1f5f7c0 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -245,4 +245,4 @@ DEPENDENCIES github-pages BUNDLED WITH - 1.17.3 + 2.1.2 diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..4d0f65f 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -15,7 +15,7 @@ groups: - rules: - name: Prometheus job missing description: A Prometheus job has disappeared - query: 'absent(up{job="my-job"})' + query: 'absent(up{job="prometheus"})' severity: warning - name: Prometheus target missing description: A Prometheus target has disappeared. An exporter might be crashed. From e009c5d8b55c9b0ef40d9e69aa4f2e46190628aa Mon Sep 17 00:00:00 2001 From: Ozarklake <67998142+Ozarklake@users.noreply.github.com> Date: Tue, 14 Jul 2020 12:55:17 +0800 Subject: [PATCH 115/126] Optimizing mysql slow query alert rules --- _data/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..d2794ec 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -351,8 +351,8 @@ groups: query: 'mysql_slave_status_master_server_id > 0 and ON (instance) (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300' severity: warning - name: MySQL slow queries - description: MySQL server is having some slow queries. - query: 'mysql_global_status_slow_queries > 0' + description: MySQL server mysql has some new slow query. + query: rate(mysql_global_status_slow_queries[2m]) > 0 severity: warning - name: MySQL restarted description: MySQL has just been restarted, less than one minute ago on {{ $labels.instance }}. From 4e66d17d01d73be97324f9ddb70f33b8246f45a3 Mon Sep 17 00:00:00 2001 From: Ozarklake Date: Fri, 17 Jul 2020 14:50:09 +0800 Subject: [PATCH 116/126] add sql server rules --- _data/rules.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index d2794ec..1ac03c8 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -465,6 +465,20 @@ groups: query: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20' severity: critical + - name: SQL Server + exporters: + - name: Ozarklake/prometheus-mssql-exporter + doc_url: https://github.com/Ozarklake/prometheus-mssql-exporter + rules: + - name: SQL Server down + description: SQl server instance is down + query: mssql_up == 0 + severity: critical + - name: SQL Server deadlock + description: SQL Server is having some deadlock. + query: irate(mssql_deadlocks[2m]) > 0 + severity: warning + - name: PGBouncer exporters: - name: spreaker/prometheus-pgbouncer-exporter From 88e812c78e123d31e055e8e596128e11def602b6 Mon Sep 17 00:00:00 2001 From: Ozarklake Date: Fri, 17 Jul 2020 14:50:09 +0800 Subject: [PATCH 117/126] add sql server rules --- _data/rules.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..5ed391f 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -465,6 +465,20 @@ groups: query: '((sum (pg_locks_count)) / (pg_settings_max_locks_per_transaction * pg_settings_max_connections)) > 0.20' severity: critical + - name: SQL Server + exporters: + - name: Ozarklake/prometheus-mssql-exporter + doc_url: https://github.com/Ozarklake/prometheus-mssql-exporter + rules: + - name: SQL Server down + description: SQl server instance is down + query: mssql_up == 0 + severity: critical + - name: SQL Server deadlock + description: SQL Server is having some deadlock. + query: irate(mssql_deadlocks[2m]) > 0 + severity: warning + - name: PGBouncer exporters: - name: spreaker/prometheus-pgbouncer-exporter From 8fb5da83decbde0e54ed67299e462ddeb877f2d8 Mon Sep 17 00:00:00 2001 From: Nirav Chotai Date: Fri, 24 Jul 2020 13:32:44 +0800 Subject: [PATCH 118/126] Fix HPA alerts - Fixing KubernetesHpaMetricAvailability - Fixing KubernetesHpaScalingAbility --- _data/rules.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..3eee7f0 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -990,11 +990,11 @@ groups: severity: critical - name: Kubernetes HPA scaling ability description: Pod is unable to scale - query: 'kube_hpa_status_condition{condition="false", status="AbleToScale"} == 1' + query: 'kube_hpa_status_condition{status="false", condition ="AbleToScale"} == 1' severity: warning - name: Kubernetes HPA metric availability - description: HPA is not able to colelct metrics - query: 'kube_hpa_status_condition{condition="false", status="ScalingActive"} == 1' + description: HPA is not able to collect metrics + query: 'kube_hpa_status_condition{status="false", condition="ScalingActive"} == 1' severity: warning - name: Kubernetes HPA scale capability description: The maximum number of desired Pods has been hit From 6c5f708179bdfee374e0efcf1b27e521e204876f Mon Sep 17 00:00:00 2001 From: Daniel Andrzejewski Date: Thu, 17 Sep 2020 15:13:42 +0200 Subject: [PATCH 119/126] node_disk_write_time_seconds_total is in seconds, not in milliseconds. node_disk_write_time_seconds_total should be grater than 0, otherwise you get +Inf result. --- _data/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..2983754 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -157,11 +157,11 @@ groups: severity: warning - name: Host unusual disk read latency description: Disk latency is growing (read operations > 100ms) - query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 100" + query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m])" severity: warning - name: Host unusual disk write latency description: Disk latency is growing (write operations > 100ms) - query: "rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 100" + query: "rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1 and rate(node_disk_writes_completed_total[1m]) > 0" severity: warning - name: Host high CPU load description: CPU load is > 80% From fc4797db9e523e0f793f0486119e94c3cd7b06e7 Mon Sep 17 00:00:00 2001 From: Daniel Andrzejewski Date: Thu, 17 Sep 2020 15:19:14 +0200 Subject: [PATCH 120/126] small fix --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 2983754..86edb23 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -157,7 +157,7 @@ groups: severity: warning - name: Host unusual disk read latency description: Disk latency is growing (read operations > 100ms) - query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m])" + query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1 and rate(node_disk_reads_completed_total[1m]) > 0" severity: warning - name: Host unusual disk write latency description: Disk latency is growing (write operations > 100ms) From 5288c9a2f53e136158ffa2012cd0e5b0ca78a1e6 Mon Sep 17 00:00:00 2001 From: fsschmitt <492108+fsschmitt@users.noreply.github.com> Date: Tue, 6 Oct 2020 13:33:50 +0100 Subject: [PATCH 121/126] Fix node_md_disks state from fail to failed --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..0985bfb 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -197,7 +197,7 @@ groups: severity: critical - name: Host RAID disk failure description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap' - query: 'node_md_disks{state="fail"} > 0' + query: 'node_md_disks{state="failed"} > 0' severity: warning - name: Host kernel version deviations description: Different kernel versions are running From 4266b4d3264cf2cfb1440f1bf3570abe048e322e Mon Sep 17 00:00:00 2001 From: fsschmitt <492108+fsschmitt@users.noreply.github.com> Date: Tue, 6 Oct 2020 14:36:22 +0100 Subject: [PATCH 122/126] Fix time unit on disk read/write latency rule --- _data/rules.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index c60ab90..0fa6a83 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -157,11 +157,11 @@ groups: severity: warning - name: Host unusual disk read latency description: Disk latency is growing (read operations > 100ms) - query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 100" + query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 0.1" severity: warning - name: Host unusual disk write latency description: Disk latency is growing (write operations > 100ms) - query: "rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 100" + query: "rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 0.1" severity: warning - name: Host high CPU load description: CPU load is > 80% From cf70272309f27e90580a70e083514f2078f92a68 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 11 Oct 2020 16:08:54 +0200 Subject: [PATCH 123/126] fix(container memory limit): filter by containers having max memory setting --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 651fa44..09231b9 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -239,7 +239,7 @@ groups: severity: warning - name: Container Memory usage description: Container Memory usage is above 80% - query: "(sum(container_memory_working_set_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes) BY (instance, name) * 100) > 80" + query: "(sum(container_memory_working_set_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 80" severity: warning - name: Container Volume usage description: Container Volume usage is above 80% From 7a609adf18f760946ee0404e1bbad983573822a7 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 11 Oct 2020 16:11:44 +0200 Subject: [PATCH 124/126] adding comment to container OOM killer warning --- _data/rules.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/_data/rules.yml b/_data/rules.yml index 09231b9..aae0373 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -241,6 +241,7 @@ groups: description: Container Memory usage is above 80% query: "(sum(container_memory_working_set_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 80" severity: warning + comments: See https://medium.com/faun/how-much-is-too-much-the-linux-oomkiller-and-used-memory-d32186f29c9d - name: Container Volume usage description: Container Volume usage is above 80% query: "(1 - (sum(container_fs_inodes_free) BY (instance) / sum(container_fs_inodes_total) BY (instance)) * 100) > 80" From bafcd1e9220ecf1f98575337ad90cf79e31c75ac Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 11 Oct 2020 17:35:46 +0200 Subject: [PATCH 125/126] Update rules.yml --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 5ed391f..1ecd043 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -476,7 +476,7 @@ groups: severity: critical - name: SQL Server deadlock description: SQL Server is having some deadlock. - query: irate(mssql_deadlocks[2m]) > 0 + query: rate(mssql_deadlocks[1m]) > 0 severity: warning - name: PGBouncer From 2f6b9832fa904296032d97b3ec6a0e90c2aad8de Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Sun, 11 Oct 2020 18:06:06 +0200 Subject: [PATCH 126/126] Update rules.yml --- _data/rules.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index bffa881..37a97c9 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -596,13 +596,17 @@ groups: description: Less than 3 nodes running in RabbitMQ cluster query: "sum(rabbitmq_build_info) < 3" severity: critical + - name: Rabbitmq instances different versions + description: Running different version of Rabbitmq in the same cluster, can lead to failure. + query: "count(count(rabbitmq_build_info) by (rabbitmq_version)) > 1" + severity: warning - name: Rabbitmq memory high description: A node use more than 90% of allocated RAM query: "rabbitmq_process_resident_memory_bytes / rabbitmq_resident_memory_limit_bytes * 100 > 90" severity: warning - name: Rabbitmq too much unack - description: Total unacknowledged messages are too high - query: "sum(rabbitmq_queue_messages_unacked) > 1000" + description: Too much unacknowledged messages + query: "sum(rabbitmq_queue_messages_unacked) BY (queue) > 1000" severity: warning - name: Rabbitmq too much connections description: The total connections of a node is too high @@ -612,9 +616,6 @@ groups: description: A queue has less than 1 consumer query: "rabbitmq_queue_consumers < 1" severity: warning - - - name: RabbitMQ - exporters: - name: kbudde/rabbitmq-exporter doc_url: https://github.com/kbudde/rabbitmq_exporter rules: