From 97ecdab26ceb4f6adec3e8044368dd20d91714b3 Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Wed, 18 Dec 2019 15:32:52 +0000 Subject: [PATCH 1/2] Added "Disk will fill in 4 hours" alert --- _data/rules.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 65bff3c..3f29300 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -45,6 +45,10 @@ services: description: Disk is almost full (< 10% left) query: 'node_filesystem_free_bytes{mountpoint ="/rootfs"} / node_filesystem_size_bytes{mountpoint ="/rootfs"} * 100 < 10' severity: warning + - name: Disk will fill in 4 hours + description: Disk will fill in 4 hours at current write rate + query: 'predict_linear(node_filesystem_free_bytes[1h], 4 * 3600) < 0' + severity: warning - name: Out of inodes description: Disk is almost running out of available inodes (< 10% left) query: 'node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint ="/rootfs"} * 100 < 10' From ce51db2a6f20f55a95bc4a51ed38b79c92e640b4 Mon Sep 17 00:00:00 2001 From: Rob Brown Date: Wed, 18 Dec 2019 15:38:23 +0000 Subject: [PATCH 2/2] Added Prometheus Not connected to alertmanager alert --- _data/rules.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/_data/rules.yml b/_data/rules.yml index 3f29300..2783148 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -7,6 +7,10 @@ services: description: Prometheus configuration reload error query: 'prometheus_config_last_reload_successful != 1' severity: error + - name: Prometheus not connected to alertmanager + description: Prometheus cannot connect the alertmanager + query: 'prometheus_notifications_alertmanagers_discovered < 1' + severity: error - name: AlertManager configuration reload description: AlertManager configuration reload error query: 'alertmanager_config_last_reload_successful != 1'