From 647508e5206051c3082cdd284ca8a976020fb520 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Mon, 1 Feb 2021 11:01:36 +0200 Subject: [PATCH 1/2] add alerts for kafka burrow exporter --- _data/rules.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index c9d1b9e..3b49736 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1105,7 +1105,21 @@ groups: query: 'sum(kafka_consumergroup_lag) by (consumergroup) > 50' severity: critical for: 1m - + - name: linkedin/Burrow + doc_url: https://github.com/linkedin/Burrow + rules: + - name: Kafka Offset Decreased + description: Kafka topic offset was decreased + query: 'delta(kafka_burrow_partition_current_offset[1m])<0' + severity: high + for: 3m + - name: Kafka Consumer Lag + description: Lag for consumer exist more than 30 minutes and it increasing + query: | + 'kafka_burrow_topic_partition_offset - on(partition, cluster, topic) group_right() kafka_burrow_partition_current_offset >= (kafka_burrow_topic_partition_offset offset 15m - on(partition, cluster, topic) group_right() kafka_burrow_partition_current_offset offset 15m) + AND kafka_burrow_topic_partition_offset - on(partition, cluster, topic) group_right() kafka_burrow_partition_current_offset > 0' + severity: high + for: 15m - name: Reverse proxies and load balancers services: From b619efac76ffb58fd74c60fa1f44bbbc14838a08 Mon Sep 17 00:00:00 2001 From: Anton Markelov Date: Mon, 1 Feb 2021 13:15:51 +0200 Subject: [PATCH 2/2] deal with proposed changes --- _data/rules.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 3b49736..25b414c 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1111,14 +1111,13 @@ groups: - name: Kafka Offset Decreased description: Kafka topic offset was decreased query: 'delta(kafka_burrow_partition_current_offset[1m])<0' - severity: high - for: 3m + severity: warning - name: Kafka Consumer Lag description: Lag for consumer exist more than 30 minutes and it increasing query: | 'kafka_burrow_topic_partition_offset - on(partition, cluster, topic) group_right() kafka_burrow_partition_current_offset >= (kafka_burrow_topic_partition_offset offset 15m - on(partition, cluster, topic) group_right() kafka_burrow_partition_current_offset offset 15m) AND kafka_burrow_topic_partition_offset - on(partition, cluster, topic) group_right() kafka_burrow_partition_current_offset > 0' - severity: high + severity: warning for: 15m - name: Reverse proxies and load balancers