From f7d08e364b32889444f7d83ee4d2358d98ae1446 Mon Sep 17 00:00:00 2001 From: Piotr Parczewski Date: Thu, 17 Dec 2020 15:00:01 +0100 Subject: [PATCH 1/2] Fix Elasticsearch "No new documents" alert. Prometheus rate() function calculates the per-second average rate of increase. This means the alert gets triggered whenever during last 10 minutes there were less than 1 document ingested *per second* (60 documents per minute). Signed-off-by: Piotr Parczewski --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 22f22c3..26ca4ff 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -781,7 +781,7 @@ groups: severity: warning - name: Elasticsearch no new documents description: No new documents for 10 min! - query: 'rate(elasticsearch_indices_docs{es_data_node="true"}[10m]) < 1' + query: 'increase(elasticsearch_indices_docs{es_data_node="true"}[10m]) < 1' severity: warning - name: Cassandra From 0add93363fb3e254c851833ffdbd485fe31d0b99 Mon Sep 17 00:00:00 2001 From: Tosin Ogunrinde Date: Wed, 30 Dec 2020 00:30:08 +0000 Subject: [PATCH 2/2] Fix JVM "JVM memory filling up" alert --- _data/rules.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_data/rules.yml b/_data/rules.yml index 9c889ce..3db80f5 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1050,7 +1050,7 @@ groups: rules: - name: JVM memory filling up description: JVM memory is filling up (> 80%) - query: 'jvm_memory_bytes_used / jvm_memory_bytes_max{area="heap"} > 0.8' + query: '(jvm_memory_used_bytes{area="heap"} / jvm_memory_max_bytes{area="heap"}) * 100 > 80' severity: warning - name: Sidekiq