From 11a78f0f065c4bb5a2601f7aafd82ba2b3cc4e50 Mon Sep 17 00:00:00 2001
From: dzaczek <jacekzaleski@protonmail.ch>
Date: Sun, 16 Feb 2025 23:46:53 +0100
Subject: [PATCH] Update google-cadvisor.yml (#382)

* Update google-cadvisor.yml

    Expression Explanation:
    The expression calculates the absolute change in CPU usage for containers by comparing the current rate of CPU usage (within the last 1 minute) with the rate of CPU usage from the previous minute. If this change exceeds 25%, the alert is triggered. Additionally, it compares the current rate of CPU usage with the rate from the previous 5 minutes to capture larger trends. If any of these conditions are met, the alert fires.

    Alert Details:
    - Alert Name: ContainerHighLowChangeCpuUsage
    - Trigger Condition: Absolute change in CPU usage exceeding 25%
    - Alert Severity: Informational (info)

* Add alert rule for high CPU usage change

* Change alert severity from warning to info

---------

Co-authored-by: Samuel Berthe <dev@samuel-berthe.fr>
---
 _data/rules.yml                                  |  4 ++++
 dist/rules/docker-containers/google-cadvisor.yml | 10 ++++++++++
 2 files changed, 14 insertions(+)

diff --git a/_data/rules.yml b/_data/rules.yml
index d33d641..f8651d6 100644
--- a/_data/rules.yml
+++ b/_data/rules.yml
@@ -396,6 +396,10 @@ groups:
                 query: 'sum(increase(container_cpu_cfs_throttled_periods_total{container!=""}[5m])) by (container, pod, namespace) / sum(increase(container_cpu_cfs_periods_total[5m])) by (container, pod, namespace) > ( 25 / 100 )'
                 severity: warning
                 for: 5m
+              - name: Container high low change CPU usage
+                description: This alert rule monitors the absolute change in CPU usage within a time window and triggers an alert when the change exceeds 25%.
+                query: '(abs((sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100) - (sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m] offset 1m)) * 100)) or abs((sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100) - (sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[5m] offset 1m)) * 100))) > 25'
+                severity: info
               - name: Container Low CPU utilization
                 description: Container CPU utilization is under 20% for 1 week. Consider reducing the allocated CPU.
                 query: '(sum(rate(container_cpu_usage_seconds_total{container!=""}[5m])) by (pod, container) / sum(container_spec_cpu_quota{container!=""}/container_spec_cpu_period{container!=""}) by (pod, container) * 100) < 20'
diff --git a/dist/rules/docker-containers/google-cadvisor.yml b/dist/rules/docker-containers/google-cadvisor.yml
index cfbc333..02ca9c3 100644
--- a/dist/rules/docker-containers/google-cadvisor.yml
+++ b/dist/rules/docker-containers/google-cadvisor.yml
@@ -57,6 +57,16 @@ groups:
       annotations:
         summary: Container high throttle rate (instance {{ $labels.instance }})
         description: "Container is being throttled\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+        
+    - alert: ContainerHighLowChangeCpuUsage
+      expr: '(abs((  sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100)- (  sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m] offset 1m)) * 100)) or abs((  sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[1m])) * 100)- (  sum by (instance, name) (rate(container_cpu_usage_seconds_total{name!=""}[5m] offset 1m)) * 100)) )>25'
+      for: 0m
+      labels:
+        severity: info
+      annotations:
+        summary: Container change CPU utilization (instance {{ $labels.instance }})
+        description: "This alert rule monitors the absolute change in CPU usage within a time window and triggers an alert when the change exceeds 25%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
 
     - alert: ContainerLowCpuUtilization
       expr: '(sum(rate(container_cpu_usage_seconds_total{container!=""}[5m])) by (pod, container) / sum(container_spec_cpu_quota{container!=""}/container_spec_cpu_period{container!=""}) by (pod, container) * 100) < 20'