mirror of
https://github.com/samber/awesome-prometheus-alerts.git
synced 2026-06-21 00:47:18 +08:00
Rework kube-state-metrics alerts (#381)
* Rework kube-state-metrics alerts: - provide meaningful labels in summary as 'instance' label hardly makes sense in most of them - rename some alerts to tell more accurate what the problem is - adjust description trying to follow some kind of the message schema found in other alerts * move changes to _data/rules.yml * Update rules.yml --------- Co-authored-by: Samuel Berthe <dev@samuel-berthe.fr>
This commit is contained in:
Pavel Timofeev
GitHub
parent
3e53d29986
commit
6b1685261d
2 changed files with 93 additions and 92 deletions
|
|
@ -1693,32 +1693,32 @@ groups:
|
|||
slug: kubestate-exporter
|
||||
doc_url: https://github.com/kubernetes/kube-state-metrics/tree/master/docs
|
||||
rules:
|
||||
- name: Kubernetes node not ready
|
||||
- name: Kubernetes Node not ready
|
||||
description: Node {{ $labels.node }} has been unready for a long time
|
||||
query: 'kube_node_status_condition{condition="Ready",status="true"} == 0'
|
||||
severity: critical
|
||||
for: 10m
|
||||
- name: Kubernetes memory pressure
|
||||
description: "{{ $labels.node }} has MemoryPressure condition"
|
||||
- name: Kubernetes Node memory pressure
|
||||
description: "Node {{ $labels.node }} has MemoryPressure condition"
|
||||
query: 'kube_node_status_condition{condition="MemoryPressure",status="true"} == 1'
|
||||
severity: critical
|
||||
for: 2m
|
||||
- name: Kubernetes disk pressure
|
||||
description: "{{ $labels.node }} has DiskPressure condition"
|
||||
- name: Kubernetes Node disk pressure
|
||||
description: "Node {{ $labels.node }} has DiskPressure condition"
|
||||
query: 'kube_node_status_condition{condition="DiskPressure",status="true"} == 1'
|
||||
severity: critical
|
||||
for: 2m
|
||||
- name: Kubernetes network unavailable
|
||||
description: "{{ $labels.node }} has NetworkUnavailable condition"
|
||||
- name: Kubernetes Node network unavailable
|
||||
description: "Node {{ $labels.node }} has NetworkUnavailable condition"
|
||||
query: 'kube_node_status_condition{condition="NetworkUnavailable",status="true"} == 1'
|
||||
severity: critical
|
||||
for: 2m
|
||||
- name: Kubernetes out of capacity
|
||||
description: "{{ $labels.node }} is out of capacity"
|
||||
- name: Kubernetes Node out of pod capacity
|
||||
description: "Node {{ $labels.node }} is out of pod capacity"
|
||||
query: 'sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(uid) group_left(node) (0 * kube_pod_info{pod_template_hash=""})) / sum by (node) (kube_node_status_allocatable{resource="pods"}) * 100 > 90'
|
||||
severity: warning
|
||||
for: 2m
|
||||
- name: Kubernetes container oom killer
|
||||
- name: Kubernetes Container oom killer
|
||||
description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes."
|
||||
query: '(kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1'
|
||||
severity: warning
|
||||
|
|
@ -1741,83 +1741,83 @@ groups:
|
|||
severity: warning
|
||||
for: 2m
|
||||
- name: Kubernetes Volume full in four days
|
||||
description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available."
|
||||
description: "Volume under {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available."
|
||||
query: 'predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0'
|
||||
severity: critical
|
||||
- name: Kubernetes PersistentVolume error
|
||||
description: "Persistent volume is in bad state"
|
||||
description: "Persistent volume {{ $labels.persistentvolume }} is in bad state"
|
||||
query: 'kube_persistentvolume_status_phase{phase=~"Failed|Pending", job="kube-state-metrics"} > 0'
|
||||
severity: critical
|
||||
- name: Kubernetes StatefulSet down
|
||||
description: A StatefulSet went down
|
||||
description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} went down
|
||||
query: 'kube_statefulset_replicas != kube_statefulset_status_replicas_ready > 0'
|
||||
severity: critical
|
||||
for: 1m
|
||||
- name: Kubernetes HPA scaling ability
|
||||
description: Pod is unable to scale
|
||||
- name: Kubernetes HPA scale inability
|
||||
description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to scale
|
||||
query: 'kube_horizontalpodautoscaler_status_condition{status="false", condition="AbleToScale"} == 1'
|
||||
severity: warning
|
||||
for: 2m
|
||||
- name: Kubernetes HPA metric availability
|
||||
description: HPA is not able to collect metrics
|
||||
- name: Kubernetes HPA metrics unavailability
|
||||
description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to collect metrics
|
||||
query: 'kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive"} == 1'
|
||||
severity: warning
|
||||
- name: Kubernetes HPA scale capability
|
||||
description: The maximum number of desired Pods has been hit
|
||||
- name: Kubernetes HPA scale maximum
|
||||
description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit maximum number of desired pods
|
||||
query: 'kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas'
|
||||
severity: info
|
||||
for: 2m
|
||||
- name: Kubernetes HPA underutilized
|
||||
description: HPA is constantly at minimum replicas for 50% of the time. Potential cost saving here.
|
||||
description: HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is constantly at minimum replicas for 50% of the time. Potential cost saving here.
|
||||
query: 'max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3' # allow minimum 3 replicas running
|
||||
severity: info
|
||||
- name: Kubernetes Pod not healthy
|
||||
description: Pod has been in a non-ready state for longer than 15 minutes.
|
||||
description: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-running state for longer than 15 minutes.
|
||||
query: 'sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"}) > 0'
|
||||
severity: critical
|
||||
for: 15m
|
||||
- name: Kubernetes pod crash looping
|
||||
description: Pod {{ $labels.pod }} is crash looping
|
||||
description: Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping
|
||||
query: 'increase(kube_pod_container_status_restarts_total[1m]) > 3'
|
||||
severity: warning
|
||||
for: 2m
|
||||
- name: Kubernetes ReplicasSet mismatch
|
||||
description: Deployment Replicas mismatch
|
||||
- name: Kubernetes ReplicaSet replicas mismatch
|
||||
description: ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset }} replicas mismatch
|
||||
query: 'kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas'
|
||||
severity: warning
|
||||
for: 10m
|
||||
- name: Kubernetes Deployment replicas mismatch
|
||||
description: Deployment Replicas mismatch
|
||||
description: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replicas mismatch
|
||||
query: 'kube_deployment_spec_replicas != kube_deployment_status_replicas_available'
|
||||
severity: warning
|
||||
for: 10m
|
||||
- name: Kubernetes StatefulSet replicas mismatch
|
||||
description: A StatefulSet does not match the expected number of replicas.
|
||||
description: StatefulSet does not match the expected number of replicas.
|
||||
query: 'kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas'
|
||||
severity: warning
|
||||
for: 10m
|
||||
- name: Kubernetes Deployment generation mismatch
|
||||
description: A Deployment has failed but has not been rolled back.
|
||||
description: Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has failed but has not been rolled back.
|
||||
query: 'kube_deployment_status_observed_generation != kube_deployment_metadata_generation'
|
||||
severity: critical
|
||||
for: 10m
|
||||
- name: Kubernetes StatefulSet generation mismatch
|
||||
description: A StatefulSet has failed but has not been rolled back.
|
||||
description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has failed but has not been rolled back.
|
||||
query: 'kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation'
|
||||
severity: critical
|
||||
for: 10m
|
||||
- name: Kubernetes StatefulSet update not rolled out
|
||||
description: StatefulSet update has not been rolled out.
|
||||
description: StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.
|
||||
query: 'max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)'
|
||||
severity: warning
|
||||
for: 10m
|
||||
- name: Kubernetes DaemonSet rollout stuck
|
||||
description: Some Pods of DaemonSet are not scheduled or not ready
|
||||
description: Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled or not ready
|
||||
query: 'kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0'
|
||||
severity: warning
|
||||
for: 10m
|
||||
- name: Kubernetes DaemonSet misscheduled
|
||||
description: Some DaemonSet Pods are running where they are not supposed to run
|
||||
description: Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run
|
||||
query: 'kube_daemonset_status_number_misscheduled > 0'
|
||||
severity: critical
|
||||
for: 1m
|
||||
|
|
@ -1827,7 +1827,7 @@ groups:
|
|||
severity: warning
|
||||
comments: |
|
||||
Threshold should be customized for each cronjob name.
|
||||
- name: Kubernetes job slow completion
|
||||
- name: Kubernetes Job slow completion
|
||||
description: Kubernetes Job {{ $labels.namespace }}/{{ $labels.job_name }} did not complete in time.
|
||||
query: 'kube_job_spec_completions - kube_job_status_succeeded - kube_job_status_failed > 0'
|
||||
severity: critical
|
||||
|
|
|
|||
121
dist/rules/kubernetes/kubestate-exporter.yml
vendored
121
dist/rules/kubernetes/kubestate-exporter.yml
vendored
|
|
@ -10,52 +10,52 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes node not ready (instance {{ $labels.instance }})
|
||||
summary: Kubernetes Node not ready (node {{ $labels.node }})
|
||||
description: "Node {{ $labels.node }} has been unready for a long time\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesMemoryPressure
|
||||
- alert: KubernetesNodeMemoryPressure
|
||||
expr: 'kube_node_status_condition{condition="MemoryPressure",status="true"} == 1'
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes memory pressure (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.node }} has MemoryPressure condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Node memory pressure (node {{ $labels.node }})
|
||||
description: "Node {{ $labels.node }} has MemoryPressure condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesDiskPressure
|
||||
- alert: KubernetesNodeDiskPressure
|
||||
expr: 'kube_node_status_condition{condition="DiskPressure",status="true"} == 1'
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes disk pressure (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.node }} has DiskPressure condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Node disk pressure (node {{ $labels.node }})
|
||||
description: "Node {{ $labels.node }} has DiskPressure condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesNetworkUnavailable
|
||||
- alert: KubernetesNodeNetworkUnavailable
|
||||
expr: 'kube_node_status_condition{condition="NetworkUnavailable",status="true"} == 1'
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes network unavailable (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.node }} has NetworkUnavailable condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Node network unavailable (node {{ $labels.node }})
|
||||
description: "Node {{ $labels.node }} has NetworkUnavailable condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesOutOfCapacity
|
||||
- alert: KubernetesNodeOutOfPodCapacity
|
||||
expr: 'sum by (node) ((kube_pod_status_phase{phase="Running"} == 1) + on(uid) group_left(node) (0 * kube_pod_info{pod_template_hash=""})) / sum by (node) (kube_node_status_allocatable{resource="pods"}) * 100 > 90'
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes out of capacity (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.node }} is out of capacity\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Node out of pod capacity (node {{ $labels.node }})
|
||||
description: "Node {{ $labels.node }} is out of pod capacity\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesContainerOomKiller
|
||||
- alert: KubernetesContainerOomKilled
|
||||
expr: '(kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1'
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes container oom killer (instance {{ $labels.instance }})
|
||||
summary: Kubernetes Container oom killed (pod {{ $labels.pod }})
|
||||
description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesJobFailed
|
||||
|
|
@ -64,7 +64,7 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes Job failed (instance {{ $labels.instance }})
|
||||
summary: Kubernetes Job failed (job_name {{ $labels.job_name }})
|
||||
description: "Job {{ $labels.namespace }}/{{ $labels.job_name }} failed to complete\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesCronjobSuspended
|
||||
|
|
@ -73,7 +73,7 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes CronJob suspended (instance {{ $labels.instance }})
|
||||
summary: Kubernetes CronJob suspended (cronjob {{ $labels.cronjob }})
|
||||
description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesPersistentvolumeclaimPending
|
||||
|
|
@ -82,7 +82,7 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes PersistentVolumeClaim pending (instance {{ $labels.instance }})
|
||||
summary: Kubernetes PersistentVolumeClaim pending (pvc {{ $labels.persistentvolumeclaim }})
|
||||
description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesVolumeOutOfDiskSpace
|
||||
|
|
@ -91,8 +91,8 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }})
|
||||
description: "Volume is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Volume out of disk space (pvc {{ $labels.persistentvolumeclaim }})
|
||||
description: "Volume under {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesVolumeFullInFourDays
|
||||
expr: 'predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0'
|
||||
|
|
@ -100,8 +100,8 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes Volume full in four days (instance {{ $labels.instance }})
|
||||
description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Volume full in four days (pvc {{ $labels.persistentvolumeclaim }})
|
||||
description: "Volume under {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesPersistentvolumeError
|
||||
expr: 'kube_persistentvolume_status_phase{phase=~"Failed|Pending", job="kube-state-metrics"} > 0'
|
||||
|
|
@ -109,8 +109,8 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes PersistentVolume error (instance {{ $labels.instance }})
|
||||
description: "Persistent volume is in bad state\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes PersistentVolume error (pv {{ $labels.persistentvolume }})
|
||||
description: "Persistent volume {{ $labels.persistentvolume }} is in bad state\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesStatefulsetDown
|
||||
expr: 'kube_statefulset_replicas != kube_statefulset_status_replicas_ready > 0'
|
||||
|
|
@ -118,35 +118,36 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes StatefulSet down (instance {{ $labels.instance }})
|
||||
description: "A StatefulSet went down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes StatefulSet down (statefulset {{ $labels.statefulset }})
|
||||
description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} went down\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesHpaScalingAbility
|
||||
- alert: KubernetesHpaScaleInability
|
||||
expr: 'kube_horizontalpodautoscaler_status_condition{status="false", condition="AbleToScale"} == 1'
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes HPA scaling ability (instance {{ $labels.instance }})
|
||||
description: "Pod is unable to scale\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes HPA scale inability (hpa {{ $labels.horizontalpodautoscaler }})
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to scale\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesHpaMetricAvailability
|
||||
- alert: KubernetesHpaMetricsUnavailability
|
||||
expr: 'kube_horizontalpodautoscaler_status_condition{status="false", condition="ScalingActive"} == 1'
|
||||
for: 0m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes HPA metric availability (instance {{ $labels.instance }})
|
||||
description: "HPA is not able to collect metrics\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes HPA metrics unavailability (hpa {{ $labels.horizontalpodautoscaler }})
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is unable to collect metrics\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesHpaScaleCapability
|
||||
|
||||
- alert: KubernetesHpaScaleMaximum
|
||||
expr: 'kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas'
|
||||
for: 2m
|
||||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: Kubernetes HPA scale capability (instance {{ $labels.instance }})
|
||||
description: "The maximum number of desired Pods has been hit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes HPA scale maximum (hpa {{ $labels.horizontalpodautoscaler }})
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} has hit maximum number of desired pods\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesHpaUnderutilized
|
||||
expr: 'max(quantile_over_time(0.5, kube_horizontalpodautoscaler_status_desired_replicas[1d]) == kube_horizontalpodautoscaler_spec_min_replicas) by (horizontalpodautoscaler) > 3'
|
||||
|
|
@ -154,8 +155,8 @@ groups:
|
|||
labels:
|
||||
severity: info
|
||||
annotations:
|
||||
summary: Kubernetes HPA underutilized (instance {{ $labels.instance }})
|
||||
description: "HPA is constantly at minimum replicas for 50% of the time. Potential cost saving here.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes HPA underutilized (hpa {{ $labels.horizontalpodautoscaler }})
|
||||
description: "HPA {{ $labels.namespace }}/{{ $labels.horizontalpodautoscaler }} is constantly at minimum replicas for 50% of the time. Potential cost saving here.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesPodNotHealthy
|
||||
expr: 'sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"}) > 0'
|
||||
|
|
@ -163,8 +164,8 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes Pod not healthy (instance {{ $labels.instance }})
|
||||
description: "Pod has been in a non-ready state for longer than 15 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Pod not healthy (pod {{ $labels.pod }})
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-running state for longer than 15 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesPodCrashLooping
|
||||
expr: 'increase(kube_pod_container_status_restarts_total[1m]) > 3'
|
||||
|
|
@ -172,17 +173,17 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes pod crash looping (instance {{ $labels.instance }})
|
||||
description: "Pod {{ $labels.pod }} is crash looping\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes pod crash looping (pod {{ $labels.pod }})
|
||||
description: "Pod {{ $labels.namespace }}/{{ $labels.pod }} is crash looping\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesReplicassetMismatch
|
||||
- alert: KubernetesReplicasetReplicasMismatch
|
||||
expr: 'kube_replicaset_spec_replicas != kube_replicaset_status_ready_replicas'
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes ReplicasSet mismatch (instance {{ $labels.instance }})
|
||||
description: "Deployment Replicas mismatch\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes ReplicaSet replicas mismatch (replicaset {{ $labels.replicaset }})
|
||||
description: "ReplicaSet {{ $labels.namespace }}/{{ $labels.replicaset }} replicas mismatch\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesDeploymentReplicasMismatch
|
||||
expr: 'kube_deployment_spec_replicas != kube_deployment_status_replicas_available'
|
||||
|
|
@ -190,8 +191,8 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes Deployment replicas mismatch (instance {{ $labels.instance }})
|
||||
description: "Deployment Replicas mismatch\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Deployment replicas mismatch (deployment {{ $labels.deployment }})
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} replicas mismatch\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesStatefulsetReplicasMismatch
|
||||
expr: 'kube_statefulset_status_replicas_ready != kube_statefulset_status_replicas'
|
||||
|
|
@ -199,8 +200,8 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes StatefulSet replicas mismatch (instance {{ $labels.instance }})
|
||||
description: "A StatefulSet does not match the expected number of replicas.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes StatefulSet replicas mismatch (statefulset {{ $labels.statefulset }})
|
||||
description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} replicas mismatch\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesDeploymentGenerationMismatch
|
||||
expr: 'kube_deployment_status_observed_generation != kube_deployment_metadata_generation'
|
||||
|
|
@ -208,8 +209,8 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes Deployment generation mismatch (instance {{ $labels.instance }})
|
||||
description: "A Deployment has failed but has not been rolled back.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes Deployment generation mismatch (deployment {{ $labels.deployment }})
|
||||
description: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} has failed but has not been rolled back.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesStatefulsetGenerationMismatch
|
||||
expr: 'kube_statefulset_status_observed_generation != kube_statefulset_metadata_generation'
|
||||
|
|
@ -217,8 +218,8 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes StatefulSet generation mismatch (instance {{ $labels.instance }})
|
||||
description: "A StatefulSet has failed but has not been rolled back.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes StatefulSet generation mismatch (statefulset {{ $labels.statefulset }})
|
||||
description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} has failed but has not been rolled back.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesStatefulsetUpdateNotRolledOut
|
||||
expr: 'max without (revision) (kube_statefulset_status_current_revision unless kube_statefulset_status_update_revision) * (kube_statefulset_replicas != kube_statefulset_status_replicas_updated)'
|
||||
|
|
@ -226,8 +227,8 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes StatefulSet update not rolled out (instance {{ $labels.instance }})
|
||||
description: "StatefulSet update has not been rolled out.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes StatefulSet update not rolled out (statefulset {{ $labels.statefulset }})
|
||||
description: "StatefulSet {{ $labels.namespace }}/{{ $labels.statefulset }} update has not been rolled out.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesDaemonsetRolloutStuck
|
||||
expr: 'kube_daemonset_status_number_ready / kube_daemonset_status_desired_number_scheduled * 100 < 100 or kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0'
|
||||
|
|
@ -235,8 +236,8 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes DaemonSet rollout stuck (instance {{ $labels.instance }})
|
||||
description: "Some Pods of DaemonSet are not scheduled or not ready\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes DaemonSet rollout stuck (daemonset {{ $labels.daemonset }})
|
||||
description: "Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are not scheduled or not ready\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesDaemonsetMisscheduled
|
||||
expr: 'kube_daemonset_status_number_misscheduled > 0'
|
||||
|
|
@ -244,8 +245,8 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes DaemonSet misscheduled (instance {{ $labels.instance }})
|
||||
description: "Some DaemonSet Pods are running where they are not supposed to run\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
summary: Kubernetes DaemonSet misscheduled (daemonset {{ $labels.daemonset }})
|
||||
description: "Some Pods of DaemonSet {{ $labels.namespace }}/{{ $labels.daemonset }} are running where they are not supposed to run\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesCronjobTooLong
|
||||
expr: 'time() - kube_cronjob_next_schedule_time > 3600'
|
||||
|
|
@ -253,7 +254,7 @@ groups:
|
|||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: Kubernetes CronJob too long (instance {{ $labels.instance }})
|
||||
summary: Kubernetes CronJob too long (cronjob {{ $labels.cronjob }})
|
||||
description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is taking more than 1h to complete.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesJobSlowCompletion
|
||||
|
|
@ -262,7 +263,7 @@ groups:
|
|||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: Kubernetes job slow completion (instance {{ $labels.instance }})
|
||||
summary: Kubernetes Job slow completion (job_name {{ $labels.job_name }})
|
||||
description: "Kubernetes Job {{ $labels.namespace }}/{{ $labels.job_name }} did not complete in time.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
||||
|
||||
- alert: KubernetesApiServerErrors
|
||||
|
|
|
|||
Loading…
Reference in a new issue