This commit is contained in:
Samuel Berthe 2025-08-28 23:04:09 +02:00
parent 6f7d265fe1
commit 9842845411
No known key found for this signature in database
GPG key ID: 64863511FFBD0E3C

View file

@ -1339,96 +1339,96 @@ groups:
doc_url: https://clickhouse.com/docs/en/operations/system-tables/metrics doc_url: https://clickhouse.com/docs/en/operations/system-tables/metrics
rules: rules:
- name: ClickHouse node down - name: ClickHouse node down
description: No metrics received from ClickHouse exporter for over 2 minutes. description: "No metrics received from ClickHouse exporter for over 2 minutes."
query: "up{job="clickhouse"} == 0" query: 'up{job="clickhouse"} == 0'
severity: critical severity: critical
for: 2m for: 2m
- name: ClickHouse Memory Usage Critical - name: ClickHouse Memory Usage Critical
description: Memory usage is critically high, over 90%. description: "Memory usage is critically high, over 90%."
query: "ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90" query: "ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90"
severity: critical severity: critical
for: 5m for: 5m
- name: ClickHouse Memory Usage Warning - name: ClickHouse Memory Usage Warning
description: Memory usage is over 80%. description: "Memory usage is over 80%."
query: "ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80" query: "ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80"
severity: warning severity: warning
for: 5m for: 5m
- name: ClickHouse Disk Space Low on Default - name: ClickHouse Disk Space Low on Default
description: Disk space on default is below 20%. description: "Disk space on default is below 20%."
query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20" query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20"
severity: warning severity: warning
for: 2m for: 2m
- name: ClickHouse Disk Space Critical on Default - name: ClickHouse Disk Space Critical on Default
description: Disk space on default disk is critically low, below 10%. description: "Disk space on default disk is critically low, below 10%."
query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10" query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10"
severity: critical severity: critical
for: 2m for: 2m
- name: ClickHouse Disk Space Low on Backups - name: ClickHouse Disk Space Low on Backups
description: Disk space on backups is below 20%. description: "Disk space on backups is below 20%."
query: "ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20" query: "ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20"
severity: warning severity: warning
for: 2m for: 2m
- name: ClickHouse Replica Errors - name: ClickHouse Replica Errors
description: Critical replica errors detected, either all replicas are stale or lost. description: "Critical replica errors detected, either all replicas are stale or lost."
query: "ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1" query: "ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1"
severity: critical severity: critical
for: 0m for: 0m
- name: ClickHouse No Available Replicas - name: ClickHouse No Available Replicas
description: No available replicas in ClickHouse. description: "No available replicas in ClickHouse."
query: "ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1" query: "ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1"
severity: critical severity: critical
for: 0m for: 0m
- name: ClickHouse No Live Replicas - name: ClickHouse No Live Replicas
description: There are too few live replicas available, risking data loss and service disruption. description: "There are too few live replicas available, risking data loss and service disruption."
query: "ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1" query: "ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1"
severity: critical severity: critical
for: 0m for: 0m
- name: ClickHouse High Network Traffic - name: ClickHouse High Network Traffic
description: Network traffic is unusually high, may affect cluster performance. description: "Network traffic is unusually high, may affect cluster performance."
query: "ClickHouseMetrics_NetworkSend > 250 or ClickHouseMetrics_NetworkReceive > 250" query: "ClickHouseMetrics_NetworkSend > 250 or ClickHouseMetrics_NetworkReceive > 250"
severity: warning severity: warning
for: 5m for: 5m
comments: | comments: |
Please replace the threshold with an appropriate value Please replace the threshold with an appropriate value
- name: ClickHouse High TCP Connections - name: ClickHouse High TCP Connections
description: High number of TCP connections, indicating heavy client or inter-cluster communication. description: "High number of TCP connections, indicating heavy client or inter-cluster communication."
query: "ClickHouseMetrics_TCPConnection > 400" query: "ClickHouseMetrics_TCPConnection > 400"
severity: warning severity: warning
for: 5m for: 5m
comments: | comments: |
Please replace the threshold with an appropriate value Please replace the threshold with an appropriate value
- name: ClickHouse Interserver Connection Issues - name: ClickHouse Interserver Connection Issues
description: An increase in interserver connections may indicate replication or distributed query handling issues. description: "An increase in interserver connections may indicate replication or distributed query handling issues."
query: "increase(ClickHouseMetrics_InterserverConnection[5m]) > 0" query: "increase(ClickHouseMetrics_InterserverConnection[5m]) > 0"
severity: warning severity: warning
for: 1m for: 1m
- name: ClickHouse ZooKeeper Connection Issues - name: ClickHouse ZooKeeper Connection Issues
description: ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination. description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination."
query: "avg(ClickHouseMetrics_ZooKeeperSession) != 1" query: "avg(ClickHouseMetrics_ZooKeeperSession) != 1"
severity: warning severity: warning
for: 3m for: 3m
- name: ClickHouse Authentication Failures - name: ClickHouse Authentication Failures
description: Authentication failures detected, indicating potential security issues or misconfiguration. description: "Authentication failures detected, indicating potential security issues or misconfiguration."
query: "increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 0" query: "increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 0"
severity: info severity: info
for: 0m for: 0m
- name: ClickHouse Access Denied Errors - name: ClickHouse Access Denied Errors
description: Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts. description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts."
query: "increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 0" query: "increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 0"
severity: info severity: info
for: 0m for: 0m
- name: ClickHouse rejected insert queries - name: ClickHouse rejected insert queries
description: INSERTs rejected due to too many active data parts. Reduce insert frequency. description: "INSERTs rejected due to too many active data parts. Reduce insert frequency."
query: "increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0" query: "increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0"
severity: warning severity: warning
for: 1m for: 1m
- name: ClickHouse delayed insert queries - name: ClickHouse delayed insert queries
description: INSERTs delayed due to high number of active parts. description: "INSERTs delayed due to high number of active parts."
query: "increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0" query: "increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0"
severity: warning severity: warning
for: 2m for: 2m
- name: ClickHouse zookeeper hardware exception - name: ClickHouse zookeeper hardware exception
description: Zookeeper hardware exception: network issues communicating with ZooKeeper description: "Zookeeper hardware exception: network issues communicating with ZooKeeper"
query: "increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0" query: "increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0"
severity: critical severity: critical
for: 1m for: 1m
@ -1440,11 +1440,10 @@ groups:
comments: | comments: |
Please replace the threshold with an appropriate value Please replace the threshold with an appropriate value
- name: ClickHouse distributed rejected inserts - name: ClickHouse distributed rejected inserts
description: INSERTs into Distributed tables rejected due to pending bytes limit. description: "INSERTs into Distributed tables rejected due to pending bytes limit."
query: "increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0" query: "increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0"
severity: critical severity: critical
for: 2m for: 2m
- name: Zookeeper - name: Zookeeper
exporters: exporters: