diff --git a/_data/rules.yml b/_data/rules.yml index 9670cfb..ffb4604 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1320,41 +1320,44 @@ groups: description: Disk space on default is below 20%. query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20" severity: warning + for: 2m - name: ClickHouse Disk Space Critical on Default description: Disk space on default disk is critically low, below 10%. query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10" severity: critical + for: 2m - name: ClickHouse Disk Space Low on Backups description: Disk space on backups is below 20%. query: "ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20" severity: warning + for: 2m - name: ClickHouse Replica Errors description: Critical replica errors detected, either all replicas are stale or lost. query: "ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1" severity: critical - for: 3m + for: 0m - name: ClickHouse No Available Replicas description: No available replicas in ClickHouse. query: "ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1" severity: critical - for: 3m + for: 0m - name: ClickHouse No Live Replicas description: There are too few live replicas available, risking data loss and service disruption. query: "ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1" severity: critical - for: 3m + for: 0m - name: ClickHouse High Network Traffic description: Network traffic is unusually high, may affect cluster performance. query: "ClickHouseMetrics_NetworkSend > 250 or ClickHouseMetrics_NetworkReceive > 250" severity: warning - for: 3m + for: 5m comments: | Please replace the threshold with an appropriate value - name: ClickHouse High TCP Connections description: High number of TCP connections, indicating heavy client or inter-cluster communication. query: "ClickHouseMetrics_TCPConnection > 400" severity: warning - for: 3m + for: 5m comments: | Please replace the threshold with an appropriate value - name: ClickHouse Interserver Connection Issues @@ -1366,15 +1369,17 @@ groups: description: ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination. query: "avg(ClickHouseMetrics_ZooKeeperSession) != 1" severity: warning - for: 1m + for: 3m - name: ClickHouse Authentication Failures description: Authentication failures detected, indicating potential security issues or misconfiguration. query: "increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 0" severity: info + for: 0m - name: ClickHouse Access Denied Errors description: Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts. query: "increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 0" severity: info + for: 0m - name: Zookeeper