diff --git a/_data/rules.yml b/_data/rules.yml index 51edb10..9670cfb 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1310,10 +1310,12 @@ groups: description: Memory usage is critically high, over 90%. query: "ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90" severity: critical + for: 5m - name: ClickHouse Memory Usage Warning description: Memory usage is over 80%. query: "ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80" severity: warning + for: 5m - name: ClickHouse Disk Space Low on Default description: Disk space on default is below 20%. query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20" @@ -1330,42 +1332,49 @@ groups: description: Critical replica errors detected, either all replicas are stale or lost. query: "ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1" severity: critical + for: 3m - name: ClickHouse No Available Replicas description: No available replicas in ClickHouse. query: "ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1" severity: critical + for: 3m - name: ClickHouse No Live Replicas description: There are too few live replicas available, risking data loss and service disruption. query: "ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1" severity: critical + for: 3m - name: ClickHouse High Network Traffic description: Network traffic is unusually high, may affect cluster performance. - query: "ClickHouseMetrics_NetworkSend > 1000 or ClickHouseMetrics_NetworkReceive > 1000" + query: "ClickHouseMetrics_NetworkSend > 250 or ClickHouseMetrics_NetworkReceive > 250" severity: warning + for: 3m comments: | Please replace the threshold with an appropriate value - name: ClickHouse High TCP Connections description: High number of TCP connections, indicating heavy client or inter-cluster communication. - query: "ClickHouseMetrics_TCPConnection > 1500" + query: "ClickHouseMetrics_TCPConnection > 400" severity: warning + for: 3m comments: | Please replace the threshold with an appropriate value - name: ClickHouse Interserver Connection Issues description: An increase in interserver connections may indicate replication or distributed query handling issues. query: "increase(ClickHouseMetrics_InterserverConnection[5m]) > 0" severity: warning + for: 1m - name: ClickHouse ZooKeeper Connection Issues description: ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination. query: "avg(ClickHouseMetrics_ZooKeeperSession) != 1" severity: warning + for: 1m - name: ClickHouse Authentication Failures description: Authentication failures detected, indicating potential security issues or misconfiguration. query: "increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 0" - severity: critical + severity: info - name: ClickHouse Access Denied Errors description: Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts. query: "increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 0" - severity: critical + severity: info - name: Zookeeper