From 7bced89d2d6084155e830420f6cb52c2f76de321 Mon Sep 17 00:00:00 2001 From: Sajjad hassanzadeh <32982356+Hassanzadeh-sd@users.noreply.github.com> Date: Fri, 29 Aug 2025 00:36:31 +0330 Subject: [PATCH 1/4] add : additional essential clickhouse alerts (#471) * add : additional essential clickhouse alerts * Add new ClickHouse alert rules for monitoring * linting --------- Co-authored-by: Samuel Berthe --- _data/rules.yml | 60 ++++++++++++++++----- dist/rules/clickhouse/embedded-exporter.yml | 54 +++++++++++++++++++ 2 files changed, 100 insertions(+), 14 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index b3df1bd..649f746 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -1338,80 +1338,112 @@ groups: slug: embedded-exporter doc_url: https://clickhouse.com/docs/en/operations/system-tables/metrics rules: + - name: ClickHouse node down + description: "No metrics received from ClickHouse exporter for over 2 minutes." + query: 'up{job="clickhouse"} == 0' + severity: critical + for: 2m - name: ClickHouse Memory Usage Critical - description: Memory usage is critically high, over 90%. + description: "Memory usage is critically high, over 90%." query: "ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90" severity: critical for: 5m - name: ClickHouse Memory Usage Warning - description: Memory usage is over 80%. + description: "Memory usage is over 80%." query: "ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80" severity: warning for: 5m - name: ClickHouse Disk Space Low on Default - description: Disk space on default is below 20%. + description: "Disk space on default is below 20%." query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20" severity: warning for: 2m - name: ClickHouse Disk Space Critical on Default - description: Disk space on default disk is critically low, below 10%. + description: "Disk space on default disk is critically low, below 10%." query: "ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10" severity: critical for: 2m - name: ClickHouse Disk Space Low on Backups - description: Disk space on backups is below 20%. + description: "Disk space on backups is below 20%." query: "ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20" severity: warning for: 2m - name: ClickHouse Replica Errors - description: Critical replica errors detected, either all replicas are stale or lost. + description: "Critical replica errors detected, either all replicas are stale or lost." query: "ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1" severity: critical for: 0m - name: ClickHouse No Available Replicas - description: No available replicas in ClickHouse. + description: "No available replicas in ClickHouse." query: "ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1" severity: critical for: 0m - name: ClickHouse No Live Replicas - description: There are too few live replicas available, risking data loss and service disruption. + description: "There are too few live replicas available, risking data loss and service disruption." query: "ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1" severity: critical for: 0m - name: ClickHouse High Network Traffic - description: Network traffic is unusually high, may affect cluster performance. + description: "Network traffic is unusually high, may affect cluster performance." query: "ClickHouseMetrics_NetworkSend > 250 or ClickHouseMetrics_NetworkReceive > 250" severity: warning for: 5m comments: | Please replace the threshold with an appropriate value - name: ClickHouse High TCP Connections - description: High number of TCP connections, indicating heavy client or inter-cluster communication. + description: "High number of TCP connections, indicating heavy client or inter-cluster communication." query: "ClickHouseMetrics_TCPConnection > 400" severity: warning for: 5m comments: | Please replace the threshold with an appropriate value - name: ClickHouse Interserver Connection Issues - description: An increase in interserver connections may indicate replication or distributed query handling issues. + description: "An increase in interserver connections may indicate replication or distributed query handling issues." query: "increase(ClickHouseMetrics_InterserverConnection[5m]) > 0" severity: warning for: 1m - name: ClickHouse ZooKeeper Connection Issues - description: ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination. + description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination." query: "avg(ClickHouseMetrics_ZooKeeperSession) != 1" severity: warning for: 3m - name: ClickHouse Authentication Failures - description: Authentication failures detected, indicating potential security issues or misconfiguration. + description: "Authentication failures detected, indicating potential security issues or misconfiguration." query: "increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 0" severity: info for: 0m - name: ClickHouse Access Denied Errors - description: Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts. + description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts." query: "increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 0" severity: info for: 0m + - name: ClickHouse rejected insert queries + description: "INSERTs rejected due to too many active data parts. Reduce insert frequency." + query: "increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0" + severity: warning + for: 1m + - name: ClickHouse delayed insert queries + description: "INSERTs delayed due to high number of active parts." + query: "increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0" + severity: warning + for: 2m + - name: ClickHouse zookeeper hardware exception + description: "Zookeeper hardware exception: network issues communicating with ZooKeeper" + query: "increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0" + severity: critical + for: 1m + - name: ClickHouse high network usage + description: High network usage. ClickHouse network usage exceeds 100MB/s. + query: "rate(ClickHouseProfileEvents_NetworkSendBytes[1m]) > 100*1024*1024 or rate(ClickHouseProfileEvents_NetworkReceiveBytes[1m]) > 100*1024*1024" + severity: warning + for: 2m + comments: | + Please replace the threshold with an appropriate value + - name: ClickHouse distributed rejected inserts + description: "INSERTs into Distributed tables rejected due to pending bytes limit." + query: "increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0" + severity: critical + for: 2m - name: Zookeeper exporters: diff --git a/dist/rules/clickhouse/embedded-exporter.yml b/dist/rules/clickhouse/embedded-exporter.yml index 3efe551..8380cfa 100644 --- a/dist/rules/clickhouse/embedded-exporter.yml +++ b/dist/rules/clickhouse/embedded-exporter.yml @@ -129,3 +129,57 @@ groups: annotations: summary: ClickHouse Access Denied Errors (instance {{ $labels.instance }}) description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickHouseExporterDown + expr: up{job="clickhouse"} == 0 + for: 1m + labels: + severity: critical + annotations: + summary: "ClickHouse exporter is down (instance {{ $labels.instance }})" + description: "No metrics received from ClickHouse exporter for over 1 minute." + + - alert: ClickHouseRejectedInserts + expr: increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Rejected INSERT queries (instance {{ $labels.instance }})" + description: "INSERTs rejected due to too many active data parts. Reduce insert frequency." + + - alert: ClickHouseDelayedInserts + expr: increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Delayed INSERTs detected (instance {{ $labels.instance }})" + description: "INSERTs delayed due to high number of active parts." + + - alert: ClickHouseZooKeeperHardwareExceptions + expr: increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "ZooKeeper hardware exceptions (instance {{ $labels.instance }})" + description: "Network issues communicating with ZooKeeper." + + - alert: ClickHouseHighNetworkUsage + expr: rate(ClickHouseProfileEvents_NetworkSendBytes[1m]) > 50000000 or rate(ClickHouseProfileEvents_NetworkReceiveBytes[1m]) > 50000000 + for: 2m + labels: + severity: warning + annotations: + summary: "High network usage (instance {{ $labels.instance }})" + description: "ClickHouse network usage exceeds 50MB/s." + + - alert: ClickHouseDistributedRejectedInserts + expr: increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0 + for: 1m + labels: + severity: critical + annotations: + summary: "Distributed rejected inserts (instance {{ $labels.instance }})" + description: "INSERTs into Distributed tables rejected due to pending bytes limit." From 3abc7144aacd16a8907a3994d5417e5a6d4ba5ad Mon Sep 17 00:00:00 2001 From: samber Date: Thu, 28 Aug 2025 21:07:00 +0000 Subject: [PATCH 2/4] Publish --- dist/rules/clickhouse/embedded-exporter.yml | 62 ++++++++++----------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/dist/rules/clickhouse/embedded-exporter.yml b/dist/rules/clickhouse/embedded-exporter.yml index 8380cfa..a4cd2a7 100644 --- a/dist/rules/clickhouse/embedded-exporter.yml +++ b/dist/rules/clickhouse/embedded-exporter.yml @@ -4,6 +4,15 @@ groups: rules: + - alert: ClickhouseNodeDown + expr: 'up{job="clickhouse"} == 0' + for: 2m + labels: + severity: critical + annotations: + summary: ClickHouse node down (instance {{ $labels.instance }}) + description: "No metrics received from ClickHouse exporter for over 2 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: ClickhouseMemoryUsageCritical expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90' for: 5m @@ -130,56 +139,47 @@ groups: summary: ClickHouse Access Denied Errors (instance {{ $labels.instance }}) description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: ClickHouseExporterDown - expr: up{job="clickhouse"} == 0 + - alert: ClickhouseRejectedInsertQueries + expr: 'increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0' for: 1m labels: - severity: critical + severity: warning annotations: - summary: "ClickHouse exporter is down (instance {{ $labels.instance }})" - description: "No metrics received from ClickHouse exporter for over 1 minute." + summary: ClickHouse rejected insert queries (instance {{ $labels.instance }}) + description: "INSERTs rejected due to too many active data parts. Reduce insert frequency.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: ClickHouseRejectedInserts - expr: increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "Rejected INSERT queries (instance {{ $labels.instance }})" - description: "INSERTs rejected due to too many active data parts. Reduce insert frequency." - - - alert: ClickHouseDelayedInserts - expr: increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0 + - alert: ClickhouseDelayedInsertQueries + expr: 'increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0' for: 2m labels: severity: warning annotations: - summary: "Delayed INSERTs detected (instance {{ $labels.instance }})" - description: "INSERTs delayed due to high number of active parts." + summary: ClickHouse delayed insert queries (instance {{ $labels.instance }}) + description: "INSERTs delayed due to high number of active parts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: ClickHouseZooKeeperHardwareExceptions - expr: increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0 + - alert: ClickhouseZookeeperHardwareException + expr: 'increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0' for: 1m labels: severity: critical annotations: - summary: "ZooKeeper hardware exceptions (instance {{ $labels.instance }})" - description: "Network issues communicating with ZooKeeper." + summary: ClickHouse zookeeper hardware exception (instance {{ $labels.instance }}) + description: "Zookeeper hardware exception: network issues communicating with ZooKeeper\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: ClickHouseHighNetworkUsage - expr: rate(ClickHouseProfileEvents_NetworkSendBytes[1m]) > 50000000 or rate(ClickHouseProfileEvents_NetworkReceiveBytes[1m]) > 50000000 + - alert: ClickhouseHighNetworkUsage + expr: 'rate(ClickHouseProfileEvents_NetworkSendBytes[1m]) > 100*1024*1024 or rate(ClickHouseProfileEvents_NetworkReceiveBytes[1m]) > 100*1024*1024' for: 2m labels: severity: warning annotations: - summary: "High network usage (instance {{ $labels.instance }})" - description: "ClickHouse network usage exceeds 50MB/s." + summary: ClickHouse high network usage (instance {{ $labels.instance }}) + description: "High network usage. ClickHouse network usage exceeds 100MB/s.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: ClickHouseDistributedRejectedInserts - expr: increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0 - for: 1m + - alert: ClickhouseDistributedRejectedInserts + expr: 'increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0' + for: 2m labels: severity: critical annotations: - summary: "Distributed rejected inserts (instance {{ $labels.instance }})" - description: "INSERTs into Distributed tables rejected due to pending bytes limit." + summary: ClickHouse distributed rejected inserts (instance {{ $labels.instance }}) + description: "INSERTs into Distributed tables rejected due to pending bytes limit.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" From 0a55137e6ae33b13a4841a6bb2d87322ebe7a4e0 Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Fri, 29 Aug 2025 15:20:21 +0200 Subject: [PATCH 3/4] Remove Screeb --- blackbox-exporter.md | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/blackbox-exporter.md b/blackbox-exporter.md index c4e5952..4782d36 100644 --- a/blackbox-exporter.md +++ b/blackbox-exporter.md @@ -9,14 +9,14 @@ You should deploy blackbox exporters in multiple Point of Presence around the globe, to monitor latency. Feel free to use the following endpoints for your own projects: -- https://screeb-probe-montreal.cleverapps.io -- https://screeb-probe-paris.cleverapps.io -- https://screeb-probe-jeddah.cleverapps.io -- https://screeb-probe-singapore.cleverapps.io -- https://screeb-probe-sydney.cleverapps.io -- https://screeb-probe-warsaw.cleverapps.io +- https://probe-montreal.cleverapps.io +- https://probe-paris.cleverapps.io +- https://probe-jeddah.cleverapps.io +- https://probe-singapore.cleverapps.io +- https://probe-sydney.cleverapps.io +- https://probe-warsaw.cleverapps.io -☝️ Logs have been disabled. More probes from the community would be appreciated, please contribute here! These blackbox exporters use the following configuration. +☝️ Logs have been disabled. More probes from the community would be appreciated, please contribute here! These blackbox exporters use the following configuration. ## Prometheus Configuration @@ -30,33 +30,33 @@ Blackbox exporters and endpoints must be declared in Prometheus. Here is a simpl # Montreal # # http - - screeb-probe-montreal.cleverapps.io:_:http_2xx:_:Montreal:_:f229cy:_:https://api.screeb.app - - screeb-probe-montreal.cleverapps.io:_:http_2xx:_:Montreal:_:f229cy:_:https://t.screeb.app/tag.js + - probe-montreal.cleverapps.io:_:http_2xx:_:Montreal:_:f229cy:_:https://api.screeb.app + - probe-montreal.cleverapps.io:_:http_2xx:_:Montreal:_:f229cy:_:https://t.screeb.app/tag.js # icmp - - screeb-probe-montreal.cleverapps.io:_:icmp_ipv4:_:Montreal:_:f229cy:_:api.screeb.app - - screeb-probe-montreal.cleverapps.io:_:icmp_ipv4:_:Montreal:_:f229cy:_:t.screeb.app + - probe-montreal.cleverapps.io:_:icmp_ipv4:_:Montreal:_:f229cy:_:api.screeb.app + - probe-montreal.cleverapps.io:_:icmp_ipv4:_:Montreal:_:f229cy:_:t.screeb.app # # Paris # # http - - screeb-probe-paris.cleverapps.io:_:http_2xx:_:Paris:_:u09tgy:_:https://api.screeb.app - - screeb-probe-paris.cleverapps.io:_:http_2xx:_:Paris:_:u09tgy:_:https://t.screeb.app/tag.js + - probe-paris.cleverapps.io:_:http_2xx:_:Paris:_:u09tgy:_:https://api.screeb.app + - probe-paris.cleverapps.io:_:http_2xx:_:Paris:_:u09tgy:_:https://t.screeb.app/tag.js # icmp - - screeb-probe-paris.cleverapps.io:_:icmp_ipv4:_:Paris:_:u09tgy:_:api.screeb.app - - screeb-probe-paris.cleverapps.io:_:icmp_ipv4:_:Paris:_:u09tgy:_:t.screeb.app + - probe-paris.cleverapps.io:_:icmp_ipv4:_:Paris:_:u09tgy:_:api.screeb.app + - probe-paris.cleverapps.io:_:icmp_ipv4:_:Paris:_:u09tgy:_:t.screeb.app # # Sydney # # http - - screeb-probe-sydney.cleverapps.io:_:http_2xx:_:Sydney:_:r3gpkn:_:https://api.screeb.app - - screeb-probe-sydney.cleverapps.io:_:http_2xx:_:Sydney:_:r3gpkn:_:https://t.screeb.app/tag.js + - probe-sydney.cleverapps.io:_:http_2xx:_:Sydney:_:r3gpkn:_:https://api.screeb.app + - probe-sydney.cleverapps.io:_:http_2xx:_:Sydney:_:r3gpkn:_:https://t.screeb.app/tag.js # icmp - - screeb-probe-sydney.cleverapps.io:_:icmp_ipv4:_:Sydney:_:r3gpkn:_:api.screeb.app - - screeb-probe-sydney.cleverapps.io:_:icmp_ipv4:_:Sydney:_:r3gpkn:_:t.screeb.app + - probe-sydney.cleverapps.io:_:icmp_ipv4:_:Sydney:_:r3gpkn:_:api.screeb.app + - probe-sydney.cleverapps.io:_:icmp_ipv4:_:Sydney:_:r3gpkn:_:t.screeb.app # ... ``` From edae18b8dfdbc93c9656458260a7f90b100c173e Mon Sep 17 00:00:00 2001 From: Samuel Berthe Date: Fri, 29 Aug 2025 15:20:48 +0200 Subject: [PATCH 4/4] Remove Screeb tag --- _layouts/default.html | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/_layouts/default.html b/_layouts/default.html index 56f4a92..3f2fe1c 100644 --- a/_layouts/default.html +++ b/_layouts/default.html @@ -152,19 +152,6 @@ - - - - -