XeLabs/awesome-prometheus-alerts
2
0
Fork 0
mirror of https://github.com/samber/awesome-prometheus-alerts.git synced 2026-06-25 02:46:59 +08:00
Code Issues Projects Releases Packages Wiki Activity

Merge branch 'master' into add-couchdb-alerts

Browse source
This commit is contained in:
Samuel Berthe 2025-09-01 15:39:44 +02:00 committed by GitHub
commit 6fd64e8c5e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 19 additions and 232 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
_data/rules.yml
View file

@ -1456,103 +1456,86 @@ groups:
query: "couchdb_httpd_node_up == 0 or couchdb_httpd_up == 0"
severity: critical
for: 2m
- name: CouchDB atom memory usage critical
description: Atom memory usage is above 90% of limit
query: "couchdb_erlang_memory_atom_used > 0.9 * couchdb_erlang_memory_atom"
severity: critical
for: 5m
- name: CouchDB open databases critical
description: Number of open databases exceeds 90% of node capacity
query: "couchdb_httpd_open_databases > 0.9 * 1000"
severity: critical
for: 5m
- name: CouchDB open OS files critical
description: CouchDB is using more than 90% of allowed OS file descriptors, may fail to open new files
query: "couchdb_httpd_open_os_files > 0.9 * 65535"
severity: critical
for: 5m
- name: CouchDB 5xx error ratio high
description: More than 5% of HTTP requests are returning 5xx errors
query: "rate(couchdb_httpd_status_codes{code=~\"5..\"}[5m]) / rate(couchdb_httpd_requests[5m]) > 0.05"
severity: critical
for: 5m
- name: CouchDB temporary view read rate critical
description: Temporary view read rate exceeds 100 reads/sec, high risk of performance degradation
query: "rate(couchdb_httpd_temporary_view_reads[5m]) > 100"
severity: critical
for: 5m
- name: CouchDB Mango queries scanning too many docs
description: Some Mango queries are scanning too many documents, consider adding indexes
query: "rate(couchdb_mango_too_many_docs_scanned[5m]) > 50"
severity: warning
for: 5m
- name: CouchDB Mango queries failed due to invalid index
description: Some Mango queries failed to execute because the index was missing or invalid
query: "rate(couchdb_mango_query_invalid_index[5m]) > 5"
severity: warning
for: 5m
- name: CouchDB Mango docs examined high
description: High number of documents examined per Mango queries, consider indexing
query: "rate(couchdb_mango_docs_examined[5m]) > 1000"
severity: warning
for: 5m
- name: CouchDB Replicator manager died
description: Replication manager process has crashed
query: "increase(couchdb_replicator_changes_manager_deaths[5m]) > 0"
severity: critical
for: 1m
- name: CouchDB Replicator queue process died
description: Replication queue process has crashed
query: "increase(couchdb_replicator_changes_queue_deaths[5m]) > 0"
severity: critical
for: 1m
- name: CouchDB Replicator reader process died
description: Replication reader process has crashed
query: "increase(couchdb_replicator_changes_reader_deaths[5m]) > 0"
severity: critical
for: 1m
- name: CouchDB Replicator failed to start
description: One or more replication tasks failed to start
query: "increase(couchdb_replicator_failed_starts[5m]) > 0"
severity: critical
for: 1m
- name: CouchDB replication cluster unstable
description: The replication cluster is unstable, replication may be interrupted
query: "couchdb_replicator_cluster_is_stable == 0"
severity: critical
for: 2m
- name: CouchDB replication read failures
description: Replication changes feed has failed reads more than 5 times in 5 minutes
query: "increase(couchdb_replicator_changes_read_failures[5m]) > 5"
severity: warning
for: 5m
- name: CouchDB file descriptors high
description: Process is using more than 85% of allowed file descriptors
query: "process_open_fds / process_max_fds > 0.85"
severity: warning
for: 5m
- name: CouchDB process restarted
description: CouchDB process has restarted recently
query: "changes(process_start_time_seconds[1h]) > 0"
severity: critical
for: 1m
- name: CouchDB critical log entries
description: Critical or error log entries detected in the last 5 minutes
query: "increase(couchdb_server_couch_log{level=~\"error|critical\"}[5m]) > 0"

13
_layouts/default.html
View file

@ -152,19 +152,6 @@
</footer>
</main>
<!-- Screeb tag -->
<script type="text/javascript">
(function (s,c,r,ee,b) {
s['ScreebObject']=r;s[r]=s[r]||function(){(s[r].q=s[r].q||[]).push(arguments)};
b=c.createElement('script');b.type='text/javascript';
b.id=r;b.src=ee;b.async=1;c.getElementsByTagName("head")[0].appendChild(b);
}(window,document,'$screeb','https://t2.screeb.app/tag.js'));
$screeb('init', '232450e3-d3fe-4240-b543-649a5041a7db');
</script>
<!-- End of Screeb tag -->
</body>
</html>

38
blackbox-exporter.md
View file

@ -9,14 +9,14 @@
You should deploy blackbox exporters in multiple Point of Presence around the globe, to monitor latency. Feel free to use the following endpoints for your own projects:
- https://screeb-probe-<b>montreal</b>.cleverapps.io
- https://screeb-probe-<b>paris</b>.cleverapps.io
- https://screeb-probe-<b>jeddah</b>.cleverapps.io
- https://screeb-probe-<b>singapore</b>.cleverapps.io
- https://screeb-probe-<b>sydney</b>.cleverapps.io
- https://screeb-probe-<b>warsaw</b>.cleverapps.io
- https://probe-<b>montreal</b>.cleverapps.io
- https://probe-<b>paris</b>.cleverapps.io
- https://probe-<b>jeddah</b>.cleverapps.io
- https://probe-<b>singapore</b>.cleverapps.io
- https://probe-<b>sydney</b>.cleverapps.io
- https://probe-<b>warsaw</b>.cleverapps.io
☝️ Logs have been disabled. More probes from the community would be appreciated, please contribute <a href="https://github.com/samber/awesome-prometheus-alerts/" target="_blank">here</a>! These blackbox exporters use the following <a href="https://github.com/ScreebApp/blackbox_exporter/blob/master/screeb.yml" target="_blank">configuration</a>.
☝️ Logs have been disabled. More probes from the community would be appreciated, please contribute <a href="https://github.com/samber/awesome-prometheus-alerts/" target="_blank">here</a>! These blackbox exporters use the following <a href="https://github.com/samber/blackbox_exporter/blob/master/samber.yml" target="_blank">configuration</a>.
## Prometheus Configuration
@ -30,33 +30,33 @@ Blackbox exporters and endpoints must be declared in Prometheus. Here is a simpl
# Montreal
#
# http
- screeb-probe-montreal.cleverapps.io:_:http_2xx:_:Montreal:_:f229cy:_:https://api.screeb.app
- screeb-probe-montreal.cleverapps.io:_:http_2xx:_:Montreal:_:f229cy:_:https://t.screeb.app/tag.js
- probe-montreal.cleverapps.io:_:http_2xx:_:Montreal:_:f229cy:_:https://api.screeb.app
- probe-montreal.cleverapps.io:_:http_2xx:_:Montreal:_:f229cy:_:https://t.screeb.app/tag.js
# icmp
- screeb-probe-montreal.cleverapps.io:_:icmp_ipv4:_:Montreal:_:f229cy:_:api.screeb.app
- screeb-probe-montreal.cleverapps.io:_:icmp_ipv4:_:Montreal:_:f229cy:_:t.screeb.app
- probe-montreal.cleverapps.io:_:icmp_ipv4:_:Montreal:_:f229cy:_:api.screeb.app
- probe-montreal.cleverapps.io:_:icmp_ipv4:_:Montreal:_:f229cy:_:t.screeb.app
#
# Paris
#
# http
- screeb-probe-paris.cleverapps.io:_:http_2xx:_:Paris:_:u09tgy:_:https://api.screeb.app
- screeb-probe-paris.cleverapps.io:_:http_2xx:_:Paris:_:u09tgy:_:https://t.screeb.app/tag.js
- probe-paris.cleverapps.io:_:http_2xx:_:Paris:_:u09tgy:_:https://api.screeb.app
- probe-paris.cleverapps.io:_:http_2xx:_:Paris:_:u09tgy:_:https://t.screeb.app/tag.js
# icmp
- screeb-probe-paris.cleverapps.io:_:icmp_ipv4:_:Paris:_:u09tgy:_:api.screeb.app
- screeb-probe-paris.cleverapps.io:_:icmp_ipv4:_:Paris:_:u09tgy:_:t.screeb.app
- probe-paris.cleverapps.io:_:icmp_ipv4:_:Paris:_:u09tgy:_:api.screeb.app
- probe-paris.cleverapps.io:_:icmp_ipv4:_:Paris:_:u09tgy:_:t.screeb.app
#
# Sydney
#
# http
- screeb-probe-sydney.cleverapps.io:_:http_2xx:_:Sydney:_:r3gpkn:_:https://api.screeb.app
- screeb-probe-sydney.cleverapps.io:_:http_2xx:_:Sydney:_:r3gpkn:_:https://t.screeb.app/tag.js
- probe-sydney.cleverapps.io:_:http_2xx:_:Sydney:_:r3gpkn:_:https://api.screeb.app
- probe-sydney.cleverapps.io:_:http_2xx:_:Sydney:_:r3gpkn:_:https://t.screeb.app/tag.js
# icmp
- screeb-probe-sydney.cleverapps.io:_:icmp_ipv4:_:Sydney:_:r3gpkn:_:api.screeb.app
- screeb-probe-sydney.cleverapps.io:_:icmp_ipv4:_:Sydney:_:r3gpkn:_:t.screeb.app
- probe-sydney.cleverapps.io:_:icmp_ipv4:_:Sydney:_:r3gpkn:_:api.screeb.app
- probe-sydney.cleverapps.io:_:icmp_ipv4:_:Sydney:_:r3gpkn:_:t.screeb.app
# ...
```

183
dist/rules/clickhouse/embedded-exporter.yml vendored
View file

@ -1,185 +1,2 @@
groups:
- name: EmbeddedExporter
rules:
- alert: ClickhouseMemoryUsageCritical
expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90'
for: 5m
labels:
severity: critical
annotations:
summary: ClickHouse Memory Usage Critical (instance {{ $labels.instance }})
description: "Memory usage is critically high, over 90%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseMemoryUsageWarning
expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80'
for: 5m
labels:
severity: warning
annotations:
summary: ClickHouse Memory Usage Warning (instance {{ $labels.instance }})
description: "Memory usage is over 80%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseDiskSpaceLowOnDefault
expr: 'ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20'
for: 2m
labels:
severity: warning
annotations:
summary: ClickHouse Disk Space Low on Default (instance {{ $labels.instance }})
description: "Disk space on default is below 20%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseDiskSpaceCriticalOnDefault
expr: 'ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10'
for: 2m
labels:
severity: critical
annotations:
summary: ClickHouse Disk Space Critical on Default (instance {{ $labels.instance }})
description: "Disk space on default disk is critically low, below 10%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseDiskSpaceLowOnBackups
expr: 'ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20'
for: 2m
labels:
severity: warning
annotations:
summary: ClickHouse Disk Space Low on Backups (instance {{ $labels.instance }})
description: "Disk space on backups is below 20%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseReplicaErrors
expr: 'ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1'
for: 0m
labels:
severity: critical
annotations:
summary: ClickHouse Replica Errors (instance {{ $labels.instance }})
description: "Critical replica errors detected, either all replicas are stale or lost.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseNoAvailableReplicas
expr: 'ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1'
for: 0m
labels:
severity: critical
annotations:
summary: ClickHouse No Available Replicas (instance {{ $labels.instance }})
description: "No available replicas in ClickHouse.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseNoLiveReplicas
expr: 'ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1'
for: 0m
labels:
severity: critical
annotations:
summary: ClickHouse No Live Replicas (instance {{ $labels.instance }})
description: "There are too few live replicas available, risking data loss and service disruption.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseHighNetworkTraffic
expr: 'ClickHouseMetrics_NetworkSend > 250 or ClickHouseMetrics_NetworkReceive > 250'
for: 5m
labels:
severity: warning
annotations:
summary: ClickHouse High Network Traffic (instance {{ $labels.instance }})
description: "Network traffic is unusually high, may affect cluster performance.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseHighTcpConnections
expr: 'ClickHouseMetrics_TCPConnection > 400'
for: 5m
labels:
severity: warning
annotations:
summary: ClickHouse High TCP Connections (instance {{ $labels.instance }})
description: "High number of TCP connections, indicating heavy client or inter-cluster communication.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseInterserverConnectionIssues
expr: 'increase(ClickHouseMetrics_InterserverConnection[5m]) > 0'
for: 1m
labels:
severity: warning
annotations:
summary: ClickHouse Interserver Connection Issues (instance {{ $labels.instance }})
description: "An increase in interserver connections may indicate replication or distributed query handling issues.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseZookeeperConnectionIssues
expr: 'avg(ClickHouseMetrics_ZooKeeperSession) != 1'
for: 3m
labels:
severity: warning
annotations:
summary: ClickHouse ZooKeeper Connection Issues (instance {{ $labels.instance }})
description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseAuthenticationFailures
expr: 'increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 0'
for: 0m
labels:
severity: info
annotations:
summary: ClickHouse Authentication Failures (instance {{ $labels.instance }})
description: "Authentication failures detected, indicating potential security issues or misconfiguration.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickhouseAccessDeniedErrors
expr: 'increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 0'
for: 0m
labels:
severity: info
annotations:
summary: ClickHouse Access Denied Errors (instance {{ $labels.instance }})
description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
- alert: ClickHouseExporterDown
expr: up{job="clickhouse"} == 0
for: 1m
labels:
severity: critical
annotations:
summary: "ClickHouse exporter is down (instance {{ $labels.instance }})"
description: "No metrics received from ClickHouse exporter for over 1 minute."
- alert: ClickHouseRejectedInserts
expr: increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0
for: 1m
labels:
severity: critical
annotations:
summary: "Rejected INSERT queries (instance {{ $labels.instance }})"
description: "INSERTs rejected due to too many active data parts. Reduce insert frequency."
- alert: ClickHouseDelayedInserts
expr: increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0
for: 2m
labels:
severity: warning
annotations:
summary: "Delayed INSERTs detected (instance {{ $labels.instance }})"
description: "INSERTs delayed due to high number of active parts."
- alert: ClickHouseZooKeeperHardwareExceptions
expr: increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0
for: 1m
labels:
severity: critical
annotations:
summary: "ZooKeeper hardware exceptions (instance {{ $labels.instance }})"
description: "Network issues communicating with ZooKeeper."
- alert: ClickHouseHighNetworkUsage
expr: rate(ClickHouseProfileEvents_NetworkSendBytes[1m]) > 50000000 or rate(ClickHouseProfileEvents_NetworkReceiveBytes[1m]) > 50000000
for: 2m
labels:
severity: warning
annotations:
summary: "High network usage (instance {{ $labels.instance }})"
description: "ClickHouse network usage exceeds 50MB/s."
- alert: ClickHouseDistributedRejectedInserts
expr: increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0
for: 1m
labels:
severity: critical
annotations:
summary: "Distributed rejected inserts (instance {{ $labels.instance }})"
description: "INSERTs into Distributed tables rejected due to pending bytes limit."