Publish

2026-06-21 00:47:18 +08:00 · 2025-09-01 13:41:06 +00:00 · 2025-09-01 13:41:06 +00:00 · 5fbce5f513
commit 5fbce5f513
parent a2c31358d1
2 changed files with 332 additions and 146 deletions
--- a/dist/rules/clickhouse/embedded-exporter.yml
+++ b/dist/rules/clickhouse/embedded-exporter.yml
@ -1,2 +1,185 @@
+groups:

+- name: EmbeddedExporter

+  rules:
+
+    - alert: ClickhouseNodeDown
+      expr: 'up{job="clickhouse"} == 0'
+      for: 2m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClickHouse node down (instance {{ $labels.instance }})
+        description: "No metrics received from ClickHouse exporter for over 2 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseMemoryUsageCritical
+      expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90'
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClickHouse Memory Usage Critical (instance {{ $labels.instance }})
+        description: "Memory usage is critically high, over 90%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseMemoryUsageWarning
+      expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse Memory Usage Warning (instance {{ $labels.instance }})
+        description: "Memory usage is over 80%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseDiskSpaceLowOnDefault
+      expr: 'ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse Disk Space Low on Default (instance {{ $labels.instance }})
+        description: "Disk space on default is below 20%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseDiskSpaceCriticalOnDefault
+      expr: 'ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10'
+      for: 2m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClickHouse Disk Space Critical on Default (instance {{ $labels.instance }})
+        description: "Disk space on default disk is critically low, below 10%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseDiskSpaceLowOnBackups
+      expr: 'ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse Disk Space Low on Backups (instance {{ $labels.instance }})
+        description: "Disk space on backups is below 20%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseReplicaErrors
+      expr: 'ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClickHouse Replica Errors (instance {{ $labels.instance }})
+        description: "Critical replica errors detected, either all replicas are stale or lost.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseNoAvailableReplicas
+      expr: 'ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClickHouse No Available Replicas (instance {{ $labels.instance }})
+        description: "No available replicas in ClickHouse.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseNoLiveReplicas
+      expr: 'ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1'
+      for: 0m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClickHouse No Live Replicas (instance {{ $labels.instance }})
+        description: "There are too few live replicas available, risking data loss and service disruption.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseHighNetworkTraffic
+      expr: 'ClickHouseMetrics_NetworkSend > 250 or ClickHouseMetrics_NetworkReceive > 250'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse High Network Traffic (instance {{ $labels.instance }})
+        description: "Network traffic is unusually high, may affect cluster performance.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseHighTcpConnections
+      expr: 'ClickHouseMetrics_TCPConnection > 400'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse High TCP Connections (instance {{ $labels.instance }})
+        description: "High number of TCP connections, indicating heavy client or inter-cluster communication.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseInterserverConnectionIssues
+      expr: 'increase(ClickHouseMetrics_InterserverConnection[5m]) > 0'
+      for: 1m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse Interserver Connection Issues (instance {{ $labels.instance }})
+        description: "An increase in interserver connections may indicate replication or distributed query handling issues.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseZookeeperConnectionIssues
+      expr: 'avg(ClickHouseMetrics_ZooKeeperSession) != 1'
+      for: 3m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse ZooKeeper Connection Issues (instance {{ $labels.instance }})
+        description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseAuthenticationFailures
+      expr: 'increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 0'
+      for: 0m
+      labels:
+        severity: info
+      annotations:
+        summary: ClickHouse Authentication Failures (instance {{ $labels.instance }})
+        description: "Authentication failures detected, indicating potential security issues or misconfiguration.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseAccessDeniedErrors
+      expr: 'increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 0'
+      for: 0m
+      labels:
+        severity: info
+      annotations:
+        summary: ClickHouse Access Denied Errors (instance {{ $labels.instance }})
+        description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseRejectedInsertQueries
+      expr: 'increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0'
+      for: 1m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse rejected insert queries (instance {{ $labels.instance }})
+        description: "INSERTs rejected due to too many active data parts. Reduce insert frequency.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseDelayedInsertQueries
+      expr: 'increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse delayed insert queries (instance {{ $labels.instance }})
+        description: "INSERTs delayed due to high number of active parts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseZookeeperHardwareException
+      expr: 'increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0'
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClickHouse zookeeper hardware exception (instance {{ $labels.instance }})
+        description: "Zookeeper hardware exception: network issues communicating with ZooKeeper\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseHighNetworkUsage
+      expr: 'rate(ClickHouseProfileEvents_NetworkSendBytes[1m]) > 100*1024*1024 or rate(ClickHouseProfileEvents_NetworkReceiveBytes[1m]) > 100*1024*1024'
+      for: 2m
+      labels:
+        severity: warning
+      annotations:
+        summary: ClickHouse high network usage (instance {{ $labels.instance }})
+        description: "High network usage. ClickHouse network usage exceeds 100MB/s.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: ClickhouseDistributedRejectedInserts
+      expr: 'increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0'
+      for: 2m
+      labels:
+        severity: critical
+      annotations:
+        summary: ClickHouse distributed rejected inserts (instance {{ $labels.instance }})
+        description: "INSERTs into Distributed tables rejected due to pending bytes limit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
--- a/dist/rules/couchdb/gesellix-couchdb-prometheus-exporter.yml
+++ b/dist/rules/couchdb/gesellix-couchdb-prometheus-exporter.yml
@ -1,164 +1,167 @@
 groups:
-  - name: couchdb-alerts
-    rules:
-      - alert: CouchDBNodeDown
-        expr: expr: couchdb_httpd_node_up == 0 or couchdb_httpd_up == 0
-        for: 2m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB node down"
-          description: "CouchDB node is not responding (node_up metric is 0) for more than 2 minutes."

-      - alert: CouchDBAtomMemoryHigh
-        expr: couchdb_erlang_memory_atom_used > 0.9 * couchdb_erlang_memory_atom
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB atom memory usage critical"
-          description: "Atom memory usage is above 90% of limit."
+- name: GesellixCouchdbPrometheusExporter

-      - alert: CouchDBOpenDatabasesCritical
-        expr: couchdb_httpd_open_databases > 0.9 * 1000
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB open databases critical"
-          description: "Number of open databases exceeds 90% of node capacity."
+  rules:

-      - alert: CouchDBOpenOSFilesCritical
-        expr: couchdb_httpd_open_os_files > 0.9 * 65535
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB open OS files critical"
-          description: "CouchDB is using more than 90% of allowed OS file descriptors, may fail to open new files."
+    - alert: CouchdbNodeDown
+      expr: 'couchdb_httpd_node_up == 0 or couchdb_httpd_up == 0'
+      for: 2m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB node down (instance {{ $labels.instance }})
+        description: "CouchDB node is not responding (node_up metric is 0) for more than 2 minutes\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDB5xxErrorsHigh
-        expr: rate(couchdb_httpd_status_codes{code=~"5.."}[5m]) / rate(couchdb_httpd_requests[5m]) > 0.05
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB 5xx error ratio high"
-          description: "More than 5% of HTTP requests are returning 5xx errors."
+    - alert: CouchdbAtomMemoryUsageCritical
+      expr: 'couchdb_erlang_memory_atom_used > 0.9 * couchdb_erlang_memory_atom'
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB atom memory usage critical (instance {{ $labels.instance }})
+        description: "Atom memory usage is above 90% of limit\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBTemporaryViewReadsCritical
-        expr: rate(couchdb_httpd_temporary_view_reads[5m]) > 100
-        for: 5m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB temporary view read rate critical"
-          description: "Temporary view read rate exceeds 100 reads/sec, high risk of performance degradation."
+    - alert: CouchdbOpenDatabasesCritical
+      expr: 'couchdb_httpd_open_databases > 0.9 * 1000'
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB open databases critical (instance {{ $labels.instance }})
+        description: "Number of open databases exceeds 90% of node capacity\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBMangoTooManyDocsScanned
-        expr: rate(couchdb_mango_too_many_docs_scanned[5m]) > 50
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "CouchDB Mango queries scanning too many docs"
-          description: "Some Mango queries are scanning too many documents, consider adding indexes."
+    - alert: CouchdbOpenOsFilesCritical
+      expr: 'couchdb_httpd_open_os_files > 0.9 * 65535'
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB open OS files critical (instance {{ $labels.instance }})
+        description: "CouchDB is using more than 90% of allowed OS file descriptors, may fail to open new files\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBMangoInvalidIndexQueries
-        expr: rate(couchdb_mango_query_invalid_index[5m]) > 5
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "CouchDB Mango queries failed due to invalid index"
-          description: "Some Mango queries failed to execute because the index was missing or invalid."
+    - alert: Couchdb5xxErrorRatioHigh
+      expr: 'rate(couchdb_httpd_status_codes{code=~"5.."}[5m]) / rate(couchdb_httpd_requests[5m]) > 0.05'
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB 5xx error ratio high (instance {{ $labels.instance }})
+        description: "More than 5% of HTTP requests are returning 5xx errors\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBMangoDocsExaminedHigh
-        expr: rate(couchdb_mango_docs_examined[5m]) > 1000
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "CouchDB Mango docs examined high"
-          description: "High number of documents examined per Mango queries, consider indexing."
+    - alert: CouchdbTemporaryViewReadRateCritical
+      expr: 'rate(couchdb_httpd_temporary_view_reads[5m]) > 100'
+      for: 5m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB temporary view read rate critical (instance {{ $labels.instance }})
+        description: "Temporary view read rate exceeds 100 reads/sec, high risk of performance degradation\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBReplicatorManagerDeaths
-        expr: increase(couchdb_replicator_changes_manager_deaths[5m]) > 0
-        for: 1m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB Replicator manager died"
-          description: "Replication manager process has crashed."
+    - alert: CouchdbMangoQueriesScanningTooManyDocs
+      expr: 'rate(couchdb_mango_too_many_docs_scanned[5m]) > 50'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CouchDB Mango queries scanning too many docs (instance {{ $labels.instance }})
+        description: "Some Mango queries are scanning too many documents, consider adding indexes\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBReplicatorQueueDeaths
-        expr: increase(couchdb_replicator_changes_queue_deaths[5m]) > 0
-        for: 1m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB Replicator queue process died"
-          description: "Replication queue process has crashed."
+    - alert: CouchdbMangoQueriesFailedDueToInvalidIndex
+      expr: 'rate(couchdb_mango_query_invalid_index[5m]) > 5'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CouchDB Mango queries failed due to invalid index (instance {{ $labels.instance }})
+        description: "Some Mango queries failed to execute because the index was missing or invalid\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBReplicatorReaderDeaths
-        expr: increase(couchdb_replicator_changes_reader_deaths[5m]) > 0
-        for: 1m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB Replicator reader process died"
-          description: "Replication reader process has crashed."
+    - alert: CouchdbMangoDocsExaminedHigh
+      expr: 'rate(couchdb_mango_docs_examined[5m]) > 1000'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CouchDB Mango docs examined high (instance {{ $labels.instance }})
+        description: "High number of documents examined per Mango queries, consider indexing\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBReplicatorFailedStarts
-        expr: increase(couchdb_replicator_failed_starts[5m]) > 0
-        for: 1m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB Replicator failed to start"
-          description: "One or more replication tasks failed to start."
+    - alert: CouchdbReplicatorManagerDied
+      expr: 'increase(couchdb_replicator_changes_manager_deaths[5m]) > 0'
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB Replicator manager died (instance {{ $labels.instance }})
+        description: "Replication manager process has crashed\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBReplicatorClusterUnstable
-        expr: couchdb_replicator_cluster_is_stable == 0
-        for: 2m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB replication cluster unstable"
-          description: "The replication cluster is unstable, replication may be interrupted."
+    - alert: CouchdbReplicatorQueueProcessDied
+      expr: 'increase(couchdb_replicator_changes_queue_deaths[5m]) > 0'
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB Replicator queue process died (instance {{ $labels.instance }})
+        description: "Replication queue process has crashed\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBReplicatorChangesReadFailures
-        expr: increase(couchdb_replicator_changes_read_failures[5m]) > 5
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "CouchDB replication read failures"
-          description: "Replication changes feed has failed reads more than 5 times in 5 minutes."
+    - alert: CouchdbReplicatorReaderProcessDied
+      expr: 'increase(couchdb_replicator_changes_reader_deaths[5m]) > 0'
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB Replicator reader process died (instance {{ $labels.instance }})
+        description: "Replication reader process has crashed\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBOpenFDsHigh
-        expr: process_open_fds / process_max_fds > 0.85
-        for: 5m
-        labels:
-          severity: warning
-        annotations:
-          summary: "CouchDB file descriptors high"
-          description: "Process is using more than 85% of allowed file descriptors."
+    - alert: CouchdbReplicatorFailedToStart
+      expr: 'increase(couchdb_replicator_failed_starts[5m]) > 0'
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB Replicator failed to start (instance {{ $labels.instance }})
+        description: "One or more replication tasks failed to start\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBProcessRestarted
-        expr: changes(process_start_time_seconds[1h]) > 0
-        for: 1m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB process restarted"
-          description: "CouchDB process has restarted recently."
+    - alert: CouchdbReplicationClusterUnstable
+      expr: 'couchdb_replicator_cluster_is_stable == 0'
+      for: 2m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB replication cluster unstable (instance {{ $labels.instance }})
+        description: "The replication cluster is unstable, replication may be interrupted\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

-      - alert: CouchDBCriticalLogs
-        expr: increase(couchdb_server_couch_log{level=~"error|critical"}[5m]) > 0
-        for: 1m
-        labels:
-          severity: critical
-        annotations:
-          summary: "CouchDB critical log entries"
-          description: "Critical or error log entries detected in the last 5 minutes."
+    - alert: CouchdbReplicationReadFailures
+      expr: 'increase(couchdb_replicator_changes_read_failures[5m]) > 5'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CouchDB replication read failures (instance {{ $labels.instance }})
+        description: "Replication changes feed has failed reads more than 5 times in 5 minutes\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: CouchdbFileDescriptorsHigh
+      expr: 'process_open_fds / process_max_fds > 0.85'
+      for: 5m
+      labels:
+        severity: warning
+      annotations:
+        summary: CouchDB file descriptors high (instance {{ $labels.instance }})
+        description: "Process is using more than 85% of allowed file descriptors\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: CouchdbProcessRestarted
+      expr: 'changes(process_start_time_seconds[1h]) > 0'
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB process restarted (instance {{ $labels.instance }})
+        description: "CouchDB process has restarted recently\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
+
+    - alert: CouchdbCriticalLogEntries
+      expr: 'increase(couchdb_server_couch_log{level=~"error|critical"}[5m]) > 0'
+      for: 1m
+      labels:
+        severity: critical
+      annotations:
+        summary: CouchDB critical log entries (instance {{ $labels.instance }})
+        description: "Critical or error log entries detected in the last 5 minutes\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"