diff --git a/dist/rules/clickhouse/embedded-exporter.yml b/dist/rules/clickhouse/embedded-exporter.yml index 139597f..a4cd2a7 100644 --- a/dist/rules/clickhouse/embedded-exporter.yml +++ b/dist/rules/clickhouse/embedded-exporter.yml @@ -1,2 +1,185 @@ +groups: +- name: EmbeddedExporter + rules: + + - alert: ClickhouseNodeDown + expr: 'up{job="clickhouse"} == 0' + for: 2m + labels: + severity: critical + annotations: + summary: ClickHouse node down (instance {{ $labels.instance }}) + description: "No metrics received from ClickHouse exporter for over 2 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseMemoryUsageCritical + expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90' + for: 5m + labels: + severity: critical + annotations: + summary: ClickHouse Memory Usage Critical (instance {{ $labels.instance }}) + description: "Memory usage is critically high, over 90%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseMemoryUsageWarning + expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80' + for: 5m + labels: + severity: warning + annotations: + summary: ClickHouse Memory Usage Warning (instance {{ $labels.instance }}) + description: "Memory usage is over 80%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseDiskSpaceLowOnDefault + expr: 'ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20' + for: 2m + labels: + severity: warning + annotations: + summary: ClickHouse Disk Space Low on Default (instance {{ $labels.instance }}) + description: "Disk space on default is below 20%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseDiskSpaceCriticalOnDefault + expr: 'ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10' + for: 2m + labels: + severity: critical + annotations: + summary: ClickHouse Disk Space Critical on Default (instance {{ $labels.instance }}) + description: "Disk space on default disk is critically low, below 10%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseDiskSpaceLowOnBackups + expr: 'ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20' + for: 2m + labels: + severity: warning + annotations: + summary: ClickHouse Disk Space Low on Backups (instance {{ $labels.instance }}) + description: "Disk space on backups is below 20%.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseReplicaErrors + expr: 'ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1' + for: 0m + labels: + severity: critical + annotations: + summary: ClickHouse Replica Errors (instance {{ $labels.instance }}) + description: "Critical replica errors detected, either all replicas are stale or lost.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseNoAvailableReplicas + expr: 'ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1' + for: 0m + labels: + severity: critical + annotations: + summary: ClickHouse No Available Replicas (instance {{ $labels.instance }}) + description: "No available replicas in ClickHouse.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseNoLiveReplicas + expr: 'ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1' + for: 0m + labels: + severity: critical + annotations: + summary: ClickHouse No Live Replicas (instance {{ $labels.instance }}) + description: "There are too few live replicas available, risking data loss and service disruption.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseHighNetworkTraffic + expr: 'ClickHouseMetrics_NetworkSend > 250 or ClickHouseMetrics_NetworkReceive > 250' + for: 5m + labels: + severity: warning + annotations: + summary: ClickHouse High Network Traffic (instance {{ $labels.instance }}) + description: "Network traffic is unusually high, may affect cluster performance.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseHighTcpConnections + expr: 'ClickHouseMetrics_TCPConnection > 400' + for: 5m + labels: + severity: warning + annotations: + summary: ClickHouse High TCP Connections (instance {{ $labels.instance }}) + description: "High number of TCP connections, indicating heavy client or inter-cluster communication.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseInterserverConnectionIssues + expr: 'increase(ClickHouseMetrics_InterserverConnection[5m]) > 0' + for: 1m + labels: + severity: warning + annotations: + summary: ClickHouse Interserver Connection Issues (instance {{ $labels.instance }}) + description: "An increase in interserver connections may indicate replication or distributed query handling issues.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseZookeeperConnectionIssues + expr: 'avg(ClickHouseMetrics_ZooKeeperSession) != 1' + for: 3m + labels: + severity: warning + annotations: + summary: ClickHouse ZooKeeper Connection Issues (instance {{ $labels.instance }}) + description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseAuthenticationFailures + expr: 'increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 0' + for: 0m + labels: + severity: info + annotations: + summary: ClickHouse Authentication Failures (instance {{ $labels.instance }}) + description: "Authentication failures detected, indicating potential security issues or misconfiguration.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseAccessDeniedErrors + expr: 'increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 0' + for: 0m + labels: + severity: info + annotations: + summary: ClickHouse Access Denied Errors (instance {{ $labels.instance }}) + description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseRejectedInsertQueries + expr: 'increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 0' + for: 1m + labels: + severity: warning + annotations: + summary: ClickHouse rejected insert queries (instance {{ $labels.instance }}) + description: "INSERTs rejected due to too many active data parts. Reduce insert frequency.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseDelayedInsertQueries + expr: 'increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 0' + for: 2m + labels: + severity: warning + annotations: + summary: ClickHouse delayed insert queries (instance {{ $labels.instance }}) + description: "INSERTs delayed due to high number of active parts.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseZookeeperHardwareException + expr: 'increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0' + for: 1m + labels: + severity: critical + annotations: + summary: ClickHouse zookeeper hardware exception (instance {{ $labels.instance }}) + description: "Zookeeper hardware exception: network issues communicating with ZooKeeper\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseHighNetworkUsage + expr: 'rate(ClickHouseProfileEvents_NetworkSendBytes[1m]) > 100*1024*1024 or rate(ClickHouseProfileEvents_NetworkReceiveBytes[1m]) > 100*1024*1024' + for: 2m + labels: + severity: warning + annotations: + summary: ClickHouse high network usage (instance {{ $labels.instance }}) + description: "High network usage. ClickHouse network usage exceeds 100MB/s.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: ClickhouseDistributedRejectedInserts + expr: 'increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 0' + for: 2m + labels: + severity: critical + annotations: + summary: ClickHouse distributed rejected inserts (instance {{ $labels.instance }}) + description: "INSERTs into Distributed tables rejected due to pending bytes limit.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" diff --git a/dist/rules/couchdb/gesellix-couchdb-prometheus-exporter.yml b/dist/rules/couchdb/gesellix-couchdb-prometheus-exporter.yml index 2f0473a..b689d0c 100644 --- a/dist/rules/couchdb/gesellix-couchdb-prometheus-exporter.yml +++ b/dist/rules/couchdb/gesellix-couchdb-prometheus-exporter.yml @@ -1,164 +1,167 @@ groups: - - name: couchdb-alerts - rules: - - alert: CouchDBNodeDown - expr: expr: couchdb_httpd_node_up == 0 or couchdb_httpd_up == 0 - for: 2m - labels: - severity: critical - annotations: - summary: "CouchDB node down" - description: "CouchDB node is not responding (node_up metric is 0) for more than 2 minutes." - - alert: CouchDBAtomMemoryHigh - expr: couchdb_erlang_memory_atom_used > 0.9 * couchdb_erlang_memory_atom - for: 5m - labels: - severity: critical - annotations: - summary: "CouchDB atom memory usage critical" - description: "Atom memory usage is above 90% of limit." +- name: GesellixCouchdbPrometheusExporter - - alert: CouchDBOpenDatabasesCritical - expr: couchdb_httpd_open_databases > 0.9 * 1000 - for: 5m - labels: - severity: critical - annotations: - summary: "CouchDB open databases critical" - description: "Number of open databases exceeds 90% of node capacity." + rules: - - alert: CouchDBOpenOSFilesCritical - expr: couchdb_httpd_open_os_files > 0.9 * 65535 - for: 5m - labels: - severity: critical - annotations: - summary: "CouchDB open OS files critical" - description: "CouchDB is using more than 90% of allowed OS file descriptors, may fail to open new files." + - alert: CouchdbNodeDown + expr: 'couchdb_httpd_node_up == 0 or couchdb_httpd_up == 0' + for: 2m + labels: + severity: critical + annotations: + summary: CouchDB node down (instance {{ $labels.instance }}) + description: "CouchDB node is not responding (node_up metric is 0) for more than 2 minutes\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDB5xxErrorsHigh - expr: rate(couchdb_httpd_status_codes{code=~"5.."}[5m]) / rate(couchdb_httpd_requests[5m]) > 0.05 - for: 5m - labels: - severity: critical - annotations: - summary: "CouchDB 5xx error ratio high" - description: "More than 5% of HTTP requests are returning 5xx errors." + - alert: CouchdbAtomMemoryUsageCritical + expr: 'couchdb_erlang_memory_atom_used > 0.9 * couchdb_erlang_memory_atom' + for: 5m + labels: + severity: critical + annotations: + summary: CouchDB atom memory usage critical (instance {{ $labels.instance }}) + description: "Atom memory usage is above 90% of limit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBTemporaryViewReadsCritical - expr: rate(couchdb_httpd_temporary_view_reads[5m]) > 100 - for: 5m - labels: - severity: critical - annotations: - summary: "CouchDB temporary view read rate critical" - description: "Temporary view read rate exceeds 100 reads/sec, high risk of performance degradation." + - alert: CouchdbOpenDatabasesCritical + expr: 'couchdb_httpd_open_databases > 0.9 * 1000' + for: 5m + labels: + severity: critical + annotations: + summary: CouchDB open databases critical (instance {{ $labels.instance }}) + description: "Number of open databases exceeds 90% of node capacity\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBMangoTooManyDocsScanned - expr: rate(couchdb_mango_too_many_docs_scanned[5m]) > 50 - for: 5m - labels: - severity: warning - annotations: - summary: "CouchDB Mango queries scanning too many docs" - description: "Some Mango queries are scanning too many documents, consider adding indexes." + - alert: CouchdbOpenOsFilesCritical + expr: 'couchdb_httpd_open_os_files > 0.9 * 65535' + for: 5m + labels: + severity: critical + annotations: + summary: CouchDB open OS files critical (instance {{ $labels.instance }}) + description: "CouchDB is using more than 90% of allowed OS file descriptors, may fail to open new files\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBMangoInvalidIndexQueries - expr: rate(couchdb_mango_query_invalid_index[5m]) > 5 - for: 5m - labels: - severity: warning - annotations: - summary: "CouchDB Mango queries failed due to invalid index" - description: "Some Mango queries failed to execute because the index was missing or invalid." + - alert: Couchdb5xxErrorRatioHigh + expr: 'rate(couchdb_httpd_status_codes{code=~"5.."}[5m]) / rate(couchdb_httpd_requests[5m]) > 0.05' + for: 5m + labels: + severity: critical + annotations: + summary: CouchDB 5xx error ratio high (instance {{ $labels.instance }}) + description: "More than 5% of HTTP requests are returning 5xx errors\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBMangoDocsExaminedHigh - expr: rate(couchdb_mango_docs_examined[5m]) > 1000 - for: 5m - labels: - severity: warning - annotations: - summary: "CouchDB Mango docs examined high" - description: "High number of documents examined per Mango queries, consider indexing." + - alert: CouchdbTemporaryViewReadRateCritical + expr: 'rate(couchdb_httpd_temporary_view_reads[5m]) > 100' + for: 5m + labels: + severity: critical + annotations: + summary: CouchDB temporary view read rate critical (instance {{ $labels.instance }}) + description: "Temporary view read rate exceeds 100 reads/sec, high risk of performance degradation\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBReplicatorManagerDeaths - expr: increase(couchdb_replicator_changes_manager_deaths[5m]) > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "CouchDB Replicator manager died" - description: "Replication manager process has crashed." + - alert: CouchdbMangoQueriesScanningTooManyDocs + expr: 'rate(couchdb_mango_too_many_docs_scanned[5m]) > 50' + for: 5m + labels: + severity: warning + annotations: + summary: CouchDB Mango queries scanning too many docs (instance {{ $labels.instance }}) + description: "Some Mango queries are scanning too many documents, consider adding indexes\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBReplicatorQueueDeaths - expr: increase(couchdb_replicator_changes_queue_deaths[5m]) > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "CouchDB Replicator queue process died" - description: "Replication queue process has crashed." + - alert: CouchdbMangoQueriesFailedDueToInvalidIndex + expr: 'rate(couchdb_mango_query_invalid_index[5m]) > 5' + for: 5m + labels: + severity: warning + annotations: + summary: CouchDB Mango queries failed due to invalid index (instance {{ $labels.instance }}) + description: "Some Mango queries failed to execute because the index was missing or invalid\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBReplicatorReaderDeaths - expr: increase(couchdb_replicator_changes_reader_deaths[5m]) > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "CouchDB Replicator reader process died" - description: "Replication reader process has crashed." + - alert: CouchdbMangoDocsExaminedHigh + expr: 'rate(couchdb_mango_docs_examined[5m]) > 1000' + for: 5m + labels: + severity: warning + annotations: + summary: CouchDB Mango docs examined high (instance {{ $labels.instance }}) + description: "High number of documents examined per Mango queries, consider indexing\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBReplicatorFailedStarts - expr: increase(couchdb_replicator_failed_starts[5m]) > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "CouchDB Replicator failed to start" - description: "One or more replication tasks failed to start." + - alert: CouchdbReplicatorManagerDied + expr: 'increase(couchdb_replicator_changes_manager_deaths[5m]) > 0' + for: 1m + labels: + severity: critical + annotations: + summary: CouchDB Replicator manager died (instance {{ $labels.instance }}) + description: "Replication manager process has crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBReplicatorClusterUnstable - expr: couchdb_replicator_cluster_is_stable == 0 - for: 2m - labels: - severity: critical - annotations: - summary: "CouchDB replication cluster unstable" - description: "The replication cluster is unstable, replication may be interrupted." + - alert: CouchdbReplicatorQueueProcessDied + expr: 'increase(couchdb_replicator_changes_queue_deaths[5m]) > 0' + for: 1m + labels: + severity: critical + annotations: + summary: CouchDB Replicator queue process died (instance {{ $labels.instance }}) + description: "Replication queue process has crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBReplicatorChangesReadFailures - expr: increase(couchdb_replicator_changes_read_failures[5m]) > 5 - for: 5m - labels: - severity: warning - annotations: - summary: "CouchDB replication read failures" - description: "Replication changes feed has failed reads more than 5 times in 5 minutes." + - alert: CouchdbReplicatorReaderProcessDied + expr: 'increase(couchdb_replicator_changes_reader_deaths[5m]) > 0' + for: 1m + labels: + severity: critical + annotations: + summary: CouchDB Replicator reader process died (instance {{ $labels.instance }}) + description: "Replication reader process has crashed\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBOpenFDsHigh - expr: process_open_fds / process_max_fds > 0.85 - for: 5m - labels: - severity: warning - annotations: - summary: "CouchDB file descriptors high" - description: "Process is using more than 85% of allowed file descriptors." + - alert: CouchdbReplicatorFailedToStart + expr: 'increase(couchdb_replicator_failed_starts[5m]) > 0' + for: 1m + labels: + severity: critical + annotations: + summary: CouchDB Replicator failed to start (instance {{ $labels.instance }}) + description: "One or more replication tasks failed to start\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBProcessRestarted - expr: changes(process_start_time_seconds[1h]) > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "CouchDB process restarted" - description: "CouchDB process has restarted recently." + - alert: CouchdbReplicationClusterUnstable + expr: 'couchdb_replicator_cluster_is_stable == 0' + for: 2m + labels: + severity: critical + annotations: + summary: CouchDB replication cluster unstable (instance {{ $labels.instance }}) + description: "The replication cluster is unstable, replication may be interrupted\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" - - alert: CouchDBCriticalLogs - expr: increase(couchdb_server_couch_log{level=~"error|critical"}[5m]) > 0 - for: 1m - labels: - severity: critical - annotations: - summary: "CouchDB critical log entries" - description: "Critical or error log entries detected in the last 5 minutes." + - alert: CouchdbReplicationReadFailures + expr: 'increase(couchdb_replicator_changes_read_failures[5m]) > 5' + for: 5m + labels: + severity: warning + annotations: + summary: CouchDB replication read failures (instance {{ $labels.instance }}) + description: "Replication changes feed has failed reads more than 5 times in 5 minutes\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: CouchdbFileDescriptorsHigh + expr: 'process_open_fds / process_max_fds > 0.85' + for: 5m + labels: + severity: warning + annotations: + summary: CouchDB file descriptors high (instance {{ $labels.instance }}) + description: "Process is using more than 85% of allowed file descriptors\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: CouchdbProcessRestarted + expr: 'changes(process_start_time_seconds[1h]) > 0' + for: 1m + labels: + severity: critical + annotations: + summary: CouchDB process restarted (instance {{ $labels.instance }}) + description: "CouchDB process has restarted recently\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + + - alert: CouchdbCriticalLogEntries + expr: 'increase(couchdb_server_couch_log{level=~"error|critical"}[5m]) > 0' + for: 1m + labels: + severity: critical + annotations: + summary: CouchDB critical log entries (instance {{ $labels.instance }}) + description: "Critical or error log entries detected in the last 5 minutes\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"