services: - name: Prometheus exporters: - rules: - name: Prometheus configuration reload description: Prometheus configuration reload error query: 'prometheus_config_last_reload_successful != 1' severity: error - name: AlertManager configuration reload description: AlertManager configuration reload error query: 'alertmanager_config_last_reload_successful != 1' severity: error - name: Exporter down description: Prometheus exporter down query: 'up == 0' severity: warning - name: Host exporters: - name: node-exporter doc_url: https://github.com/prometheus/node_exporter rules: - name: Out of memory description: Node memory is filling up (< 10% left) query: 'node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10' severity: warning - name: Unusual network throughput in description: Host network interfaces are probably receiving too much data (> 100 MB/s) query: 'sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100' severity: warning - name: Unusual network throughput out description: Host network interfaces are probably sending too much data (> 100 MB/s) query: 'sum by (instance) (irate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100' severity: warning - name: Unusual disk read rate description: Disk is probably reading too much data (> 50 MB/s) query: 'sum by (instance) (irate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50' severity: warning - name: Unusual disk write rate description: Disk is probably writing too much data (> 50 MB/s) query: 'sum by (instance) (irate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50' severity: warning - name: Out of disk space description: Disk is almost full (< 10% left) query: 'node_filesystem_free_bytes{mountpoint ="/rootfs"} / node_filesystem_size_bytes{mountpoint ="/rootfs"} * 100 < 10' severity: warning - name: Out of inodes description: Disk is almost running out of available inodes (< 10% left) query: 'node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint ="/rootfs"} * 100 < 10' severity: warning - name: Unusual disk read latency description: Disk latency is growing (read operations > 100ms) query: 'rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 100' severity: warning - name: Unusual disk write latency description: Disk latency is growing (write operations > 100ms) query: 'rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 100' severity: warning - name: High CPU load description: CPU load is > 80% query: '100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80' severity: warning - name: Context switching description: Context switching is growing on node (> 1000 / s) query: 'rate(node_context_switches_total[5m]) > 1000' severity: warning comments: | 1000 context switches is an arbitrary number. Alert threshold depends on nature of application. Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58 - name: Swap is filling up description: Swap is filling up (>80%) query: '(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80' severity: warning - name: SystemD service crashed description: 'SystemD service crashed' query: 'node_systemd_unit_state{state="failed"} == 1' severity: warning - name: Docker containers exporters: - name: cAdvisor doc_url: https://github.com/google/cadvisor rules: - name: Container killed description: A container has disappeared query: 'time() - container_last_seen > 60' severity: warning - name: Container CPU usage description: Container CPU usage is above 80% query: '(sum(rate(container_cpu_usage_seconds_total[3m])) BY (ip, name) * 100) > 80' severity: warning - name: Container Memory usage description: Container Memory usage is above 80% query: '(sum(container_memory_usage_bytes) BY (ip) / sum(container_memory_max_usage_bytes) BY (ip) * 100) > 80' severity: warning - name: Container Volume usage description: Container Volume usage is above 80% query: '(1 - (sum(container_fs_inodes_free) BY (ip) / sum(container_fs_inodes_total) BY (ip)) * 100) > 80' severity: warning - name: Container Volume IO usage description: Container Volume IO usage is above 80% query: '(sum(container_fs_io_current) BY (ip, name) * 100) > 80' severity: warning - name: Nginx exporters: - name: nginx-lua-prometheus doc_url: https://github.com/knyar/nginx-lua-prometheus rules: - name: HTTP errors 4xx description: Too many HTTP requests with status 4xx (> 5%) query: 'sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' severity: error - name: HTTP errors 5xx description: Too many HTTP requests with status 5xx (> 5%) query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5' severity: error - name: RabbitMQ exporters: - name: kbudde/rabbitmq-exporter doc_url: https://github.com/kbudde/rabbitmq_exporter rules: - name: Rabbitmq down description: RabbitMQ node down query: 'rabbitmq_up == 0' severity: error - name: Cluster down description: Less than 3 nodes running in RabbitMQ cluster query: 'rabbitmq_running < 3' severity: error - name: Cluster partition description: Cluster partition query: 'rabbitmq_partitions > 0' severity: error - name: Out of memory description: Memory available for RabbmitMQ is low (< 10%) query: 'rabbitmq_node_mem_used / rabbitmq_node_mem_limit * 100 > 90' severity: warning - name: Too many connections description: RabbitMQ instance has too many connections (> 1000) query: 'rabbitmq_connectionsTotal > 1000' severity: warning - name: Dead letter queue filling up description: Dead letter queue is filling up (> 10 msgs) query: 'rabbitmq_queue_messages{queue="my-dead-letter-queue"} > 10' severity: error - name: Too many messages in queue description: Queue is filling up (> 1000 msgs) query: 'rabbitmq_queue_messages_ready{queue="my-queue"} > 1000' severity: warning - name: Slow queue consuming description: Queue messages are consumed slowly (> 60s) query: 'time() - rabbitmq_queue_head_message_timestamp{queue="my-queue"} > 60' severity: warning - name: No consumer description: Queue has no consumer query: 'rabbitmq_queue_consumers == 0' severity: error - name: Too many consumers description: Queue should have only 1 consumer query: 'rabbitmq_queue_consumers > 1' severity: error - name: Unactive exchange description: Exchange receive less than 5 msgs per second query: 'rate(rabbitmq_exchange_messages_published_in_total{exchange="my-exchange"}[1m]) < 5' severity: warning - name: MySQL exporters: - name: prometheus/mysqld_exporter doc_url: https://github.com/prometheus/mysqld_exporter rules: - name: PostgreSQL exporters: - name: wrouesnel/postgres_exporter doc_url: https://github.com/wrouesnel/postgres_exporter/ rules: - name: PostgreSQL down description: PostgreSQL instance is down query: 'pg_up == 0' severity: error - name: Replication lag description: PostgreSQL replication lag is going up (> 10s) query: 'pg_replication_lag > 10' severity: warning - name: Table not vaccumed description: Table has not been vaccum for 24 hours query: 'time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24' severity: warning - name: Table not analyzed description: Table has not been analyzed for 24 hours query: 'time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24' severity: warning - name: Too many connections description: PostgreSQL instance has too many connections query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) > 100' severity: warning - name: Not enough connections description: PostgreSQL instance should have more connections (> 5) query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) < 5' severity: warning - name: Dead locks description: PostgreSQL has dead-locks query: 'rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0' severity: warning - name: Slow queries description: PostgreSQL executes slow queries (> 1min) query: 'avg(rate(pg_stat_activity_max_tx_duration{datname!~"template.*"}[1m])) BY (datname) > 60' severity: warning - name: High rollback rate description: Ratio of transactions being aborted compared to committed is > 2 % query: 'rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02' severity: warning - name: Redis exporters: - name: oliver006/redis_exporter doc_url: https://github.com/oliver006/redis_exporter rules: - name: Redis down description: Redis instance is down query: 'redis_up == 0' severity: error - name: Missing backup description: Redis has not been backuped for 24 hours query: 'time() - redis_rdb_last_save_timestamp_seconds > 60 * 60 * 24' severity: error - name: Out of memory description: Redis is running out of memory (> 90%) query: 'redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90' severity: warning - name: Replication broken description: Redis instance lost a slave query: 'delta(redis_connected_slaves[1m]) < 0' severity: error - name: Too many connections description: Redis instance has too many connections query: 'redis_connected_clients > 100' severity: warning - name: Not enough connections description: Redis instance should have more connections (> 5) query: 'redis_connected_clients < 5' severity: warning - name: Rejected connections description: Some connections to Redis has been rejected query: 'increase(redis_rejected_connections_total[1m]) > 0' severity: error - name: MongoDB exporters: - name: dcu/mongodb_exporter doc_url: https://github.com/percona/mongodb_exporter rules: - name: MongoDB replication lag description: Mongodb replication lag is more than 10s query: 'avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}) > 10' severity: error - name: MongoDB replication headroom description: MongoDB replication headroom is <= 0 query: '(avg(mongodb_replset_oplog_tail_timestamp - mongodb_replset_oplog_head_timestamp) - (avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}))) <= 0' severity: error - name: MongoDB replication Status 3 description: MongoDB Replication set member either perform startup self-checks, or transition from completing a rollback or resync query: 'mongodb_replset_member_state == 3' severity: error - name: MongoDB replication Status 6 description: MongoDB Replication set member as seen from another member of the set, is not yet known query: 'mongodb_replset_member_state == 6' severity: error - name: MongoDB replication Status 8 description: MongoDB Replication set member as seen from another member of the set, is unreachable query: 'mongodb_replset_member_state == 8' severity: error - name: MongoDB replication Status 9 description: MongoDB Replication set member is actively performing a rollback. Data is not available for reads query: 'mongodb_replset_member_state == 9' severity: error - name: MongoDB replication Status 10 description: MongoDB Replication set member was once in a replica set but was subsequently removed query: 'mongodb_replset_member_state == 10' severity: error - name: MongoDB number cursors open description: Too many cursors opened by MongoDB for clients (> 10k) query: 'mongodb_metrics_cursor_open{state="total_open"} > 10000' severity: warning - name: MongoDB cursors timeouts description: Too many cursors are timing out query: 'increase(mongodb_metrics_cursor_timed_out_total[10m]) > 100' severity: warning - name: MongoDB too many connections description: Too many connections query: 'mongodb_connections{state="current"} > 500' severity: warning - name: MongoDB virtual memory usage description: High memory usage query: '(sum(mongodb_memory{type="virtual"}) BY (ip) / sum(mongodb_memory{type="mapped"}) BY (ip)) > 3' severity: warning - name: Elasticsearch exporters: - name: justwatchcom/elasticsearch_exporter doc_url: https://github.com/justwatchcom/elasticsearch_exporter rules: - name: Elastic Heap Usage Too High description: 'The heap usage is over 90% for 5m' query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90' severity: error - name: Elastic Heap Usage warning description: 'The heap usage is over 80% for 5m' query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80' severity: warning - name: Elastic Cluster Red description: Elastic Cluster Red status query: 'elasticsearch_cluster_health_status{color="red"} == 1' severity: error - name: Elastic Cluster Yellow description: Elastic Cluster Yellow status query: 'elasticsearch_cluster_health_status{color="yellow"} == 1' severity: warning - name: Number of Elastic Healthy Nodes description: 'Number Healthy Nodes less then number_of_nodes' query: 'elasticsearch_cluster_health_number_of_nodes < number_of_nodes' severity: error - name: Number of Elastic Healthy Data Nodes description: 'Number Healthy Data Nodes less then number_of_data_nodes' query: 'elasticsearch_cluster_health_number_of_data_nodes < number_of_data_nodes' severity: error - name: Number of relocation shards description: 'Number of relocation shards for 20 min' query: 'elasticsearch_cluster_health_relocating_shards > 0' severity: error - name: Number of initializing shards description: 'Number of initializing shards for 10 min' query: 'elasticsearch_cluster_health_initializing_shards > 0' severity: error - name: Number of unassigned shards description: 'Number of unassigned shards for 2 min' query: 'elasticsearch_cluster_health_unassigned_shards > 0' severity: error - name: Number of pending tasks description: 'Number of pending tasks for 10 min. Cluster works slowly.' query: 'elasticsearch_cluster_health_number_of_pending_tasks > 0' severity: warning - name: Elastic no new documents description: No new documents for 10 min! query: 'rate(elasticsearch_indices_docs{es_master_node="false"}[10m]) < 1' severity: warning - name: Cassandra exporters: - name: criteo/cassandra_exporter doc_url: https://github.com/criteo/cassandra_exporter rules: - name: Apache exporters: - name: Lusitaniae/apache_exporter doc_url: https://github.com/Lusitaniae/apache_exporter rules: - name: HaProxy exporters: - name: prometheus/haproxy_exporter doc_url: https://github.com/prometheus/haproxy_exporter rules: - name: Traefik v1.* exporters: - rules: - name: Traefik backend down description: All Traefik backends are down query: 'count(traefik_backend_server_up) by (backend) == 0' severity: critical - name: Traefik backend errors description: Traefik backend error rate is above 10% query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[5m])) by (backend) / sum(rate(traefik_backend_requests_total[5m])) by (backend) > 0.1' severity: critical - name: PHP-FPM exporters: - name: bakins/php-fpm-exporter doc_url: https://github.com/bakins/php-fpm-exporter rules: - name: Java exporters: - name: java-client doc_url: https://github.com/prometheus/client_java rules: - name: JVM memory filling up description: JVM memory is filling up (> 80%) query: 'jvm_memory_bytes_used / jvm_memory_bytes_max{area="heap"} > 0.8' severity: warning - name: ZFS exporters: - name: node-exporteer doc_url: https://github.com/prometheus/node_exporter rules: - name: Kubernetes exporters: - name: kubelet rules: - name: Volume out of disk space description: Volume is almost full (< 10% left) query: 'kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10' severity: warning - name: Volume full in four days description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available." query: '100 * (kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) < 15 and predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0' severity: error - name: StatefulSet down description: A StatefulSet went down query: '(kube_statefulset_status_replicas_ready / kube_statefulset_status_replicas_current) != 1' severity: error - name: Nomad exporters: - name: samber/prometheus-nomad-exporter doc_url: https://github.com/samber/prometheus-nomad-exporter rules: - name: Consul exporters: - name: prometheus/consul_exporter doc_url: https://github.com/prometheus/consul_exporter rules: - name: Service healthcheck failed description: 'Service: `{{ $labels.service_name }}` Healthcheck: `{{ $labels.service_id }}`' query: 'consul_catalog_service_node_healthy == 0' severity: error - name: Missing Consul master node description: Numbers of consul raft peers less then expected query: 'consul_raft_peers < number_of_consul_master' severity: error - name: Etcd exporters: - rules: - name: Insufficient Members description: Etcd cluster should have an odd number of members query: 'count(etcd_server_id) > (count(etcd_server_id) / 2 - 1)' severity: error - name: No Leader description: Etcd cluster have no leader query: 'etcd_server_has_leader == 0' severity: error - name: High number of leader changes description: Etcd leader changed more than 3 times during last hour query: 'increase(etcd_server_leader_changes_seen_total[1h]) > 3' severity: warning - name: High number of failed GRPC requests description: More than 1% GRPC request failure detected in Etcd for 5 minutes query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.01' severity: warning - name: High number of failed GRPC requests description: More than 5% GRPC request failure detected in Etcd for 5 minutes query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.05' severity: error - name: GRPC requests slow description: GRPC requests slowing down, 99th percentil is over 0.15s for 5 minutes query: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15' severity: warning - name: High number of failed HTTP requests description: More than 1% HTTP failure detected in Etcd for 5 minutes query: 'sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.01' severity: warning - name: High number of failed HTTP requests description: More than 5% HTTP failure detected in Etcd for 5 minutes query: 'sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.05' severity: error - name: HTTP requests slow description: HTTP requests slowing down, 99th percentil is over 0.15s for 5 minutes query: 'histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15' severity: warning - name: Etcd member communication slow description: Etcd member communication slowing down, 99th percentil is over 0.15s for 5 minutes query: 'histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) > 0.15' severity: warning - name: High number of failed proposals description: Etcd server got more than 5 failed proposals past hour query: 'increase(etcd_server_proposals_failed_total[1h]) > 5' severity: warning - name: High fsync durations description: Etcd WAL fsync duration increasing, 99th percentil is over 0.5s for 5 minutes query: 'histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 0.5' severity: warning - name: High commit durations description: Etcd commit duration increasing, 99th percentil is over 0.25s for 5 minutes query: 'histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) > 0.25' severity: warning - name: Zookeeper exporters: - name: cloudflare/kafka_zookeeper_exporter doc_url: https://github.com/cloudflare/kafka_zookeeper_exporter rules: - name: Kafka exporters: - name: danielqsj/kafka_exporter doc_url: https://github.com/danielqsj/kafka_exporter rules: - name: Kafka Topics description: Kafka topic in-sync partition query: 'sum(kafka_topic_partition_in_sync_replica) by (topic) < 3' severity: error - name: Kafka consumers group description: Kafka consumers group query: 'sum(kafka_consumergroup_lag) by (consumergroup) > 50' severity: error - name: Linkerd exporters: - rules: - name: Istio exporters: - rules: - name: Blackbox exporters: - name: prometheus/blackbox_exporter doc_url: https://github.com/prometheus/blackbox_exporter rules: - name: Probe failed description: Probe failed query: probe_success == 0 severity: error - name: Slow probe description: Blackbox probe took more than 1s to complete query: 'avg_over_time(probe_duration_seconds[1m]) > 1' severity: warning - name: HTTP Status Code description: HTTP status code is not 200-399 query: 'probe_http_status_code <= 199 OR probe_http_status_code >= 400' severity: error - name: SSL certificate will expire soon description: SSL certificate expires in 30 days query: 'probe_ssl_earliest_cert_expiry - time() < 86400 * 30' severity: warning - name: SSL certificate has expired description: SSL certificate has expired already query: 'probe_ssl_earliest_cert_expiry - time() <= 0' severity: error - name: HTTP slow requests description: HTTP request took more than 1s query: 'avg_over_time(probe_http_duration_seconds[1m]) > 1' severity: warning - name: Slow ping description: Blackbox ping took more than 1s query: 'avg_over_time(probe_icmp_duration_seconds[1m]) > 1' severity: warning - name: Windows Server exporters: - name: martinlindhe/wmi_exporter doc_url: https://github.com/martinlindhe/wmi_exporter rules: - name: Collector Error description: 'Collector {{ $labels.collector }} was not successful' query: 'wmi_exporter_collector_success == 0' severity: error - name: Service Status description: Windows Service state is not OK query: 'wmi_service_status{status="ok"} != 1' severity: error - name: CPU Usage description: CPU Usage is more than 80% query: '100 - (avg by (instance) (irate(wmi_cpu_time_total{mode="idle"}[2m])) * 100) > 80' severity: warning - name: Memory Usage description: Memory Usage is more than 90% query: '100*(wmi_os_physical_memory_free_bytes) / wmi_cs_physical_memory_bytes > 90' severity: warning - name: Disk Space Usage description: Disk Space on Drive is used more than 80% query: '100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{} / 1024 / 1024)) > 80' severity: error - name: OpenEBS exporters: - rules: - name: Used pool capacity description: 'OpenEBS Pool use more than 80% of his capacity\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' query: '(openebs_used_pool_capacity_percent) > 80' severity: warning - name: Minio exporters: - rules: - name: Disk down description: 'Minio Disk is down\n VALUE = {{ $value }}\n LABELS: {{ $labels }}' query: 'minio_offline_disks > 0' severity: error - name: Juniper exporters: - name: czerwonk/junos_exporter doc_url: https://github.com/czerwonk/junos_exporter rules: - name: Switch is down description: The switch appears to be down query: junos_up == 0 severity: error - name: High Bandwith Usage 1GiB description: Interface is highly saturated for at least 1 min. (> 0.90GiB/s) query: 'irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.90' severity: error - name: High Bandwith Usage 1GiB description: Interface is getting saturated for at least 1 min. (> 0.80GiB/s) query: 'irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.80' severity: warning - name: CoreDNS exporters: - rules: - name: CoreDNS Panic Count description: Number of CoreDNS panics encountered query: 'increase(coredns_panic_count_total[10m]) > 0' severity: error