awesome-prometheus-alerts/_data/rules.yml

services:
  - name: Prometheus internals
    exporters:
      - rules:
          - name: Prometheus configuration reload failure
            description: Prometheus configuration reload error
            query: "prometheus_config_last_reload_successful != 1"
            severity: warning
          - name: Prometheus AlertManager configuration reload failure
            description: AlertManager configuration reload error
            query: "alertmanager_config_last_reload_successful != 1"
            severity: warning
          - name: Prometheus not connected to alertmanager
            description: Prometheus cannot connect the alertmanager
            query: "prometheus_notifications_alertmanagers_discovered < 1"
            severity: error
          - name: Prometheus Exporter down
            description: Prometheus exporter down
            query: "up == 0"
            severity: error
          - name: Prometheus rule evaluation failures
            description: 'Prometheus encountered {{ $value }} rule evaluation failures, leading to potentially ignored alerts.'
            query: 'increase(prometheus_rule_evaluation_failures_total[3m]) > 0'
            severity: error
          - name: Prometheus template text expansion failures
            description: 'Prometheus encountered {{ $value }} template text expansion failures'
            query: 'increase(prometheus_template_text_expansion_failures_total[3m]) > 0'
            severity: error
          - name: Prometheus rule evaluation slow
            description: 'Prometheus rule evaluation took more time than the scheduled interval. I indicates a slower storage backend access or too complex query.'
            query: 'prometheus_rule_group_last_duration_seconds < prometheus_rule_group_interval_seconds'
            severity: error
          - name: Prometheus TSDB checkpoint creation failures
            description: 'Prometheus encountered {{ $value }} checkpoint creation failures'
            query: 'increase(prometheus_tsdb_checkpoint_creations_failed_total[3m]) > 0'
            severity: error
          - name: Prometheus TSDB checkpoint deletion failures
            description: 'Prometheus encountered {{ $value }} checkpoint deletion failures'
            query: 'increase(prometheus_tsdb_checkpoint_deletions_failed_total[3m]) > 0'
            severity: error
          - name: Prometheus TSDB compactions failed
            description: 'Prometheus encountered {{ $value }} TSDB compactions failures'
            query: 'increase(prometheus_tsdb_compactions_failed_total[3m]) > 0'
            severity: error
          - name: Prometheus TSDB head truncations failed
            description: 'Prometheus encountered {{ $value }} TSDB head truncation failures'
            query: 'increase(prometheus_tsdb_head_truncations_failed_total[3m]) > 0'
            severity: error
          - name: Prometheus TSDB reload failures
            description: 'Prometheus encountered {{ $value }} TSDB reload failures'
            query: 'increase(prometheus_tsdb_reloads_failures_total[3m]) > 0'
            severity: error
          - name: Prometheus TSDB WAL corruptions
            description: 'Prometheus encountered {{ $value }} TSDB WAL corruptions'
            query: 'increase(prometheus_tsdb_wal_corruptions_total[3m]) > 0'
            severity: error
          - name: Prometheus TSDB WAL truncations failed
            description: 'Prometheus encountered {{ $value }} TSDB WAL truncation failures'
            query: 'increase(prometheus_tsdb_wal_truncations_failed_total[3m]) > 0'
            severity: error

  - name: Host and hardware
    exporters:
      - name: node-exporter
        doc_url: https://github.com/prometheus/node_exporter
        rules:
          - name: Host out of memory
            description: Node memory is filling up (< 10% left)
            query: "node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10"
            severity: warning
          - name: Host unusual network throughput in
            description: Host network interfaces are probably receiving too much data (> 100 MB/s)
            query: "sum by (instance) (irate(node_network_receive_bytes_total[2m])) / 1024 / 1024 > 100"
            severity: warning
          - name: Host unusual network throughput out
            description: Host network interfaces are probably sending too much data (> 100 MB/s)
            query: "sum by (instance) (irate(node_network_transmit_bytes_total[2m])) / 1024 / 1024 > 100"
            severity: warning
          - name: Host unusual disk read rate
            description: Disk is probably reading too much data (> 50 MB/s)
            query: "sum by (instance) (irate(node_disk_read_bytes_total[2m])) / 1024 / 1024 > 50"
            severity: warning
          - name: Host unusual disk write rate
            description: Disk is probably writing too much data (> 50 MB/s)
            query: "sum by (instance) (irate(node_disk_written_bytes_total[2m])) / 1024 / 1024 > 50"
            severity: warning
          - name: Host out of disk space
            description: Disk is almost full (< 10% left)
            query: '(node_filesystem_avail_bytes{mountpoint="/rootfs"}  * 100) / node_filesystem_size_bytes{mountpoint="/rootfs"} < 10'
            severity: warning
          - name: Host disk will fill in 4 hours
            description: Disk will fill in 4 hours at current write rate
            query: 'predict_linear(node_filesystem_free_bytes{fstype!~"tmpfs"}[1h], 4 * 3600) < 0'
            severity: warning
          - name: Host out of inodes
            description: Disk is almost running out of available inodes (< 10% left)
            query: 'node_filesystem_files_free{mountpoint ="/rootfs"} / node_filesystem_files{mountpoint ="/rootfs"} * 100 < 10'
            severity: warning
          - name: Host unusual disk read latency
            description: Disk latency is growing (read operations > 100ms)
            query: "rate(node_disk_read_time_seconds_total[1m]) / rate(node_disk_reads_completed_total[1m]) > 100"
            severity: warning
          - name: Host unusual disk write latency
            description: Disk latency is growing (write operations > 100ms)
            query: "rate(node_disk_write_time_seconds_total[1m]) / rate(node_disk_writes_completed_total[1m]) > 100"
            severity: warning
          - name: Host high CPU load
            description: CPU load is > 80%
            query: '100 - (avg by(instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80'
            severity: warning
          - name: Host context switching
            description: Context switching is growing on node (> 1000 / s)
            query: "rate(node_context_switches_total[5m]) > 1000"
            severity: warning
            comments: |
              1000 context switches is an arbitrary number.
              Alert threshold depends on nature of application.
              Please read: https://github.com/samber/awesome-prometheus-alerts/issues/58              
          - name: Host swap is filling up
            description: Swap is filling up (>80%)
            query: "(1 - (node_memory_SwapFree_bytes / node_memory_SwapTotal_bytes)) * 100 > 80"
            severity: warning
          - name: Host SystemD service crashed
            description: "SystemD service crashed"
            query: 'node_systemd_unit_state{state="failed"} == 1'
            severity: warning
          - name: Host physical component too hot
            description: "Physical hardware component too hot"
            query: "node_hwmon_temp_celsius > 75"
            severity: warning
          - name: Host node overtemperature alarm
            description: "Physical node temperature alarm triggered"
            query: "node_hwmon_temp_alarm == 1"
            severity: error
          - name: Host RAID array got inactive
            description: 'RAID array {{ $labels.device }} is in degraded state due to one or more disks failures. Number of spare drives is insufficient to fix issue automatically.'
            query: 'node_md_state{state="inactive"} > 0'
            severity: error
          - name: Host RAID disk failure
            description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap'
            query: 'node_md_disks{state="fail"} > 0'
            severity: warning

  - name: Docker containers
    exporters:
      - name: cAdvisor
        doc_url: https://github.com/google/cadvisor
        rules:
          - name: Container killed
            description: A container has disappeared
            query: "time() - container_last_seen > 60"
            severity: warning
          - name: Container CPU usage
            description: Container CPU usage is above 80%
            query: "(sum(rate(container_cpu_usage_seconds_total[3m])) BY (instance, name) * 100) > 80"
            severity: warning
          - name: Container Memory usage
            description: Container Memory usage is above 80%
            query: "(sum(container_memory_usage_bytes) BY (instance, name) / sum(container_spec_memory_limit_bytes) BY (instance, name) * 100) > 80"
            severity: warning
          - name: Container Volume usage
            description: Container Volume usage is above 80%
            query: "(1 - (sum(container_fs_inodes_free) BY (instance) / sum(container_fs_inodes_total) BY (instance)) * 100) > 80"
            severity: warning
          - name: Container Volume IO usage
            description: Container Volume IO usage is above 80%
            query: "(sum(container_fs_io_current) BY (instance, name) * 100) > 80"
            severity: warning

  - name: RabbitMQ
    exporters:
      - name: kbudde/rabbitmq-exporter
        doc_url: https://github.com/kbudde/rabbitmq_exporter
        rules:
          - name: Rabbitmq down
            description: RabbitMQ node down
            query: "rabbitmq_up == 0"
            severity: error
          - name: Rabbitmq cluster down
            description: Less than 3 nodes running in RabbitMQ cluster
            query: "sum(rabbitmq_running) < 3"
            severity: error
          - name: Rabbitmq cluster partition
            description: Cluster partition
            query: "rabbitmq_partitions > 0"
            severity: error
          - name: Rabbitmq out of memory
            description: Memory available for RabbmitMQ is low (< 10%)
            query: "rabbitmq_node_mem_used / rabbitmq_node_mem_limit * 100 > 90"
            severity: warning
          - name: Rabbitmq too many connections
            description: RabbitMQ instance has too many connections (> 1000)
            query: "rabbitmq_connectionsTotal > 1000"
            severity: warning
          - name: Rabbitmq dead letter queue filling up
            description: Dead letter queue is filling up (> 10 msgs)
            query: 'rabbitmq_queue_messages{queue="my-dead-letter-queue"} > 10'
            severity: error
          - name: Rabbitmq too many messages in queue
            description: Queue is filling up (> 1000 msgs)
            query: 'rabbitmq_queue_messages_ready{queue="my-queue"} > 1000'
            severity: warning
          - name: Rabbitmq slow queue consuming
            description: Queue messages are consumed slowly (> 60s)
            query: 'time() - rabbitmq_queue_head_message_timestamp{queue="my-queue"} > 60'
            severity: warning
          - name: Rabbitmq no consumer
            description: Queue has no consumer
            query: "rabbitmq_queue_consumers == 0"
            severity: error
          - name: Rabbitmq too many consumers
            description: Queue should have only 1 consumer
            query: "rabbitmq_queue_consumers > 1"
            severity: error
          - name: Rabbitmq unactive exchange
            description: Exchange receive less than 5 msgs per second
            query: 'rate(rabbitmq_exchange_messages_published_in_total{exchange="my-exchange"}[1m]) < 5'
            severity: warning

  - name: MySQL
    exporters:
      - name: prometheus/mysqld_exporter
        doc_url: https://github.com/prometheus/mysqld_exporter
        rules:

  - name: PostgreSQL
    exporters:
      - name: wrouesnel/postgres_exporter
        doc_url: https://github.com/wrouesnel/postgres_exporter/
        rules:
          - name: Postgresql down
            description: PostgreSQL instance is down
            query: "pg_up == 0"
            severity: error
          - name: Postgresql replication lag
            description: PostgreSQL replication lag is going up (> 10s)
            query: "pg_replication_lag > 10"
            severity: warning
            comments: |
              A label excluding master nodes should be added to this query,
              in order to monitor lag on standby servers only.
              Exporter does not guarantee a NaN value for pg_replication_log on promoted master nodes.
              See https://github.com/samber/awesome-prometheus-alerts/issues/74              
          - name: Postgresql table not vaccumed
            description: Table has not been vaccum for 24 hours
            query: "time() - pg_stat_user_tables_last_autovacuum > 60 * 60 * 24"
            severity: warning
          - name: Postgresql table not analyzed
            description: Table has not been analyzed for 24 hours
            query: "time() - pg_stat_user_tables_last_autoanalyze > 60 * 60 * 24"
            severity: warning
          - name: Postgresql too many connections
            description: PostgreSQL instance has too many connections
            query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) > 100'
            severity: warning
          - name: Postgresql not enough connections
            description: PostgreSQL instance should have more connections (> 5)
            query: 'sum by (datname) (pg_stat_activity_count{datname!~"template.*|postgres"}) < 5'
            severity: warning
          - name: Postgresql dead locks
            description: PostgreSQL has dead-locks
            query: 'rate(pg_stat_database_deadlocks{datname!~"template.*|postgres"}[1m]) > 0'
            severity: warning
          - name: Postgresql slow queries
            description: PostgreSQL executes slow queries (> 1min)
            query: 'avg(rate(pg_stat_activity_max_tx_duration{datname!~"template.*"}[1m])) BY (datname) > 60'
            severity: warning
          - name: Postgresql high rollback rate
            description: Ratio of transactions being aborted compared to committed is > 2 %
            query: 'rate(pg_stat_database_xact_rollback{datname!~"template.*"}[3m]) / rate(pg_stat_database_xact_commit{datname!~"template.*"}[3m]) > 0.02'
            severity: warning

  - name: Redis
    exporters:
      - name: oliver006/redis_exporter
        doc_url: https://github.com/oliver006/redis_exporter
        rules:
          - name: Redis down
            description: Redis instance is down
            query: "redis_up == 0"
            severity: error
          - name: Redis missing backup
            description: Redis has not been backuped for 24 hours
            query: "time() - redis_rdb_last_save_timestamp_seconds > 60 * 60 * 24"
            severity: error
          - name: Redis out of memory
            description: Redis is running out of memory (> 90%)
            query: "redis_memory_used_bytes / redis_total_system_memory_bytes * 100 > 90"
            severity: warning
          - name: Redis replication broken
            description: Redis instance lost a slave
            query: "delta(redis_connected_slaves[1m]) < 0"
            severity: error
          - name: Redis too many connections
            description: Redis instance has too many connections
            query: "redis_connected_clients > 100"
            severity: warning
          - name: Redis not enough connections
            description: Redis instance should have more connections (> 5)
            query: "redis_connected_clients < 5"
            severity: warning
          - name: Redis rejected connections
            description: Some connections to Redis has been rejected
            query: "increase(redis_rejected_connections_total[1m]) > 0"
            severity: error

  - name: MongoDB
    exporters:
      - name: dcu/mongodb_exporter
        doc_url: https://github.com/percona/mongodb_exporter
        rules:
          - name: MongoDB replication lag
            description: Mongodb replication lag is more than 10s
            query: 'avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}) > 10'
            severity: error
          - name: MongoDB replication headroom
            description: MongoDB replication headroom is <= 0
            query: '(avg(mongodb_replset_oplog_tail_timestamp - mongodb_replset_oplog_head_timestamp) - (avg(mongodb_replset_member_optime_date{state="PRIMARY"}) - avg(mongodb_replset_member_optime_date{state="SECONDARY"}))) <= 0'
            severity: error
          - name: MongoDB replication Status 3
            description: MongoDB Replication set member either perform startup self-checks, or transition from completing a rollback or resync
            query: "mongodb_replset_member_state == 3"
            severity: error
          - name: MongoDB replication Status 6
            description: MongoDB Replication set member as seen from another member of the set, is not yet known
            query: "mongodb_replset_member_state == 6"
            severity: error
          - name: MongoDB replication Status 8
            description: MongoDB Replication set member as seen from another member of the set, is unreachable
            query: "mongodb_replset_member_state == 8"
            severity: error
          - name: MongoDB replication Status 9
            description: MongoDB Replication set member is actively performing a rollback. Data is not available for reads
            query: "mongodb_replset_member_state == 9"
            severity: error
          - name: MongoDB replication Status 10
            description: MongoDB Replication set member was once in a replica set but was subsequently removed
            query: "mongodb_replset_member_state == 10"
            severity: error
          - name: MongoDB number cursors open
            description: Too many cursors opened by MongoDB for clients (> 10k)
            query: 'mongodb_metrics_cursor_open{state="total_open"} > 10000'
            severity: warning
          - name: MongoDB cursors timeouts
            description: Too many cursors are timing out
            query: "increase(mongodb_metrics_cursor_timed_out_total[10m]) > 100"
            severity: warning
          - name: MongoDB too many connections
            description: Too many connections
            query: 'mongodb_connections{state="current"} > 500'
            severity: warning
          - name: MongoDB virtual memory usage
            description: High memory usage
            query: '(sum(mongodb_memory{type="virtual"}) BY (ip) / sum(mongodb_memory{type="mapped"}) BY (ip)) > 3'
            severity: warning

  - name: Elasticsearch
    exporters:
      - name: justwatchcom/elasticsearch_exporter
        doc_url: https://github.com/justwatchcom/elasticsearch_exporter
        rules:
          - name: Elasticsearch Heap Usage Too High
            description: "The heap usage is over 90% for 5m"
            query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 90'
            severity: error
          - name: Elasticsearch Heap Usage warning
            description: "The heap usage is over 80% for 5m"
            query: '(elasticsearch_jvm_memory_used_bytes{area="heap"} / elasticsearch_jvm_memory_max_bytes{area="heap"}) * 100 > 80'
            severity: warning
          - name: Elasticsearch Cluster Red
            description: Elastic Cluster Red status
            query: 'elasticsearch_cluster_health_status{color="red"} == 1'
            severity: error
          - name: Elasticsearch Cluster Yellow
            description: Elastic Cluster Yellow status
            query: 'elasticsearch_cluster_health_status{color="yellow"} == 1'
            severity: warning
          - name: Elasticsearch Healthy Nodes
            description: "Number Healthy Nodes less then number_of_nodes"
            query: "elasticsearch_cluster_health_number_of_nodes < number_of_nodes"
            severity: error
          - name: Elasticsearch Healthy Data Nodes
            description: "Number Healthy Data Nodes less then number_of_data_nodes"
            query: "elasticsearch_cluster_health_number_of_data_nodes < number_of_data_nodes"
            severity: error
          - name: Elasticsearch relocation shards
            description: "Number of relocation shards for 20 min"
            query: "elasticsearch_cluster_health_relocating_shards > 0"
            severity: error
          - name: Elasticsearch initializing shards
            description: "Number of initializing shards for 10 min"
            query: "elasticsearch_cluster_health_initializing_shards > 0"
            severity: warning
          - name: Elasticsearch unassigned shards
            description: "Number of unassigned shards for 2 min"
            query: "elasticsearch_cluster_health_unassigned_shards > 0"
            severity: error
          - name: Elasticsearch pending tasks
            description: "Number of pending tasks for 10 min. Cluster works slowly."
            query: "elasticsearch_cluster_health_number_of_pending_tasks > 0"
            severity: warning
          - name: Elasticsearch no new documents
            description: No new documents for 10 min!
            query: 'rate(elasticsearch_indices_docs{es_data_node="true"}[10m]) < 1'
            severity: warning

  - name: Cassandra
    exporters:
      - name: instaclustr/cassandra-exporter
        doc_url: https://github.com/instaclustr/cassandra-exporter
        rules:

      - name: criteo/cassandra_exporter
        doc_url: https://github.com/criteo/cassandra_exporter
        rules:
          - name: Cassandra hints count
            description: Cassandra hints count has changed on {{ $labels.instance }} some nodes may go down
            query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:totalhints:count"}[1m]) > 3'
            severity: error
          - name: Cassandra compaction task pending
            description: Many Cassandra compaction tasks are pending. You might need to increase I/O capacity by adding nodes to the cluster.
            query: 'avg_over_time(cassandra_stats{name="org:apache:cassandra:metrics:compaction:pendingtasks:value"}[30m]) > 100'
            severity: warning
          - name: Cassandra viewwrite latency
            description: High viewwrite latency on {{ $labels.instance }} cassandra node
            query: 'cassandra_stats{name="org:apache:cassandra:metrics:clientrequest:viewwrite:viewwritelatency:99thpercentile",service="cas"} > 100000'
            severity: warning
          - name: Cassandra cool hacker
            description: Increase of Cassandra authentication failures
            query: 'irate(cassandra_stats{name="org:apache:cassandra:metrics:client:authfailure:count"}[1m]) > 5'
            severity: warning
          - name: Cassandra node down
            description: Cassandra node down
            query: 'sum(cassandra_stats{name="org:apache:cassandra:net:failuredetector:downendpointcount"}) by (service,group,cluster,env) > 0'
            severity: error
          - name: Cassandra commitlog pending tasks
            description: Unexpected number of Cassandra commitlog pending tasks
            query: 'cassandra_stats{name="org:apache:cassandra:metrics:commitlog:pendingtasks:value"} > 15'
            severity: warning
          - name: Cassandra compaction executor blocked tasks
            description: Some Cassandra compaction executor tasks are blocked
            query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:compactionexecutor:currentlyblockedtasks:count"} > 0'
            severity: warning
          - name: Cassandra flush writer blocked tasks
            description: Some Cassandra flush writer tasks are blocked
            query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:memtableflushwriter:currentlyblockedtasks:count"} > 0'
            severity: warning
          - name: Cassandra repair pending tasks
            description: Some Cassandra repair tasks are pending
            query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:antientropystage:pendingtasks:value"} > 2'
            severity: warning
          - name: Cassandra repair blocked tasks
            description: Some Cassandra repair tasks are blocked
            query: 'cassandra_stats{name="org:apache:cassandra:metrics:threadpools:internal:antientropystage:currentlyblockedtasks:count"} > 0'
            severity: warning
          - name: Cassandra connection timeouts total
            description: Some connection between nodes are ending in timeout
            query: 'rate(cassandra_stats{name="org:apache:cassandra:metrics:connection:totaltimeouts:count"}[1m]) > 5'
            severity: error
          - name: Cassandra storage exceptions
            description: Something is going wrong with cassandra storage
            query: 'changes(cassandra_stats{name="org:apache:cassandra:metrics:storage:exceptions:count"}[1m]) > 1'
            severity: error

  - name: Nginx
    exporters:
      - name: nginx-lua-prometheus
        doc_url: https://github.com/knyar/nginx-lua-prometheus
        rules:
          - name: Nginx high HTTP 4xx error rate
            description: Too many HTTP requests with status 4xx (> 5%)
            query: 'sum(rate(nginx_http_requests_total{status=~"^4.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5'
            severity: error
          - name: Nginx high HTTP 5xx error rate
            description: Too many HTTP requests with status 5xx (> 5%)
            query: 'sum(rate(nginx_http_requests_total{status=~"^5.."}[1m])) / sum(rate(nginx_http_requests_total[1m])) * 100 > 5'
            severity: error

  - name: Apache
    exporters:
      - name: Lusitaniae/apache_exporter
        doc_url: https://github.com/Lusitaniae/apache_exporter
        rules:

  - name: HaProxy
    exporters:
      - name: Embedded exporter (HAProxy >= v2)
        doc_url: https://github.com/haproxy/haproxy/tree/master/contrib/prometheus-exporter
        rules:
      - name: prometheus/haproxy_exporter (HAProxy < v2)
        doc_url: https://github.com/prometheus/haproxy_exporter
        rules:
          - name: HAProxy down
            description: HAProxy down
            query: 'haproxy_up = 0'
            severity: error
          - name: HAProxy high HTTP 4xx error rate backend
            description: Too many HTTP requests with status 4xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }}
            query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5'
            severity: error
          - name: HAProxy high HTTP 4xx error rate backend
            description: Too many HTTP requests with status 5xx (> 5%) on backend {{ $labels.fqdn }}/{{ $labels.backend }}
            query: 'sum by (backend) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5'
            severity: error
          - name: HAProxy high HTTP 4xx error rate server
            description: Too many HTTP requests with status 4xx (> 5%) on server {{ $labels.server }}
            query: 'sum by (server) irate(haproxy_server_http_responses_total{code="4xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5'
            severity: error
          - name: HAProxy high HTTP 5xx error rate server
            description: Too many HTTP requests with status 5xx (> 5%) on server {{ $labels.server }}
            query: 'sum by (server) irate(haproxy_server_http_responses_total{code="5xx"}[1m]) / sum by (backend) irate(haproxy_server_http_responses_total{}[1m]) * 100 > 5'
            severity: error
          - name: HAProxy backend connection errors
            description: Too many connection errors to {{ $labels.fqdn }}/{{ $labels.backend }} backend (> 5%). Request throughput may be to high.
            query: 'sum by (backend) rate(haproxy_backend_connection_errors_total[1m]) * 100 > 5'
            severity: error
          - name: HAProxy server response errors
            description: Too many response errors to {{ $labels.server }} server (> 5%).
            query: 'sum by (server) rate(haproxy_server_response_errors_total[1m]) * 100 > 5'
            severity: error
          - name: HAProxy server connection errors
            description: Too many connection errors to {{ $labels.server }} server (> 5%). Request throughput may be to high.
            query: 'sum by (server) rate(haproxy_server_connection_errors_total[1m]) * 100 > 5'
            severity: error
          - name: HAProxy backend max active session
            description: HAproxy backend {{ $labels.fqdn }}/{{ $labels.backend }} is reaching session limit (> 80%).
            query: 'avg_over_time((sum by (backend) (haproxy_server_max_sessions) / sum by (backend) (haproxy_server_limit_sessions)) [2m]) * 100 > 80'
            severity: warning
          - name: HAProxy pending requests
            description: Some HAProxy requests are pending on {{ $labels.fqdn }}/{{ $labels.backend }} backend
            query: 'sum by (backend) haproxy_backend_current_queue > 0'
            severity: warning
          - name: HAProxy HTTP slowing down
            description: Average request time is increasing
            query: 'avg by (backend) (haproxy_backend_http_total_time_average_seconds) > 2'
            severity: warning
          - name: HAProxy retry high
            description: High rate of retry on {{ $labels.fqdn }}/{{ $labels.backend }} backend
            query: 'rate(sum by (backend) (haproxy_backend_retry_warnings_total)) > 10'
            severity: warning
          - name: HAProxy backend down
            description: HAProxy backend is down
            query: 'haproxy_backend_up = 0'
            severity: error
          - name: HAProxy server down
            description: HAProxy server is down
            query: 'haproxy_server_up = 0'
            severity: error
          - name: HAProxy frontend security blocked requests
            description: HAProxy is blocking requests for security reason
            query: 'rate(sum by (frontend) (haproxy_frontend_requests_denied_total)) > 10'
            severity: warning
          - name: HAProxy server healthcheck failure
            description: Some server healthcheck are failing on {{ $labels.server }}
            query: 'increase(haproxy_server_check_failures_total) > 0'
            severity: warning

  - name: Traefik
    exporters:
      - name: Embedded exporter
        doc_url: https://docs.traefik.io/observability/metrics/prometheus/
        rules:
          - name: Traefik backend down
            description: All Traefik backends are down
            query: "count(traefik_backend_server_up) by (backend) == 0"
            severity: error
          - name: Traefik high HTTP 4xx error rate backend
            description: Traefik backend 4xx error rate is above 5%
            query: 'sum(rate(traefik_backend_requests_total{code=~"4.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5'
            severity: error
          - name: Traefik high HTTP 5xx error rate backend
            description: Traefik backend 5xx error rate is above 5%
            query: 'sum(rate(traefik_backend_requests_total{code=~"5.*"}[3m])) by (backend) / sum(rate(traefik_backend_requests_total[3m])) by (backend) * 100 > 5'
            severity: error

  - name: PHP-FPM
    exporters:
      - name: bakins/php-fpm-exporter
        doc_url: https://github.com/bakins/php-fpm-exporter
        rules:

  - name: JVM
    exporters:
      - name: java-client
        doc_url: https://github.com/prometheus/client_java
        rules:
          - name: JVM memory filling up
            description: JVM memory is filling up (> 80%)
            query: 'jvm_memory_bytes_used / jvm_memory_bytes_max{area="heap"} > 0.8'
            severity: warning

  - name: ZFS
    exporters:
      - name: node-exporter
        doc_url: https://github.com/prometheus/node_exporter
        rules:

  - name: Kubernetes
    exporters:
      - name: kube-state-metrics
        doc_url: https://github.com/kubernetes/kube-state-metrics/tree/master/docs
        rules:
          - name: Kubernetes MemoryPressure
            description: "{{ $labels.node }} has MemoryPressure condition"
            query: 'kube_node_status_condition{condition="MemoryPressure",status="true"} == 1'
            severity: error
          - name: Kubernetes DiskPressure
            description: "{{ $labels.node }} has DiskPressure condition"
            query: 'kube_node_status_condition{condition="DiskPressure",status="true"} == 1'
            severity: error
          - name: Kubernetes OutOfDisk
            description: "{{ $labels.node }} has OutOfDisk condition"
            query: 'kube_node_status_condition{condition="OutOfDisk",status="true"} == 1'
            severity: error
          - name: Kubernetes Job failed
            description: "Job {{$labels.namespace}}/{{$labels.exported_job}} failed to complete"
            query: "kube_job_status_failed > 0"
            severity: warning
          - name: Kubernetes CronJob suspended
            description: "CronJob {{ $labels.namespace }}/{{ $labels.cronjob }} is suspended"
            query: "kube_cronjob_spec_suspend != 0"
            severity: info
          - name: Kubernetes PersistentVolumeClaim pending
            description: "PersistentVolumeClaim {{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is pending"
            query: 'kube_persistentvolumeclaim_status_phase{phase="Pending"} == 1'
            severity: warning
          - name: Kubernetes Volume out of disk space
            description: Volume is almost full (< 10% left)
            query: "kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10"
            severity: warning
          - name: Kubernetes Volume full in four days
            description: "{{ $labels.namespace }}/{{ $labels.persistentvolumeclaim }} is expected to fill up within four days. Currently {{ $value | humanize }}% is available."
            query: "100 * (kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes) < 15 and predict_linear(kubelet_volume_stats_available_bytes[6h], 4 * 24 * 3600) < 0"
            severity: error
          - name: Kubernetes StatefulSet down
            description: A StatefulSet went down
            query: "(kube_statefulset_status_replicas_ready / kube_statefulset_status_replicas_current) != 1"
            severity: error

  - name: Nomad
    exporters:
      - name: samber/prometheus-nomad-exporter
        doc_url: https://github.com/samber/prometheus-nomad-exporter
        rules:

  - name: Consul
    exporters:
      - name: prometheus/consul_exporter
        doc_url: https://github.com/prometheus/consul_exporter
        rules:
          - name: Consul service healthcheck failed
            description: "Service: `{{ $labels.service_name }}` Healthcheck: `{{ $labels.service_id }}`"
            query: "consul_catalog_service_node_healthy == 0"
            severity: error
          - name: Consul missing master node
            description: Numbers of consul raft peers should be 3, in order to preserve quorum.
            query: "consul_raft_peers < 3"
            severity: error
          - name: Consul agent unhealthy
            description: A Consul agent is down
            query: 'consul_health_node_status{status="critical"} == 1'
            severity: error

  - name: Etcd
    exporters:
      - rules:
          - name: Etcd insufficient Members
            description: Etcd cluster should have an odd number of members
            query: "count(etcd_server_id) % 2 == 0"
            severity: error
          - name: Etcd no Leader
            description: Etcd cluster have no leader
            query: "etcd_server_has_leader == 0"
            severity: error
          - name: Etcd high number of leader changes
            description: Etcd leader changed more than 3 times during last hour
            query: "increase(etcd_server_leader_changes_seen_total[1h]) > 3"
            severity: warning
          - name: Etcd high number of failed GRPC requests
            description: More than 1% GRPC request failure detected in Etcd for 5 minutes
            query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.01'
            severity: warning
          - name: Etcd high number of failed GRPC requests
            description: More than 5% GRPC request failure detected in Etcd for 5 minutes
            query: 'sum(rate(grpc_server_handled_total{grpc_code!="OK"}[5m])) BY (grpc_service, grpc_method) / sum(rate(grpc_server_handled_total[5m])) BY (grpc_service, grpc_method) > 0.05'
            severity: error
          - name: Etcd GRPC requests slow
            description: GRPC requests slowing down, 99th percentil is over 0.15s for 5 minutes
            query: 'histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15'
            severity: warning
          - name: Etcd high number of failed HTTP requests
            description: More than 1% HTTP failure detected in Etcd for 5 minutes
            query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.01"
            severity: warning
          - name: Etcd high number of failed HTTP requests
            description: More than 5% HTTP failure detected in Etcd for 5 minutes
            query: "sum(rate(etcd_http_failed_total[5m])) BY (method) / sum(rate(etcd_http_received_total[5m])) BY (method) > 0.05"
            severity: error
          - name: Etcd HTTP requests slow
            description: HTTP requests slowing down, 99th percentil is over 0.15s for 5 minutes
            query: "histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15"
            severity: warning
          - name: Etcd member communication slow
            description: Etcd member communication slowing down, 99th percentil is over 0.15s for 5 minutes
            query: "histogram_quantile(0.99, rate(etcd_network_peer_round_trip_time_seconds_bucket[5m])) > 0.15"
            severity: warning
          - name: Etcd high number of failed proposals
            description: Etcd server got more than 5 failed proposals past hour
            query: "increase(etcd_server_proposals_failed_total[1h]) > 5"
            severity: warning
          - name: Etcd high fsync durations
            description: Etcd WAL fsync duration increasing, 99th percentil is over 0.5s for 5 minutes
            query: "histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 0.5"
            severity: warning
          - name: Etcd high commit durations
            description: Etcd commit duration increasing, 99th percentil is over 0.25s for 5 minutes
            query: "histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) > 0.25"
            severity: warning

  - name: Zookeeper
    exporters:
      - name: cloudflare/kafka_zookeeper_exporter
        doc_url: https://github.com/cloudflare/kafka_zookeeper_exporter
        rules:

  - name: Kafka
    exporters:
      - name: danielqsj/kafka_exporter
        doc_url: https://github.com/danielqsj/kafka_exporter
        rules:
          - name: Kafka topics replicas
            description: Kafka topic in-sync partition
            query: "sum(kafka_topic_partition_in_sync_replica) by (topic) < 3"
            severity: error
          - name: Kafka consumers group
            description: Kafka consumers group
            query: "sum(kafka_consumergroup_lag) by (consumergroup) > 50"
            severity: error

  - name: Linkerd
    exporters:
      - rules:

  - name: Istio
    exporters:
      - rules:

  - name: Blackbox
    exporters:
      - name: prometheus/blackbox_exporter
        doc_url: https://github.com/prometheus/blackbox_exporter
        rules:
          - name: Blackbox probe failed
            description: Probe failed
            query: probe_success == 0
            severity: error
          - name: Blackbox slow probe
            description: Blackbox probe took more than 1s to complete
            query: "avg_over_time(probe_duration_seconds[1m]) > 1"
            severity: warning
          - name: Blackbox HTTP Status Code
            description: HTTP status code is not 200-399
            query: "probe_http_status_code <= 199 OR probe_http_status_code >= 400"
            severity: error
          - name: Blackbox SSL certificate will expire soon
            description: SSL certificate expires in 30 days
            query: "probe_ssl_earliest_cert_expiry - time() < 86400 * 30"
            severity: warning
          - name: Blackbox SSL certificate expired
            description: SSL certificate has expired already
            query: "probe_ssl_earliest_cert_expiry - time()  <= 0"
            severity: error
          - name: Blackbox HTTP slow requests
            description: HTTP request took more than 1s
            query: "avg_over_time(probe_http_duration_seconds[1m]) > 1"
            severity: warning
          - name: Blackbox slow ping
            description: Blackbox ping took more than 1s
            query: "avg_over_time(probe_icmp_duration_seconds[1m]) > 1"
            severity: warning

  - name: Windows Server
    exporters:
      - name: martinlindhe/wmi_exporter
        doc_url: https://github.com/martinlindhe/wmi_exporter
        rules:
          - name: Windows Server collector Error
            description: "Collector {{ $labels.collector }} was not successful"
            query: "wmi_exporter_collector_success == 0"
            severity: error
          - name: Windows Server service Status
            description: Windows Service state is not OK
            query: 'wmi_service_status{status="ok"} != 1'
            severity: error
          - name: Windows Server CPU Usage
            description: CPU Usage is more than 80%
            query: '100 - (avg by (instance) (irate(wmi_cpu_time_total{mode="idle"}[2m])) * 100) > 80'
            severity: warning
          - name: Windows Server memory Usage
            description: Memory Usage is more than 90%
            query: "100*(wmi_os_physical_memory_free_bytes) / wmi_cs_physical_memory_bytes > 90"
            severity: warning
          - name: Windows Server disk Space Usage
            description: Disk Space on Drive is used more than 80%
            query: "100.0 - 100 * ((wmi_logical_disk_free_bytes{} / 1024 / 1024 ) / (wmi_logical_disk_size_bytes{}  / 1024 / 1024)) > 80"
            severity: error

  - name: OpenEBS
    exporters:
      - rules:
          - name: OpenEBS used pool capacity
            description: 'OpenEBS Pool use more than 80% of his capacity\n  VALUE = {{ $value }}\n  LABELS: {{ $labels }}'
            query: "(openebs_used_pool_capacity_percent) > 80"
            severity: warning

  - name: Minio
    exporters:
      - rules:
          - name: Minio disk offline
            description: 'Minio disk is offline'
            query: "minio_offline_disks > 0"
            severity: error
          - name: Minio storage space exhausted
            description: 'Minio storage space is low (< 10 GB)'
            query: "minio_disk_storage_free_bytes / 1024 / 1024 / 1024 < 10"
            severity: warning

  - name: Juniper
    exporters:
      - name: czerwonk/junos_exporter
        doc_url: https://github.com/czerwonk/junos_exporter
        rules:
          - name: Juniper switch down
            description: The switch appears to be down
            query: junos_up == 0
            severity: error
          - name: Juniper high Bandwith Usage 1GiB
            description: Interface is highly saturated for at least 1 min. (> 0.90GiB/s)
            query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.90"
            severity: error
          - name: Juniper high Bandwith Usage 1GiB
            description: Interface is getting saturated for at least 1 min. (> 0.80GiB/s)
            query: "irate(junos_interface_transmit_bytes[1m]) * 8 > 1e+9 * 0.80"
            severity: warning

  - name: CoreDNS
    exporters:
      - rules:
          - name: CoreDNS Panic Count
            description: Number of CoreDNS panics encountered
            query: "increase(coredns_panic_count_total[10m]) > 0"
            severity: error