awesome-prometheus-alerts/dist/rules/clickhouse/embedded-exporter.yml

groups:

- name: EmbeddedExporter

  
  rules:

    # Adjust the job label to match your Prometheus configuration.
    - alert: ClickhouseNodeDown
      expr: 'up{job="clickhouse"} == 0'
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: ClickHouse node down (instance {{ $labels.instance }})
        description: "No metrics received from ClickHouse exporter for over 2 minutes.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseMemoryUsageCritical
      expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 90 and ClickHouseAsyncMetrics_CGroupMemoryTotal > 0'
      for: 5m
      labels:
        severity: critical
      annotations:
        summary: ClickHouse Memory Usage Critical (instance {{ $labels.instance }})
        description: "Memory usage is critically high, over 90%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseMemoryUsageWarning
      expr: 'ClickHouseAsyncMetrics_CGroupMemoryUsed / ClickHouseAsyncMetrics_CGroupMemoryTotal * 100 > 80 and ClickHouseAsyncMetrics_CGroupMemoryTotal > 0'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse Memory Usage Warning (instance {{ $labels.instance }})
        description: "Memory usage is over 80%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseDiskSpaceLowOnDefault
      expr: 'ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 20 and (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) > 0'
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse Disk Space Low on Default (instance {{ $labels.instance }})
        description: "Disk space on default is below 20%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseDiskSpaceCriticalOnDefault
      expr: 'ClickHouseAsyncMetrics_DiskAvailable_default / (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) * 100 < 10 and (ClickHouseAsyncMetrics_DiskAvailable_default + ClickHouseAsyncMetrics_DiskUsed_default) > 0'
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: ClickHouse Disk Space Critical on Default (instance {{ $labels.instance }})
        description: "Disk space on default disk is critically low, below 10%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseDiskSpaceLowOnBackups
      expr: 'ClickHouseAsyncMetrics_DiskAvailable_backups / (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) * 100 < 20 and (ClickHouseAsyncMetrics_DiskAvailable_backups + ClickHouseAsyncMetrics_DiskUsed_backups) > 0'
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse Disk Space Low on Backups (instance {{ $labels.instance }})
        description: "Disk space on backups is below 20%.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseReplicaErrors
      expr: 'ClickHouseErrorMetric_ALL_REPLICAS_ARE_STALE == 1 or ClickHouseErrorMetric_ALL_REPLICAS_LOST == 1'
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: ClickHouse Replica Errors (instance {{ $labels.instance }})
        description: "Critical replica errors detected, either all replicas are stale or lost.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseNoAvailableReplicas
      expr: 'ClickHouseErrorMetric_NO_AVAILABLE_REPLICA == 1'
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: ClickHouse No Available Replicas (instance {{ $labels.instance }})
        description: "No available replicas in ClickHouse.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseNoLiveReplicas
      expr: 'ClickHouseErrorMetric_TOO_FEW_LIVE_REPLICAS == 1'
      for: 0m
      labels:
        severity: critical
      annotations:
        summary: ClickHouse No Live Replicas (instance {{ $labels.instance }})
        description: "There are too few live replicas available, risking data loss and service disruption.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    # Please replace the threshold with an appropriate value
    - alert: ClickhouseHighTcpConnections
      expr: 'ClickHouseMetrics_TCPConnection > 400'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse High TCP Connections (instance {{ $labels.instance }})
        description: "High number of TCP connections, indicating heavy client or inter-cluster communication.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    # Adjust the threshold based on your cluster size and expected replication traffic.
    - alert: ClickhouseInterserverConnectionIssues
      expr: 'ClickHouseMetrics_InterserverConnection > 50'
      for: 5m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse Interserver Connection Issues (instance {{ $labels.instance }})
        description: "High number of interserver connections may indicate replication or distributed query handling issues.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseZookeeperConnectionIssues
      expr: 'ClickHouseMetrics_ZooKeeperSession != 1'
      for: 3m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse ZooKeeper Connection Issues (instance {{ $labels.instance }})
        description: "ClickHouse is experiencing issues with ZooKeeper connections, which may affect cluster state and coordination.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseAuthenticationFailures
      expr: 'increase(ClickHouseErrorMetric_AUTHENTICATION_FAILED[5m]) > 3'
      for: 0m
      labels:
        severity: info
      annotations:
        summary: ClickHouse Authentication Failures (instance {{ $labels.instance }})
        description: "Authentication failures detected, indicating potential security issues or misconfiguration.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseAccessDeniedErrors
      expr: 'increase(ClickHouseErrorMetric_RESOURCE_ACCESS_DENIED[5m]) > 3'
      for: 0m
      labels:
        severity: info
      annotations:
        summary: ClickHouse Access Denied Errors (instance {{ $labels.instance }})
        description: "Access denied errors have been logged, which could indicate permission issues or unauthorized access attempts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseRejectedInsertQueries
      expr: 'increase(ClickHouseProfileEvents_RejectedInserts[1m]) > 2'
      for: 1m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse rejected insert queries (instance {{ $labels.instance }})
        description: "INSERTs rejected due to too many active data parts. Reduce insert frequency.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseDelayedInsertQueries
      expr: 'increase(ClickHouseProfileEvents_DelayedInserts[5m]) > 10'
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse delayed insert queries (instance {{ $labels.instance }})
        description: "INSERTs delayed due to high number of active parts.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseZookeeperHardwareException
      expr: 'increase(ClickHouseProfileEvents_ZooKeeperHardwareExceptions[1m]) > 0'
      for: 1m
      labels:
        severity: critical
      annotations:
        summary: ClickHouse zookeeper hardware exception (instance {{ $labels.instance }})
        description: "Zookeeper hardware exception: network issues communicating with ZooKeeper\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    # Please replace the threshold with an appropriate value
    - alert: ClickhouseHighNetworkUsage
      expr: 'rate(ClickHouseProfileEvents_NetworkSendBytes[1m]) > 100*1024*1024 or rate(ClickHouseProfileEvents_NetworkReceiveBytes[1m]) > 100*1024*1024'
      for: 2m
      labels:
        severity: warning
      annotations:
        summary: ClickHouse high network usage (instance {{ $labels.instance }})
        description: "High network usage. ClickHouse network usage exceeds 100MB/s.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"

    - alert: ClickhouseDistributedRejectedInserts
      expr: 'increase(ClickHouseProfileEvents_DistributedRejectedInserts[5m]) > 3'
      for: 2m
      labels:
        severity: critical
      annotations:
        summary: ClickHouse distributed rejected inserts (instance {{ $labels.instance }})
        description: "INSERTs into Distributed tables rejected due to pending bytes limit.\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"