From 486025036040c4a8bd5d87e82843021528343917 Mon Sep 17 00:00:00 2001 From: Matthias Crauwels Date: Mon, 30 Mar 2020 11:24:58 +0200 Subject: [PATCH 1/2] added some extra MySQL checks --- _data/rules.yml | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 8ad6eb5..6de52eb 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -312,11 +312,27 @@ groups: rules: - name: MySQL down description: MySQL instance is down on {{ $labels.instance }} - query: 'mysql_up == 0' + query: 'avg by (instance) (mysql_up) == 0' severity: error - name: MySQL too many connections description: 'More than 80% of MySQL connections are in use on {{ $labels.instance }}' - query: '(mysql_global_status_threads_connected / mysql_global_variables_max_connections) * 100 > 80' + query: 'avg by(instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by(instance) (mysql_global_variables_max_connections) * 100 > 80' + severity: warning + - name: MySQL high threads running + description: 'More than 60% of MySQL connections are in running state on {{ $labels.instance }}' + query: 'avg by(instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by(instance) (mysql_global_variables_max_connections) * 100 > 60' + severity: warning + - name: MySQL Slave IO thread not running on {{ $instance.instance }} + description: 'MySQL Slave IO thread not running on {{ $labels.instance }}' + query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and avg by (instance) (mysql_slave_status_slave_io_running) == 0' + severity: error + - name: MySQL Slave SQL thread not running on {{ $instance.instance }} + description: 'MySQL Slave SQL thread not running on {{ $labels.instance }}' + query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and avg by (instance) (mysql_slave_status_slave_sql_running) == 0' + severity: error + - name: MySQL Slave replication lag {{ $instance.instance }} + description: 'MysqL replication lag on {{ $labels.instance }}' + query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and (avg by (instance) (max_over_time(mysql_slave_status_seconds_behind_master[1m])) - avg by (instance) (mysql_slave_status_sql_delay)) > 300' severity: warning - name: MySQL slow queries description: MySQL server is having some slow queries. From 79b5ad3b5d3a0921b614905a37c2aae775e9ba93 Mon Sep 17 00:00:00 2001 From: Matthias Crauwels Date: Tue, 31 Mar 2020 11:42:05 +0200 Subject: [PATCH 2/2] removed avg grouping where possible --- _data/rules.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/_data/rules.yml b/_data/rules.yml index 6de52eb..396b8e7 100644 --- a/_data/rules.yml +++ b/_data/rules.yml @@ -312,27 +312,27 @@ groups: rules: - name: MySQL down description: MySQL instance is down on {{ $labels.instance }} - query: 'avg by (instance) (mysql_up) == 0' + query: 'mysql_up == 0' severity: error - name: MySQL too many connections description: 'More than 80% of MySQL connections are in use on {{ $labels.instance }}' - query: 'avg by(instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by(instance) (mysql_global_variables_max_connections) * 100 > 80' + query: 'avg by (instance) (max_over_time(mysql_global_status_threads_connected[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 80' severity: warning - name: MySQL high threads running description: 'More than 60% of MySQL connections are in running state on {{ $labels.instance }}' - query: 'avg by(instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by(instance) (mysql_global_variables_max_connections) * 100 > 60' + query: 'avg by (instance) (max_over_time(mysql_global_status_threads_running[5m])) / avg by (instance) (mysql_global_variables_max_connections) * 100 > 60' severity: warning - name: MySQL Slave IO thread not running on {{ $instance.instance }} description: 'MySQL Slave IO thread not running on {{ $labels.instance }}' - query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and avg by (instance) (mysql_slave_status_slave_io_running) == 0' + query: 'mysql_slave_status_master_server_id > 0 and mysql_slave_status_slave_io_running == 0' severity: error - name: MySQL Slave SQL thread not running on {{ $instance.instance }} description: 'MySQL Slave SQL thread not running on {{ $labels.instance }}' - query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and avg by (instance) (mysql_slave_status_slave_sql_running) == 0' + query: 'mysql_slave_status_master_server_id > 0 and mysql_slave_status_slave_sql_running == 0' severity: error - name: MySQL Slave replication lag {{ $instance.instance }} description: 'MysqL replication lag on {{ $labels.instance }}' - query: 'avg by(instance) (mysql_slave_status_master_server_id) > 0 and (avg by (instance) (max_over_time(mysql_slave_status_seconds_behind_master[1m])) - avg by (instance) (mysql_slave_status_sql_delay)) > 300' + query: 'mysql_slave_status_master_server_id > 0 and (mysql_slave_status_seconds_behind_master - mysql_slave_status_sql_delay) > 300' severity: warning - name: MySQL slow queries description: MySQL server is having some slow queries.