tune Targets Missing rules (#497)

* tune Targets Missing rules

* reworked query logic

* Update rules.yml

---------

Co-authored-by: Samuel Berthe <dev@samuel-berthe.fr>

_data/rules.yml

@@ -19,8 +19,11 @@ groups:
                 severity: warning
               - name: Prometheus target missing
                 description: A Prometheus target has disappeared. An exporter might be crashed.
-                query: "up == 0"
+                query: "up == 0 unless on(job) (sum by (job) (up) == 0)"
                 severity: critical
+                comments: |
+                  Only fire if at least one target in the job is still up.
+                  If all targets are down, PrometheusJobMissing or PrometheusAllTargetsMissing will fire instead.
               - name: Prometheus all targets missing
                 description: A Prometheus job does not have living target anymore.
                 query: "sum by (job) (up) == 0"

dist/rules/prometheus-self-monitoring/embedded-exporter.yml

@@ -15,7 +15,7 @@ groups:
         description: "A Prometheus job has disappeared\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}"
 
     - alert: PrometheusTargetMissing
-      expr: 'up == 0'
+      expr: 'up == 0 unless on(job) (sum by (job) (up) == 0)'
       for: 0m
       labels:
         severity: critical