promtool: Add more negative alert tests

Signed-off-by: David Leadbeater <dgl@dgl.cx>

cmd/promtool/testdata/failing.yml

@@ -1,5 +1,8 @@
+rule_files:
+  - rules.yml
+
 tests:
-  # Simple failing test.
+  # Simple failing test, depends on no rules.
   - interval: 1m
     name: "Failing test"
     input_series:
@@ -18,3 +21,18 @@ tests:
         alertname: Test
         exp_alerts:
           - exp_labels: {}
+
+  # Alerts firing, but no alert expected by the test.
+  - interval: 1m
+    name: Failing alert test
+    input_series:
+      - series: 'up{job="test"}'
+        values: 0x10
+
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: InstanceDown
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: AlwaysFiring
+        exp_alerts: []

cmd/promtool/testdata/rules.yml

@@ -11,6 +11,8 @@ groups:
         annotations:
           summary: "Instance {{ $labels.instance }} down"
           description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+      - alert: AlwaysFiring
+        expr: 1
 
   - name: rules
     rules:

cmd/promtool/testdata/unittest.yml

@@ -83,10 +83,17 @@ tests:
       - expr: count(ALERTS) by (alertname, alertstate)
         eval_time: 4m
         exp_samples:
+        - labels: '{alertname="AlwaysFiring",alertstate="firing"}'
+          value: 1
         - labels: '{alertname="InstanceDown",alertstate="pending"}'
           value: 1
 
     alert_rule_test:
+      - eval_time: 1d
+        alertname: AlwaysFiring
+        exp_alerts:
+          - {}
+
       - eval_time: 1d
         alertname: InstanceDown
         exp_alerts:
@@ -98,6 +105,15 @@ tests:
               summary: "Instance localhost:9090 down"
               description: "localhost:9090 of job prometheus has been down for more than 5 minutes."
 
+      - eval_time: 0
+        alertname: AlwaysFiring
+        exp_alerts:
+          - {}
+
+      - eval_time: 0
+        alertname: InstanceDown
+        exp_alerts: []
+
   # Tests for interval vs evaluation_interval.
   - interval: 1s
     input_series: