diff --git a/cmd/promtool/testdata/failing.yml b/cmd/promtool/testdata/failing.yml
index b524b4521..720d773ce 100644
--- a/cmd/promtool/testdata/failing.yml
+++ b/cmd/promtool/testdata/failing.yml
@@ -1,5 +1,8 @@
+rule_files:
+  - rules.yml
+
 tests:
-  # Simple failing test.
+  # Simple failing test, depends on no rules.
   - interval: 1m
     name: "Failing test"
     input_series:
@@ -18,3 +21,18 @@ tests:
         alertname: Test
         exp_alerts:
           - exp_labels: {}
+
+  # Alerts firing, but no alert expected by the test.
+  - interval: 1m
+    name: Failing alert test
+    input_series:
+      - series: 'up{job="test"}'
+        values: 0x10
+
+    alert_rule_test:
+      - eval_time: 5m
+        alertname: InstanceDown
+        exp_alerts: []
+      - eval_time: 5m
+        alertname: AlwaysFiring
+        exp_alerts: []
diff --git a/cmd/promtool/testdata/rules.yml b/cmd/promtool/testdata/rules.yml
index 7eff7f8f0..590998d88 100644
--- a/cmd/promtool/testdata/rules.yml
+++ b/cmd/promtool/testdata/rules.yml
@@ -11,6 +11,8 @@ groups:
         annotations:
           summary: "Instance {{ $labels.instance }} down"
           description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."
+      - alert: AlwaysFiring
+        expr: 1
 
   - name: rules
     rules:
diff --git a/cmd/promtool/testdata/unittest.yml b/cmd/promtool/testdata/unittest.yml
index f0f333b59..d6221048d 100644
--- a/cmd/promtool/testdata/unittest.yml
+++ b/cmd/promtool/testdata/unittest.yml
@@ -83,10 +83,17 @@ tests:
       - expr: count(ALERTS) by (alertname, alertstate)
         eval_time: 4m
         exp_samples:
+        - labels: '{alertname="AlwaysFiring",alertstate="firing"}'
+          value: 1
         - labels: '{alertname="InstanceDown",alertstate="pending"}'
           value: 1
 
     alert_rule_test:
+      - eval_time: 1d
+        alertname: AlwaysFiring
+        exp_alerts:
+          - {}
+
       - eval_time: 1d
         alertname: InstanceDown
         exp_alerts:
@@ -98,6 +105,15 @@ tests:
               summary: "Instance localhost:9090 down"
               description: "localhost:9090 of job prometheus has been down for more than 5 minutes."
 
+      - eval_time: 0
+        alertname: AlwaysFiring
+        exp_alerts:
+          - {}
+
+      - eval_time: 0
+        alertname: InstanceDown
+        exp_alerts: []
+
   # Tests for interval vs evaluation_interval.
   - interval: 1s
     input_series: