Merge pull request #124 from matthiasr/end-to-end-test

Add an end to end test.

.travis.yml

@@ -11,3 +11,5 @@ install:
 
 script:
   - go test -v ./...
+  - go build
+  - ./end-to-end-test.sh -v

collector/bonding_linux.go

@@ -25,10 +25,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const (
-	sysfsNet = "/sys/class/net"
-)
-
 type bondingCollector struct {
 	slaves, active *prometheus.GaugeVec
 }
@@ -62,7 +58,7 @@ func NewBondingCollector() (Collector, error) {
 
 // Update reads and exposes bonding states, implements Collector interface. Caution: This works only on linux.
 func (c *bondingCollector) Update(ch chan<- prometheus.Metric) (err error) {
-	bondingStats, err := readBondingStats(sysfsNet)
+	bondingStats, err := readBondingStats(sysFilePath("class/net"))
 	if err != nil {
 		return err
 	}

collector/bonding_linux_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestBonding(t *testing.T) {
-	bondingStats, err := readBondingStats("fixtures/bonding")
+	bondingStats, err := readBondingStats("fixtures/sys/class/net")
 	if err != nil {
 		t.Fatal(err)
 	}

collector/diskstats_linux_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func TestDiskStats(t *testing.T) {
-	file, err := os.Open("fixtures/diskstats")
+	file, err := os.Open("fixtures/proc/diskstats")
 	if err != nil {
 		t.Fatal(err)
 	}

collector/filefd_linux_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func TestFileFDStats(t *testing.T) {
-	file, err := os.Open("fixtures/file-nr")
+	file, err := os.Open("fixtures/proc/sys/fs/file-nr")
 	if err != nil {
 		t.Fatal(err)
 	}

collector/fixtures/megacli

@@ -1,3 +1,3 @@
 #!/usr/bin/env bash
 
-cat fixtures/megacli_disks.txt
+cat "$(dirname "$0")/megacli_disks.txt"

collector/fixtures/proc/loadavg

@@ -0,0 +1 @@
+0.21 0.37 0.39 1/719 19737

collector/fixtures/proc/mounts

@@ -0,0 +1,30 @@
+rootfs / rootfs rw 0 0
+sysfs /sys sysfs rw,nosuid,nodev,noexec,relatime 0 0
+proc /proc proc rw,nosuid,nodev,noexec,relatime 0 0
+udev /dev devtmpfs rw,relatime,size=10240k,nr_inodes=1008585,mode=755 0 0
+devpts /dev/pts devpts rw,nosuid,noexec,relatime,gid=5,mode=620,ptmxmode=000 0 0
+tmpfs /run tmpfs rw,nosuid,relatime,size=1617716k,mode=755 0 0
+/dev/dm-2 / ext4 rw,relatime,errors=remount-ro,data=ordered 0 0
+securityfs /sys/kernel/security securityfs rw,nosuid,nodev,noexec,relatime 0 0
+tmpfs /dev/shm tmpfs rw,nosuid,nodev 0 0
+tmpfs /run/lock tmpfs rw,nosuid,nodev,noexec,relatime,size=5120k 0 0
+tmpfs /sys/fs/cgroup tmpfs ro,nosuid,nodev,noexec,mode=755 0 0
+cgroup /sys/fs/cgroup/systemd cgroup rw,nosuid,nodev,noexec,relatime,xattr,release_agent=/lib/systemd/systemd-cgroups-agent,name=systemd 0 0
+pstore /sys/fs/pstore pstore rw,nosuid,nodev,noexec,relatime 0 0
+cgroup /sys/fs/cgroup/cpuset cgroup rw,nosuid,nodev,noexec,relatime,cpuset 0 0
+cgroup /sys/fs/cgroup/cpu,cpuacct cgroup rw,nosuid,nodev,noexec,relatime,cpu,cpuacct 0 0
+cgroup /sys/fs/cgroup/devices cgroup rw,nosuid,nodev,noexec,relatime,devices 0 0
+cgroup /sys/fs/cgroup/freezer cgroup rw,nosuid,nodev,noexec,relatime,freezer 0 0
+cgroup /sys/fs/cgroup/net_cls,net_prio cgroup rw,nosuid,nodev,noexec,relatime,net_cls,net_prio 0 0
+cgroup /sys/fs/cgroup/blkio cgroup rw,nosuid,nodev,noexec,relatime,blkio 0 0
+cgroup /sys/fs/cgroup/perf_event cgroup rw,nosuid,nodev,noexec,relatime,perf_event 0 0
+systemd-1 /proc/sys/fs/binfmt_misc autofs rw,relatime,fd=22,pgrp=1,timeout=300,minproto=5,maxproto=5,direct 0 0
+mqueue /dev/mqueue mqueue rw,relatime 0 0
+debugfs /sys/kernel/debug debugfs rw,relatime 0 0
+hugetlbfs /dev/hugepages hugetlbfs rw,relatime 0 0
+fusectl /sys/fs/fuse/connections fusectl rw,relatime 0 0
+/dev/sda3 /boot ext2 rw,relatime 0 0
+rpc_pipefs /run/rpc_pipefs rpc_pipefs rw,relatime 0 0
+binfmt_misc /proc/sys/fs/binfmt_misc binfmt_misc rw,relatime 0 0
+tmpfs /run/user/1000 tmpfs rw,nosuid,nodev,relatime,size=808860k,mode=700,uid=1000,gid=1000 0 0
+gvfsd-fuse /run/user/1000/gvfs fuse.gvfsd-fuse rw,nosuid,nodev,relatime,user_id=1000,group_id=1000 0 0

collector/fixtures/proc/net/snmp

@@ -0,0 +1,12 @@
+Ip: Forwarding DefaultTTL InReceives InHdrErrors InAddrErrors ForwDatagrams InUnknownProtos InDiscards InDelivers OutRequests OutDiscards OutNoRoutes ReasmTimeout ReasmReqds ReasmOKs ReasmFails FragOKs FragFails FragCreates
+Ip: 1 64 57740232 0 25 397750 0 0 57340175 55365537 0 54 0 0 0 0 0 0 0
+Icmp: InMsgs InErrors InCsumErrors InDestUnreachs InTimeExcds InParmProbs InSrcQuenchs InRedirects InEchos InEchoReps InTimestamps InTimestampReps InAddrMasks InAddrMaskReps OutMsgs OutErrors OutDestUnreachs OutTimeExcds OutParmProbs OutSrcQuenchs OutRedirects OutEchos OutEchoReps OutTimestamps OutTimestampReps OutAddrMasks OutAddrMaskReps
+Icmp: 104 0 0 104 0 0 0 0 0 0 0 0 0 0 120 0 120 0 0 0 0 0 0 0 0 0 0
+IcmpMsg: InType3 OutType3
+IcmpMsg: 104 120
+Tcp: RtoAlgorithm RtoMin RtoMax MaxConn ActiveOpens PassiveOpens AttemptFails EstabResets CurrEstab InSegs OutSegs RetransSegs InErrs OutRsts InCsumErrors
+Tcp: 1 200 120000 -1 3556 230 341 161 0 57252008 54915039 227 5 1003 0
+Udp: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors
+Udp: 88542 120 0 53028 0 0 0
+UdpLite: InDatagrams NoPorts InErrors OutDatagrams RcvbufErrors SndbufErrors InCsumErrors
+UdpLite: 0 0 0 0 0 0 0

collector/fixtures/proc/stat

@@ -0,0 +1,16 @@
+cpu  301854 612 111922 8979004 3552 2 3944 0 0 0
+cpu0 44490 19 21045 1087069 220 1 3410 0 0 0
+cpu1 47869 23 16474 1110787 591 0 46 0 0 0
+cpu2 46504 36 15916 1112321 441 0 326 0 0 0
+cpu3 47054 102 15683 1113230 533 0 60 0 0 0
+cpu4 28413 25 10776 1140321 217 0 8 0 0 0
+cpu5 29271 101 11586 1136270 672 0 30 0 0 0
+cpu6 29152 36 10276 1139721 319 0 29 0 0 0
+cpu7 29098 268 10164 1139282 555 0 31 0 0 0
+intr 8885917 17 0 0 0 0 0 0 0 1 79281 0 0 0 0 0 0 0 231237 0 0 0 0 250586 103 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 223424 190745 13 906 1283803 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0
+ctxt 38014093
+btime 1418183276
+processes 26442
+procs_running 2
+procs_blocked 0
+softirq 5057579 250191 1481983 1647 211099 186066 0 1783454 622196 12499 508444

collector/interrupts_linux_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func TestInterrupts(t *testing.T) {
-	file, err := os.Open("fixtures/interrupts")
+	file, err := os.Open("fixtures/proc/interrupts")
 	if err != nil {
 		t.Fatal(err)
 	}

collector/ipvs_test.go

@@ -120,7 +120,7 @@ var (
 )
 
 func TestIPVSCollector(t *testing.T) {
-	if err := flag.Set("collector.procfs", "fixtures"); err != nil {
+	if err := flag.Set("collector.procfs", "fixtures/proc"); err != nil {
 		t.Fatal(err)
 	}
 	collector, err := newIPVSCollector()
@@ -182,7 +182,7 @@ func (c miniCollector) Describe(ch chan<- *prometheus.Desc) {
 }
 
 func TestIPVSCollectorResponse(t *testing.T) {
-	if err := flag.Set("collector.procfs", "fixtures"); err != nil {
+	if err := flag.Set("collector.procfs", "fixtures/proc"); err != nil {
 		t.Fatal(err)
 	}
 	collector, err := NewIPVSCollector()
@@ -194,7 +194,7 @@ func TestIPVSCollectorResponse(t *testing.T) {
 	rw := httptest.NewRecorder()
 	prometheus.Handler().ServeHTTP(rw, &http.Request{})
 
-	metricsFile := "fixtures/net/ip_vs_result.txt"
+	metricsFile := "fixtures/ip_vs_result.txt"
 	wantMetrics, err := ioutil.ReadFile(metricsFile)
 	if err != nil {
 		t.Fatalf("unable to read input test file %s: %s", metricsFile, err)

collector/mdadm.go

@@ -102,7 +102,7 @@ func evalBuildline(buildline string) (int64, error) {
 func parseMdstat(mdStatusFilePath string) ([]mdStatus, error) {
 	content, err := ioutil.ReadFile(mdStatusFilePath)
 	if err != nil {
-		return []mdStatus{}, fmt.Errorf("error parsing mdstatus: %s", err)
+		return []mdStatus{}, fmt.Errorf("error parsing mdstat: %s", err)
 	}
 
 	mdStatusFile := string(content)
@@ -140,13 +140,13 @@ func parseMdstat(mdStatusFilePath string) ([]mdStatus, error) {
 		isActive := (mainLine[2] == "active") // activity status of said md-device
 
 		if len(lines) <= i+3 {
-			return mdStates, fmt.Errorf("error parsing mdstatus: entry for %s has fewer lines than expected", currentMD)
+			return mdStates, fmt.Errorf("error parsing mdstat: entry for %s has fewer lines than expected", currentMD)
 		}
 
 		active, total, size, err := evalStatusline(lines[i+1]) // parse statusline, always present
 
 		if err != nil {
-			return mdStates, fmt.Errorf("error parsing mdstatus: %s", err)
+			return mdStates, fmt.Errorf("error parsing mdstat: %s", err)
 		}
 
 		// Now get the number of synced blocks.
@@ -165,7 +165,7 @@ func parseMdstat(mdStatusFilePath string) ([]mdStatus, error) {
 		if strings.Contains(lines[j], "recovery") || strings.Contains(lines[j], "resync") {
 			syncedBlocks, err = evalBuildline(lines[j])
 			if err != nil {
-				return mdStates, fmt.Errorf("error parsing mdstatus: %s", err)
+				return mdStates, fmt.Errorf("error parsing mdstat: %s", err)
 			}
 		} else {
 			syncedBlocks = size
@@ -221,11 +221,12 @@ var (
 )
 
 func (c *mdadmCollector) Update(ch chan<- prometheus.Metric) (err error) {
-	statusfile := procFilePath("mdstatus")
+	statusfile := procFilePath("mdstat")
 	// take care we don't crash on non-existent statusfiles
 	_, err = os.Stat(statusfile)
 	if os.IsNotExist(err) {
 		// no such file or directory, nothing to do, just return
+		log.Debugf("Not collecting mdstat, file does not exist: %s", statusfile)
 		return nil
 	}
 

collector/mdadm_test.go

@@ -18,7 +18,7 @@ import (
 )
 
 func TestMdadm(t *testing.T) {
-	mdStates, err := parseMdstat("fixtures/mdstat")
+	mdStates, err := parseMdstat("fixtures/proc/mdstat")
 
 	if err != nil {
 		t.Fatalf("parsing of reference-file failed entirely: %s", err)

collector/meminfo_linux_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func TestMemInfo(t *testing.T) {
-	file, err := os.Open("fixtures/meminfo")
+	file, err := os.Open("fixtures/proc/meminfo")
 	if err != nil {
 		t.Fatal(err)
 	}

collector/netdev_linux_test.go

@@ -20,7 +20,7 @@ import (
 )
 
 func TestNetDevStats(t *testing.T) {
-	file, err := os.Open("fixtures/net-dev")
+	file, err := os.Open("fixtures/proc/net/dev")
 	if err != nil {
 		t.Fatal(err)
 	}

collector/netstat_linux_test.go

@@ -18,7 +18,7 @@ import (
 	"testing"
 )
 
-var fileName = "fixtures/netstat"
+var fileName = "fixtures/proc/net/netstat"
 
 func TestNetStats(t *testing.T) {
 	file, err := os.Open(fileName)

collector/paths.go

@@ -23,8 +23,13 @@ import (
 var (
 	// The path of the proc filesystem.
 	procPath = flag.String("collector.procfs", procfs.DefaultMountPoint, "procfs mountpoint.")
+	sysPath  = flag.String("collector.sysfs", "/sys", "sysfs mountpoint.")
 )
 
 func procFilePath(name string) string {
 	return path.Join(*procPath, name)
 }
+
+func sysFilePath(name string) string {
+	return path.Join(*sysPath, name)
+}

collector/paths_test.go

@@ -0,0 +1,77 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package collector
+
+import (
+	"flag"
+	"testing"
+
+	"github.com/prometheus/procfs"
+)
+
+func TestDefaultProcPath(t *testing.T) {
+	if err := flag.Set("collector.procfs", procfs.DefaultMountPoint); err != nil {
+		t.Fatal(err)
+	}
+
+	if got, want := procFilePath("somefile"), "/proc/somefile"; got != want {
+		t.Errorf("Expected: %s, Got: %s", want, got)
+	}
+
+	if got, want := procFilePath("some/file"), "/proc/some/file"; got != want {
+		t.Errorf("Expected: %s, Got: %s", want, got)
+	}
+}
+
+func TestCustomProcPath(t *testing.T) {
+	if err := flag.Set("collector.procfs", "./../some/./place/"); err != nil {
+		t.Fatal(err)
+	}
+
+	if got, want := procFilePath("somefile"), "../some/place/somefile"; got != want {
+		t.Errorf("Expected: %s, Got: %s", want, got)
+	}
+
+	if got, want := procFilePath("some/file"), "../some/place/some/file"; got != want {
+		t.Errorf("Expected: %s, Got: %s", want, got)
+	}
+}
+
+func TestDefaultSysPath(t *testing.T) {
+	if err := flag.Set("collector.sysfs", "/sys"); err != nil {
+		t.Fatal(err)
+	}
+
+	if got, want := sysFilePath("somefile"), "/sys/somefile"; got != want {
+		t.Errorf("Expected: %s, Got: %s", want, got)
+	}
+
+	if got, want := sysFilePath("some/file"), "/sys/some/file"; got != want {
+		t.Errorf("Expected: %s, Got: %s", want, got)
+	}
+}
+
+func TestCustomSysPath(t *testing.T) {
+	if err := flag.Set("collector.sysfs", "./../some/./place/"); err != nil {
+		t.Fatal(err)
+	}
+
+	if got, want := sysFilePath("somefile"), "../some/place/somefile"; got != want {
+		t.Errorf("Expected: %s, Got: %s", want, got)
+	}
+
+	if got, want := sysFilePath("some/file"), "../some/place/some/file"; got != want {
+		t.Errorf("Expected: %s, Got: %s", want, got)
+	}
+}

collector/sockstat_linux_test.go

@@ -20,7 +20,7 @@ import (
 )
 
 func TestSockStats(t *testing.T) {
-	file, err := os.Open("fixtures/sockstat")
+	file, err := os.Open("fixtures/proc/net/sockstat")
 	if err != nil {
 		t.Fatal(err)
 	}

collector/tcpstat_linux_test.go

@@ -19,7 +19,7 @@ import (
 )
 
 func TestTCPStat(t *testing.T) {
-	file, err := os.Open("fixtures/tcpstat")
+	file, err := os.Open("fixtures/proc/net/tcpstat")
 	if err != nil {
 		t.Fatal(err)
 	}

collector/textfile_test.go

@@ -14,6 +14,7 @@
 package collector
 
 import (
+	"flag"
 	"io/ioutil"
 	"sort"
 	"strings"
@@ -46,6 +47,13 @@ func TestParseTextFiles(t *testing.T) {
 			path: test.path,
 		}
 
+		// Suppress a log message about `nonexistent_path` not existing, this is
+		// expected and clutters the test output.
+		err := flag.Set("log.level", "fatal")
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		mfs := c.parseTextFiles()
 		textMFs := make([]string, 0, len(mfs))
 		for _, mf := range mfs {

end-to-end-test.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+set +euf +o pipefail
+
+cd "$(dirname $0)"
+
+port="$((10000 + (RANDOM % 10000)))"
+tmpdir=$(mktemp -d /tmp/node_exporter_e2e_test.XXXXXX)
+
+skip_re="^(go_|node_exporter_|process_|node_textfile_mtime)"
+
+keep=0; update=0; verbose=0
+while getopts 'hkuv' opt
+do
+  case "$opt" in
+    k)
+      keep=1
+      ;;
+    u)
+      update=1
+      ;;
+    v)
+      verbose=1
+      set -x
+      ;;
+    *)
+      echo "Usage: $0 [-k] [-u] [-v]"
+      echo "  -k: keep temporary files and leave node_exporter running"
+      echo "  -u: update fixture"
+      echo "  -v: verbose output"
+      exit 1
+      ;;
+  esac
+done
+
+
+./node_exporter \
+  -collector.procfs="collector/fixtures/proc" \
+  -collector.sysfs="collector/fixtures/sys" \
+  -collectors.enabled="diskstats,filefd,loadavg,mdadm,meminfo,netdev,netstat,sockstat,stat,textfile,bonding,megacli" \
+  -collector.textfile.directory="collector/fixtures/textfile/two_metric_files/" \
+  -collector.megacli.command="collector/fixtures/megacli" \
+  -web.listen-address "127.0.0.1:${port}" \
+  -log.level="debug" > "${tmpdir}/node_exporter.log" 2>&1 &
+
+echo $! > "${tmpdir}/node_exporter.pid"
+
+finish() {
+  if [ ${verbose} -ne 0 ]
+  then
+    echo "LOG ====================="
+    cat "${tmpdir}/node_exporter.log"
+    echo "========================="
+  fi
+
+  if [ ${update} -ne 0 ]
+  then
+    cp "${tmpdir}/e2e-output.txt" "collector/fixtures/e2e-output.txt"
+  fi
+
+  if [ ${keep} -eq 0 ]
+  then
+    kill -9 "$(cat ${tmpdir}/node_exporter.pid)"
+    # This silences the "Killed" message
+    wait "$(cat ${tmpdir}/node_exporter.pid)" > /dev/null 2>&1
+    rm -rf "${tmpdir}"
+  fi
+}
+
+trap finish EXIT
+
+get() {
+  if which curl > /dev/null 2>&1
+  then
+    curl -s -f "$@"
+  elif which wget > /dev/null 2>&1
+  then
+    wget -O - "$@"
+  else
+    echo "Neither curl nor wget found"
+    exit 1
+  fi
+}
+
+sleep 1
+
+get "127.0.0.1:${port}/metrics" > "${tmpdir}/e2e-output.txt"
+
+diff -u \
+  <(grep -E -v "${skip_re}" "collector/fixtures/e2e-output.txt") \
+  <(grep -E -v "${skip_re}" "${tmpdir}/e2e-output.txt")