From ed7a10dde280a7d355c912a089bacbdffa46a18a Mon Sep 17 00:00:00 2001 From: Marc Tuduri Date: Tue, 26 Sep 2023 15:17:45 +0200 Subject: [PATCH] Refactor collectors - part 2 --- collector/config.go | 11 ++++++ collector/logind_linux.go | 1 - collector/qdisc_linux.go | 28 ++++++++------- collector/rapl_linux.go | 18 ++++++---- collector/runit.go | 18 ++++++---- collector/schedstat_linux.go | 6 ++-- collector/selinux_linux.go | 6 ++-- collector/slabinfo_linux.go | 6 ++-- collector/sockstat_linux.go | 6 ++-- collector/softirqs_common.go | 7 ++-- collector/softnet_linux.go | 6 ++-- collector/stat_linux.go | 18 ++++++---- collector/supervisord.go | 19 +++++++---- collector/sysctl_linux.go | 20 ++++++----- collector/systemd_linux.go | 60 ++++++++++++++++----------------- collector/tapestats_linux.go | 18 +++++----- collector/tcpstat_linux.go | 6 ++-- collector/textfile.go | 17 ++++++---- collector/thermal_darwin.go | 4 ++- collector/thermal_zone_linux.go | 6 ++-- collector/time.go | 6 ++-- collector/timex.go | 6 ++-- collector/udp_queues_linux.go | 6 ++-- collector/uname.go | 6 ++-- collector/vmstat_linux.go | 18 +++++----- collector/wifi_linux.go | 20 ++++++----- collector/xfs_linux.go | 6 ++-- collector/zfs.go | 6 ++-- collector/zfs_freebsd.go | 6 ++-- collector/zfs_solaris.go | 4 ++- collector/zoneinfo_linux.go | 6 ++-- kingpinconfig/flags.go | 38 +++++++++++++++++++++ 32 files changed, 268 insertions(+), 141 deletions(-) diff --git a/collector/config.go b/collector/config.go index a2247a71..7abd93a8 100644 --- a/collector/config.go +++ b/collector/config.go @@ -31,4 +31,15 @@ type NodeCollectorConfig struct { NTP NTPConfig Perf PerfConfig PowerSupplyClass PowerSupplyClassConfig + Qdisc QdiscConfig + Rapl RaplConfig + Runit RunitConfig + Stat StatConfig + Supervisord SupervisordConfig + Sysctl SysctlConfig + Systemd SystemdConfig + Tapestats TapestatsConfig + TextFile TextFileConfig + VmStat VmStatConfig + Wifi WifiConfig } diff --git a/collector/logind_linux.go b/collector/logind_linux.go index 9958edbd..42c0da36 100644 --- a/collector/logind_linux.go +++ b/collector/logind_linux.go @@ -82,7 +82,6 @@ type logindSeatEntry struct { } func init() { - registerCollector("logind", defaultDisabled, NewLogindCollector) registerCollector("logind", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { return NewLogindCollector(logger) }) diff --git a/collector/qdisc_linux.go b/collector/qdisc_linux.go index 6944d598..d5cf96f3 100644 --- a/collector/qdisc_linux.go +++ b/collector/qdisc_linux.go @@ -22,7 +22,6 @@ import ( "os" "path/filepath" - "github.com/alecthomas/kingpin/v2" "github.com/ema/qdisc" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -38,21 +37,25 @@ type qdiscStatCollector struct { overlimits typedDesc qlength typedDesc backlog typedDesc + config QdiscConfig } -var ( - collectorQdisc = kingpin.Flag("collector.qdisc.fixtures", "test fixtures to use for qdisc collector end-to-end testing").Default("").String() - collectorQdiskDeviceInclude = kingpin.Flag("collector.qdisk.device-include", "Regexp of qdisk devices to include (mutually exclusive to device-exclude).").String() - collectorQdiskDeviceExclude = kingpin.Flag("collector.qdisk.device-exclude", "Regexp of qdisk devices to exclude (mutually exclusive to device-include).").String() -) - func init() { - registerCollector("qdisc", defaultDisabled, NewQdiscStatCollector) + registerCollector("qdisc", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(QdiscConfig) + return NewQdiscStatCollector(cfg, logger) + }) +} + +type QdiscConfig struct { + Fixtures *string + DeviceInclude *string + DeviceExclude *string } // NewQdiscStatCollector returns a new Collector exposing queuing discipline statistics. -func NewQdiscStatCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { - if *collectorQdiskDeviceExclude != "" && *collectorQdiskDeviceInclude != "" { +func NewQdiscStatCollector(config QdiscConfig, logger log.Logger) (Collector, error) { + if *config.DeviceExclude != "" && *config.DeviceInclude != "" { return nil, fmt.Errorf("collector.qdisk.device-include and collector.qdisk.device-exclude are mutaly exclusive") } @@ -93,7 +96,8 @@ func NewQdiscStatCollector(config NodeCollectorConfig, logger log.Logger) (Colle []string{"device", "kind"}, nil, ), prometheus.GaugeValue}, logger: logger, - deviceFilter: newDeviceFilter(*collectorQdiskDeviceExclude, *collectorQdiskDeviceExclude), + deviceFilter: newDeviceFilter(*config.DeviceExclude, *config.DeviceExclude), + config: config, }, nil } @@ -113,7 +117,7 @@ func (c *qdiscStatCollector) Update(ch chan<- prometheus.Metric) error { var msgs []qdisc.QdiscInfo var err error - fixtures := *collectorQdisc + fixtures := *c.config.Fixtures if fixtures == "" { msgs, err = qdisc.Get() diff --git a/collector/rapl_linux.go b/collector/rapl_linux.go index a908c0ec..0158285f 100644 --- a/collector/rapl_linux.go +++ b/collector/rapl_linux.go @@ -22,7 +22,6 @@ import ( "os" "strconv" - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -36,18 +35,22 @@ type raplCollector struct { logger log.Logger joulesMetricDesc *prometheus.Desc + config RaplConfig } func init() { - registerCollector(raplCollectorSubsystem, defaultEnabled, NewRaplCollector) + registerCollector(raplCollectorSubsystem, defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(RaplConfig) + return NewRaplCollector(cfg, logger) + }) } -var ( - raplZoneLabel = kingpin.Flag("collector.rapl.enable-zone-label", "Enables service unit metric unit_start_time_seconds").Bool() -) +type RaplConfig struct { + ZoneLabel *bool +} // NewRaplCollector returns a new Collector exposing RAPL metrics. -func NewRaplCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewRaplCollector(config RaplConfig, logger log.Logger) (Collector, error) { fs, err := sysfs.NewFS(*sysPath) if err != nil { @@ -64,6 +67,7 @@ func NewRaplCollector(config NodeCollectorConfig, logger log.Logger) (Collector, fs: fs, logger: logger, joulesMetricDesc: joulesMetricDesc, + config: config, } return &collector, nil } @@ -96,7 +100,7 @@ func (c *raplCollector) Update(ch chan<- prometheus.Metric) error { joules := float64(microJoules) / 1000000.0 - if *raplZoneLabel { + if *c.config.ZoneLabel { ch <- c.joulesMetricWithZoneLabel(rz, joules) } else { ch <- c.joulesMetric(rz, joules) diff --git a/collector/runit.go b/collector/runit.go index 913c4833..483dbe11 100644 --- a/collector/runit.go +++ b/collector/runit.go @@ -17,29 +17,34 @@ package collector import ( - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus-community/go-runit/runit" "github.com/prometheus/client_golang/prometheus" ) -var runitServiceDir = kingpin.Flag("collector.runit.servicedir", "Path to runit service directory.").Default("/etc/service").String() - type runitCollector struct { state typedDesc stateDesired typedDesc stateNormal typedDesc stateTimestamp typedDesc logger log.Logger + config RunitConfig } func init() { - registerCollector("runit", defaultDisabled, NewRunitCollector) + registerCollector("runit", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(RunitConfig) + return NewRunitCollector(cfg, logger) + }) +} + +type RunitConfig struct { + ServiceDir *string } // NewRunitCollector returns a new Collector exposing runit statistics. -func NewRunitCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewRunitCollector(config RunitConfig, logger log.Logger) (Collector, error) { var ( subsystem = "service" constLabels = prometheus.Labels{"supervisor": "runit"} @@ -70,11 +75,12 @@ func NewRunitCollector(config NodeCollectorConfig, logger log.Logger) (Collector labelNames, constLabels, ), prometheus.GaugeValue}, logger: logger, + config: config, }, nil } func (c *runitCollector) Update(ch chan<- prometheus.Metric) error { - services, err := runit.GetServices(*runitServiceDir) + services, err := runit.GetServices(*c.config.ServiceDir) if err != nil { return err } diff --git a/collector/schedstat_linux.go b/collector/schedstat_linux.go index 1d4d85e2..d2f566bb 100644 --- a/collector/schedstat_linux.go +++ b/collector/schedstat_linux.go @@ -53,7 +53,7 @@ var ( ) // NewSchedstatCollector returns a new Collector exposing task scheduler statistics -func NewSchedstatCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewSchedstatCollector(logger log.Logger) (Collector, error) { fs, err := procfs.NewFS(*procPath) if err != nil { return nil, fmt.Errorf("failed to open procfs: %w", err) @@ -68,7 +68,9 @@ type schedstatCollector struct { } func init() { - registerCollector("schedstat", defaultEnabled, NewSchedstatCollector) + registerCollector("schedstat", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewSchedstatCollector(logger) + }) } func (c *schedstatCollector) Update(ch chan<- prometheus.Metric) error { diff --git a/collector/selinux_linux.go b/collector/selinux_linux.go index fd00da56..970cdb9c 100644 --- a/collector/selinux_linux.go +++ b/collector/selinux_linux.go @@ -30,11 +30,13 @@ type selinuxCollector struct { } func init() { - registerCollector("selinux", defaultEnabled, NewSelinuxCollector) + registerCollector("selinux", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewSelinuxCollector(logger) + }) } // NewSelinuxCollector returns a new Collector exposing SELinux statistics. -func NewSelinuxCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewSelinuxCollector(logger log.Logger) (Collector, error) { const subsystem = "selinux" return &selinuxCollector{ diff --git a/collector/slabinfo_linux.go b/collector/slabinfo_linux.go index ad7667a8..d8f0ed8f 100644 --- a/collector/slabinfo_linux.go +++ b/collector/slabinfo_linux.go @@ -32,10 +32,12 @@ type slabinfoCollector struct { } func init() { - registerCollector("slabinfo", defaultDisabled, NewSlabinfoCollector) + registerCollector("slabinfo", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + return NewSlabinfoCollector(logger) + }) } -func NewSlabinfoCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewSlabinfoCollector(logger log.Logger) (Collector, error) { fs, err := procfs.NewFS(*procPath) if err != nil { return nil, fmt.Errorf("failed to open procfs: %w", err) diff --git a/collector/sockstat_linux.go b/collector/sockstat_linux.go index 7ae194e0..cf2831fa 100644 --- a/collector/sockstat_linux.go +++ b/collector/sockstat_linux.go @@ -39,11 +39,13 @@ type sockStatCollector struct { } func init() { - registerCollector(sockStatSubsystem, defaultEnabled, NewSockStatCollector) + registerCollector(sockStatSubsystem, defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewSockStatCollector(logger) + }) } // NewSockStatCollector returns a new Collector exposing socket stats. -func NewSockStatCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewSockStatCollector(logger log.Logger) (Collector, error) { return &sockStatCollector{logger}, nil } diff --git a/collector/softirqs_common.go b/collector/softirqs_common.go index 7306b7aa..1bfac97d 100644 --- a/collector/softirqs_common.go +++ b/collector/softirqs_common.go @@ -18,6 +18,7 @@ package collector import ( "fmt" + "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs" @@ -30,11 +31,13 @@ type softirqsCollector struct { } func init() { - registerCollector("softirqs", defaultDisabled, NewSoftirqsCollector) + registerCollector("softirqs", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + return NewSoftirqsCollector(logger) + }) } // NewSoftirqsCollector returns a new Collector exposing softirq stats. -func NewSoftirqsCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewSoftirqsCollector(logger log.Logger) (Collector, error) { desc := typedDesc{prometheus.NewDesc( namespace+"_softirqs_functions_total", "Softirq counts per CPU.", diff --git a/collector/softnet_linux.go b/collector/softnet_linux.go index 654226d9..954c4c59 100644 --- a/collector/softnet_linux.go +++ b/collector/softnet_linux.go @@ -42,11 +42,13 @@ const ( ) func init() { - registerCollector("softnet", defaultEnabled, NewSoftnetCollector) + registerCollector("softnet", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewSoftnetCollector(logger) + }) } // NewSoftnetCollector returns a new Collector exposing softnet metrics. -func NewSoftnetCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewSoftnetCollector(logger log.Logger) (Collector, error) { fs, err := procfs.NewFS(*procPath) if err != nil { return nil, fmt.Errorf("failed to open procfs: %w", err) diff --git a/collector/stat_linux.go b/collector/stat_linux.go index 2e75a5fd..cf77368b 100644 --- a/collector/stat_linux.go +++ b/collector/stat_linux.go @@ -19,7 +19,6 @@ package collector import ( "fmt" - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs" @@ -35,16 +34,22 @@ type statCollector struct { procsBlocked *prometheus.Desc softIRQ *prometheus.Desc logger log.Logger + config StatConfig } -var statSoftirqFlag = kingpin.Flag("collector.stat.softirq", "Export softirq calls per vector").Default("false").Bool() - func init() { - registerCollector("stat", defaultEnabled, NewStatCollector) + registerCollector("stat", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(StatConfig) + return NewStatCollector(cfg, logger) + }) +} + +type StatConfig struct { + Softirq *bool } // NewStatCollector returns a new Collector exposing kernel/system statistics. -func NewStatCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewStatCollector(config StatConfig, logger log.Logger) (Collector, error) { fs, err := procfs.NewFS(*procPath) if err != nil { return nil, fmt.Errorf("failed to open procfs: %w", err) @@ -87,6 +92,7 @@ func NewStatCollector(config NodeCollectorConfig, logger log.Logger) (Collector, []string{"vector"}, nil, ), logger: logger, + config: config, }, nil } @@ -106,7 +112,7 @@ func (c *statCollector) Update(ch chan<- prometheus.Metric) error { ch <- prometheus.MustNewConstMetric(c.procsRunning, prometheus.GaugeValue, float64(stats.ProcessesRunning)) ch <- prometheus.MustNewConstMetric(c.procsBlocked, prometheus.GaugeValue, float64(stats.ProcessesBlocked)) - if *statSoftirqFlag { + if *c.config.Softirq { si := stats.SoftIRQ for _, vec := range []struct { diff --git a/collector/supervisord.go b/collector/supervisord.go index d53a9fcb..f44f95b6 100644 --- a/collector/supervisord.go +++ b/collector/supervisord.go @@ -24,7 +24,6 @@ import ( "net/url" "time" - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/mattn/go-xmlrpc" @@ -32,8 +31,7 @@ import ( ) var ( - supervisordURL = kingpin.Flag("collector.supervisord.url", "XML RPC endpoint.").Default("http://localhost:9001/RPC2").Envar("SUPERVISORD_URL").String() - xrpc *xmlrpc.Client + xrpc *xmlrpc.Client ) type supervisordCollector struct { @@ -45,17 +43,24 @@ type supervisordCollector struct { } func init() { - registerCollector("supervisord", defaultDisabled, NewSupervisordCollector) + registerCollector("supervisord", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(SupervisordConfig) + return NewSupervisordCollector(cfg, logger) + }) +} + +type SupervisordConfig struct { + URL *string } // NewSupervisordCollector returns a new Collector exposing supervisord statistics. -func NewSupervisordCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewSupervisordCollector(config SupervisordConfig, logger log.Logger) (Collector, error) { var ( subsystem = "supervisord" labelNames = []string{"name", "group"} ) - if u, err := url.Parse(*supervisordURL); err == nil && u.Scheme == "unix" { + if u, err := url.Parse(*config.URL); err == nil && u.Scheme == "unix" { // Fake the URI scheme as http, since net/http.*Transport.roundTrip will complain // about a non-http(s) transport. xrpc = xmlrpc.NewClient("http://unix/RPC2") @@ -66,7 +71,7 @@ func NewSupervisordCollector(config NodeCollectorConfig, logger log.Logger) (Col }, } } else { - xrpc = xmlrpc.NewClient(*supervisordURL) + xrpc = xmlrpc.NewClient(*config.URL) } level.Warn(logger).Log("msg", "This collector is deprecated and will be removed in the next major version release.") diff --git a/collector/sysctl_linux.go b/collector/sysctl_linux.go index bc1a3307..9954e4aa 100644 --- a/collector/sysctl_linux.go +++ b/collector/sysctl_linux.go @@ -18,16 +18,12 @@ import ( "strconv" "strings" - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs" ) var ( - sysctlInclude = kingpin.Flag("collector.sysctl.include", "Select sysctl metrics to include").Strings() - sysctlIncludeInfo = kingpin.Flag("collector.sysctl.include-info", "Select sysctl metrics to include as info metrics").Strings() - sysctlInfoDesc = prometheus.NewDesc(prometheus.BuildFQName(namespace, "sysctl", "info"), "sysctl info", []string{"name", "value", "index"}, nil) ) @@ -38,10 +34,18 @@ type sysctlCollector struct { } func init() { - registerCollector("sysctl", defaultDisabled, NewSysctlCollector) + registerCollector("sysctl", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(SysctlConfig) + return NewSysctlCollector(cfg, logger) + }) } -func NewSysctlCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +type SysctlConfig struct { + Include *[]string + IncludeInfo *[]string +} + +func NewSysctlCollector(config SysctlConfig, logger log.Logger) (Collector, error) { fs, err := procfs.NewFS(*procPath) if err != nil { return nil, fmt.Errorf("failed to open sysfs: %w", err) @@ -52,7 +56,7 @@ func NewSysctlCollector(config NodeCollectorConfig, logger log.Logger) (Collecto sysctls: []*sysctl{}, } - for _, include := range *sysctlInclude { + for _, include := range *config.Include { sysctl, err := newSysctl(include, true) if err != nil { return nil, err @@ -60,7 +64,7 @@ func NewSysctlCollector(config NodeCollectorConfig, logger log.Logger) (Collecto c.sysctls = append(c.sysctls, sysctl) } - for _, include := range *sysctlIncludeInfo { + for _, include := range *config.IncludeInfo { sysctl, err := newSysctl(include, false) if err != nil { return nil, err diff --git a/collector/systemd_linux.go b/collector/systemd_linux.go index 4ef3fbd2..3fe79316 100644 --- a/collector/systemd_linux.go +++ b/collector/systemd_linux.go @@ -27,7 +27,6 @@ import ( "sync" "time" - "github.com/alecthomas/kingpin/v2" "github.com/coreos/go-systemd/v22/dbus" "github.com/go-kit/log" "github.com/go-kit/log/level" @@ -43,21 +42,7 @@ const ( var ( systemdUnitIncludeSet bool - systemdUnitInclude = kingpin.Flag("collector.systemd.unit-include", "Regexp of systemd units to include. Units must both match include and not match exclude to be included.").Default(".+").PreAction(func(c *kingpin.ParseContext) error { - systemdUnitIncludeSet = true - return nil - }).String() - oldSystemdUnitInclude = kingpin.Flag("collector.systemd.unit-whitelist", "DEPRECATED: Use --collector.systemd.unit-include").Hidden().String() systemdUnitExcludeSet bool - systemdUnitExclude = kingpin.Flag("collector.systemd.unit-exclude", "Regexp of systemd units to exclude. Units must both match include and not match exclude to be included.").Default(".+\\.(automount|device|mount|scope|slice)").PreAction(func(c *kingpin.ParseContext) error { - systemdUnitExcludeSet = true - return nil - }).String() - oldSystemdUnitExclude = kingpin.Flag("collector.systemd.unit-blacklist", "DEPRECATED: Use collector.systemd.unit-exclude").Hidden().String() - systemdPrivate = kingpin.Flag("collector.systemd.private", "Establish a private, direct connection to systemd without dbus (Strongly discouraged since it requires root. For testing purposes only).").Hidden().Bool() - enableTaskMetrics = kingpin.Flag("collector.systemd.enable-task-metrics", "Enables service unit tasks metrics unit_tasks_current and unit_tasks_max").Bool() - enableRestartsMetrics = kingpin.Flag("collector.systemd.enable-restarts-metrics", "Enables service unit metric service_restart_total").Bool() - enableStartTimeMetrics = kingpin.Flag("collector.systemd.enable-start-time-metrics", "Enables service unit metric unit_start_time_seconds").Bool() systemdVersionRE = regexp.MustCompile(`[0-9]{3,}(\.[0-9]+)?`) ) @@ -79,16 +64,31 @@ type systemdCollector struct { systemdUnitIncludePattern *regexp.Regexp systemdUnitExcludePattern *regexp.Regexp logger log.Logger + config SystemdConfig } var unitStatesName = []string{"active", "activating", "deactivating", "inactive", "failed"} func init() { - registerCollector("systemd", defaultDisabled, NewSystemdCollector) + registerCollector("systemd", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(SystemdConfig) + return NewSystemdCollector(cfg, logger) + }) +} + +type SystemdConfig struct { + UnitInclude *string + UnitExclude *string + OldUnitInclude *string + OldUnitExclude *string + Private *bool + EnableTaskMetrics *bool + EnableRestartsMetrics *bool + EnableStartTimeMetrics *bool } // NewSystemdCollector returns a new Collector exposing systemd statistics. -func NewSystemdCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewSystemdCollector(config SystemdConfig, logger log.Logger) (Collector, error) { const subsystem = "systemd" unitDesc := prometheus.NewDesc( @@ -134,26 +134,26 @@ func NewSystemdCollector(config NodeCollectorConfig, logger log.Logger) (Collect prometheus.BuildFQName(namespace, subsystem, "version"), "Detected systemd version", []string{"version"}, nil) - if *oldSystemdUnitExclude != "" { + if *config.OldUnitExclude != "" { if !systemdUnitExcludeSet { level.Warn(logger).Log("msg", "--collector.systemd.unit-blacklist is DEPRECATED and will be removed in 2.0.0, use --collector.systemd.unit-exclude") - *systemdUnitExclude = *oldSystemdUnitExclude + *config.UnitExclude = *config.OldUnitExclude } else { return nil, errors.New("--collector.systemd.unit-blacklist and --collector.systemd.unit-exclude are mutually exclusive") } } - if *oldSystemdUnitInclude != "" { + if *config.OldUnitInclude != "" { if !systemdUnitIncludeSet { level.Warn(logger).Log("msg", "--collector.systemd.unit-whitelist is DEPRECATED and will be removed in 2.0.0, use --collector.systemd.unit-include") - *systemdUnitInclude = *oldSystemdUnitInclude + *config.UnitInclude = *config.OldUnitInclude } else { return nil, errors.New("--collector.systemd.unit-whitelist and --collector.systemd.unit-include are mutually exclusive") } } - level.Info(logger).Log("msg", "Parsed flag --collector.systemd.unit-include", "flag", *systemdUnitInclude) - systemdUnitIncludePattern := regexp.MustCompile(fmt.Sprintf("^(?:%s)$", *systemdUnitInclude)) - level.Info(logger).Log("msg", "Parsed flag --collector.systemd.unit-exclude", "flag", *systemdUnitExclude) - systemdUnitExcludePattern := regexp.MustCompile(fmt.Sprintf("^(?:%s)$", *systemdUnitExclude)) + level.Info(logger).Log("msg", "Parsed flag --collector.systemd.unit-include", "flag", *config.UnitInclude) + systemdUnitIncludePattern := regexp.MustCompile(fmt.Sprintf("^(?:%s)$", *config.UnitInclude)) + level.Info(logger).Log("msg", "Parsed flag --collector.systemd.unit-exclude", "flag", *config.UnitExclude) + systemdUnitExcludePattern := regexp.MustCompile(fmt.Sprintf("^(?:%s)$", *config.UnitExclude)) return &systemdCollector{ unitDesc: unitDesc, @@ -178,7 +178,7 @@ func NewSystemdCollector(config NodeCollectorConfig, logger log.Logger) (Collect // to reduce wait time for responses. func (c *systemdCollector) Update(ch chan<- prometheus.Metric) error { begin := time.Now() - conn, err := newSystemdDbusConn() + conn, err := newSystemdDbusConn(c.config.Private) if err != nil { return fmt.Errorf("couldn't get dbus connection: %w", err) } @@ -221,7 +221,7 @@ func (c *systemdCollector) Update(ch chan<- prometheus.Metric) error { level.Debug(c.logger).Log("msg", "collectUnitStatusMetrics took", "duration_seconds", time.Since(begin).Seconds()) }() - if *enableStartTimeMetrics { + if *c.config.EnableStartTimeMetrics { wg.Add(1) go func() { defer wg.Done() @@ -231,7 +231,7 @@ func (c *systemdCollector) Update(ch chan<- prometheus.Metric) error { }() } - if *enableTaskMetrics { + if *c.config.EnableTaskMetrics { wg.Add(1) go func() { defer wg.Done() @@ -295,7 +295,7 @@ func (c *systemdCollector) collectUnitStatusMetrics(conn *dbus.Conn, ch chan<- p c.unitDesc, prometheus.GaugeValue, isActive, unit.Name, stateName, serviceType) } - if *enableRestartsMetrics && strings.HasSuffix(unit.Name, ".service") { + if *c.config.EnableRestartsMetrics && strings.HasSuffix(unit.Name, ".service") { // NRestarts wasn't added until systemd 235. restartsCount, err := conn.GetUnitTypePropertyContext(context.TODO(), unit.Name, "Service", "NRestarts") if err != nil { @@ -434,7 +434,7 @@ func (c *systemdCollector) collectSystemState(conn *dbus.Conn, ch chan<- prometh return nil } -func newSystemdDbusConn() (*dbus.Conn, error) { +func newSystemdDbusConn(systemdPrivate *bool) (*dbus.Conn, error) { if *systemdPrivate { return dbus.NewSystemdConnectionContext(context.TODO()) } diff --git a/collector/tapestats_linux.go b/collector/tapestats_linux.go index b0a91014..42af04c6 100644 --- a/collector/tapestats_linux.go +++ b/collector/tapestats_linux.go @@ -21,17 +21,12 @@ import ( "os" "regexp" - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/procfs/sysfs" ) -var ( - ignoredTapeDevices = kingpin.Flag("collector.tapestats.ignored-devices", "Regexp of devices to ignore for tapestats.").Default("^$").String() -) - type tapestatsCollector struct { ignoredDevicesPattern *regexp.Regexp ioNow *prometheus.Desc @@ -49,12 +44,19 @@ type tapestatsCollector struct { } func init() { - registerCollector("tapestats", defaultEnabled, NewTapestatsCollector) + registerCollector("tapestats", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(TapestatsConfig) + return NewTapestatsCollector(cfg, logger) + }) +} + +type TapestatsConfig struct { + IgnoredDevices *string } // NewTapestatsCollector returns a new Collector exposing tape device stats. // Docs from https://www.kernel.org/doc/html/latest/scsi/st.html#sysfs-and-statistics-for-tape-devices -func NewTapestatsCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewTapestatsCollector(config TapestatsConfig, logger log.Logger) (Collector, error) { var tapeLabelNames = []string{"device"} fs, err := sysfs.NewFS(*sysPath) @@ -65,7 +67,7 @@ func NewTapestatsCollector(config NodeCollectorConfig, logger log.Logger) (Colle tapeSubsystem := "tape" return &tapestatsCollector{ - ignoredDevicesPattern: regexp.MustCompile(*ignoredTapeDevices), + ignoredDevicesPattern: regexp.MustCompile(*config.IgnoredDevices), ioNow: prometheus.NewDesc( prometheus.BuildFQName(namespace, tapeSubsystem, "io_now"), diff --git a/collector/tcpstat_linux.go b/collector/tcpstat_linux.go index f18058d6..120face8 100644 --- a/collector/tcpstat_linux.go +++ b/collector/tcpstat_linux.go @@ -64,11 +64,13 @@ type tcpStatCollector struct { } func init() { - registerCollector("tcpstat", defaultDisabled, NewTCPStatCollector) + registerCollector("tcpstat", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + return NewTCPStatCollector(logger) + }) } // NewTCPStatCollector returns a new Collector exposing network stats. -func NewTCPStatCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewTCPStatCollector(logger log.Logger) (Collector, error) { return &tcpStatCollector{ desc: typedDesc{prometheus.NewDesc( prometheus.BuildFQName(namespace, "tcp", "connection_states"), diff --git a/collector/textfile.go b/collector/textfile.go index 3d944727..e9f26981 100644 --- a/collector/textfile.go +++ b/collector/textfile.go @@ -24,7 +24,6 @@ import ( "strings" "time" - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -33,8 +32,7 @@ import ( ) var ( - textFileDirectory = kingpin.Flag("collector.textfile.directory", "Directory to read text files with metrics from.").Default("").String() - mtimeDesc = prometheus.NewDesc( + mtimeDesc = prometheus.NewDesc( "node_textfile_mtime_seconds", "Unixtime mtime of textfiles successfully read.", []string{"file"}, @@ -50,14 +48,21 @@ type textFileCollector struct { } func init() { - registerCollector("textfile", defaultEnabled, NewTextFileCollector) + registerCollector("textfile", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(TextFileConfig) + return NewTextFileCollector(cfg, logger) + }) +} + +type TextFileConfig struct { + Directory *string } // NewTextFileCollector returns a new Collector exposing metrics read from files // in the given textfile directory. -func NewTextFileCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewTextFileCollector(config TextFileConfig, logger log.Logger) (Collector, error) { c := &textFileCollector{ - path: *textFileDirectory, + path: *config.Directory, logger: logger, } return c, nil diff --git a/collector/thermal_darwin.go b/collector/thermal_darwin.go index d7330970..6bb4d0f9 100644 --- a/collector/thermal_darwin.go +++ b/collector/thermal_darwin.go @@ -63,7 +63,9 @@ type thermCollector struct { const thermal = "thermal" func init() { - registerCollector(thermal, defaultEnabled, NewThermCollector) + registerCollector(thermal, defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewThermCollector(logger) + }) } // NewThermCollector returns a new Collector exposing current CPU power levels. diff --git a/collector/thermal_zone_linux.go b/collector/thermal_zone_linux.go index dde39203..7f1acbb9 100644 --- a/collector/thermal_zone_linux.go +++ b/collector/thermal_zone_linux.go @@ -39,11 +39,13 @@ type thermalZoneCollector struct { } func init() { - registerCollector("thermal_zone", defaultEnabled, NewThermalZoneCollector) + registerCollector("thermal_zone", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewThermalZoneCollector(logger) + }) } // NewThermalZoneCollector returns a new Collector exposing kernel/system statistics. -func NewThermalZoneCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewThermalZoneCollector(logger log.Logger) (Collector, error) { fs, err := sysfs.NewFS(*sysPath) if err != nil { return nil, fmt.Errorf("failed to open sysfs: %w", err) diff --git a/collector/time.go b/collector/time.go index a245eeb5..be510c30 100644 --- a/collector/time.go +++ b/collector/time.go @@ -33,12 +33,14 @@ type timeCollector struct { } func init() { - registerCollector("time", defaultEnabled, NewTimeCollector) + registerCollector("time", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewTimeCollector(logger) + }) } // NewTimeCollector returns a new Collector exposing the current system time in // seconds since epoch. -func NewTimeCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewTimeCollector(logger log.Logger) (Collector, error) { const subsystem = "time" return &timeCollector{ now: typedDesc{prometheus.NewDesc( diff --git a/collector/timex.go b/collector/timex.go index 5fa6014e..93d1d01a 100644 --- a/collector/timex.go +++ b/collector/timex.go @@ -62,11 +62,13 @@ type timexCollector struct { } func init() { - registerCollector("timex", defaultEnabled, NewTimexCollector) + registerCollector("timex", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewTimexCollector(logger) + }) } // NewTimexCollector returns a new Collector exposing adjtime(3) stats. -func NewTimexCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewTimexCollector(logger log.Logger) (Collector, error) { const subsystem = "timex" return &timexCollector{ diff --git a/collector/udp_queues_linux.go b/collector/udp_queues_linux.go index c58634d7..3e30e39f 100644 --- a/collector/udp_queues_linux.go +++ b/collector/udp_queues_linux.go @@ -36,11 +36,13 @@ type ( ) func init() { - registerCollector("udp_queues", defaultEnabled, NewUDPqueuesCollector) + registerCollector("udp_queues", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewUDPqueuesCollector(logger) + }) } // NewUDPqueuesCollector returns a new Collector exposing network udp queued bytes. -func NewUDPqueuesCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewUDPqueuesCollector(logger log.Logger) (Collector, error) { fs, err := procfs.NewFS(*procPath) if err != nil { return nil, fmt.Errorf("failed to open procfs: %w", err) diff --git a/collector/uname.go b/collector/uname.go index e03101db..46b1cb51 100644 --- a/collector/uname.go +++ b/collector/uname.go @@ -49,11 +49,13 @@ type uname struct { } func init() { - registerCollector("uname", defaultEnabled, newUnameCollector) + registerCollector("uname", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return newUnameCollector(logger) + }) } // NewUnameCollector returns new unameCollector. -func newUnameCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func newUnameCollector(logger log.Logger) (Collector, error) { return &unameCollector{logger}, nil } diff --git a/collector/vmstat_linux.go b/collector/vmstat_linux.go index ac08e8bb..c9d7ed41 100644 --- a/collector/vmstat_linux.go +++ b/collector/vmstat_linux.go @@ -24,7 +24,6 @@ import ( "strconv" "strings" - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" ) @@ -33,22 +32,25 @@ const ( vmStatSubsystem = "vmstat" ) -var ( - vmStatFields = kingpin.Flag("collector.vmstat.fields", "Regexp of fields to return for vmstat collector.").Default("^(oom_kill|pgpg|pswp|pg.*fault).*").String() -) - type vmStatCollector struct { fieldPattern *regexp.Regexp logger log.Logger } func init() { - registerCollector("vmstat", defaultEnabled, NewvmStatCollector) + registerCollector(vmStatSubsystem, defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(VmStatConfig) + return NewvmStatCollector(cfg, logger) + }) +} + +type VmStatConfig struct { + Fields *string } // NewvmStatCollector returns a new Collector exposing vmstat stats. -func NewvmStatCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { - pattern := regexp.MustCompile(*vmStatFields) +func NewvmStatCollector(config VmStatConfig, logger log.Logger) (Collector, error) { + pattern := regexp.MustCompile(*config.Fields) return &vmStatCollector{ fieldPattern: pattern, logger: logger, diff --git a/collector/wifi_linux.go b/collector/wifi_linux.go index 782a25eb..5c056618 100644 --- a/collector/wifi_linux.go +++ b/collector/wifi_linux.go @@ -23,7 +23,6 @@ import ( "os" "path/filepath" - "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/mdlayher/wifi" @@ -46,14 +45,18 @@ type wifiCollector struct { stationBeaconLossTotal *prometheus.Desc logger log.Logger + config WifiConfig } -var ( - collectorWifi = kingpin.Flag("collector.wifi.fixtures", "test fixtures to use for wifi collector metrics").Default("").String() -) - func init() { - registerCollector("wifi", defaultDisabled, NewWifiCollector) + registerCollector("wifi", defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + cfg := config.(WifiConfig) + return NewWifiCollector(cfg, logger) + }) +} + +type WifiConfig struct { + Fixtures *string } var _ wifiStater = &wifi.Client{} @@ -67,7 +70,7 @@ type wifiStater interface { } // NewWifiCollector returns a new Collector exposing Wifi statistics. -func NewWifiCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewWifiCollector(config WifiConfig, logger log.Logger) (Collector, error) { const ( subsystem = "wifi" ) @@ -161,11 +164,12 @@ func NewWifiCollector(config NodeCollectorConfig, logger log.Logger) (Collector, nil, ), logger: logger, + config: config, }, nil } func (c *wifiCollector) Update(ch chan<- prometheus.Metric) error { - stat, err := newWifiStater(*collectorWifi) + stat, err := newWifiStater(*c.config.Fixtures) if err != nil { // Cannot access wifi metrics, report no error. if errors.Is(err, os.ErrNotExist) { diff --git a/collector/xfs_linux.go b/collector/xfs_linux.go index e24b1da4..ed3dea98 100644 --- a/collector/xfs_linux.go +++ b/collector/xfs_linux.go @@ -31,11 +31,13 @@ type xfsCollector struct { } func init() { - registerCollector("xfs", defaultEnabled, NewXFSCollector) + registerCollector("xfs", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewXFSCollector(logger) + }) } // NewXFSCollector returns a new Collector exposing XFS statistics. -func NewXFSCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewXFSCollector(logger log.Logger) (Collector, error) { fs, err := xfs.NewFS(*procPath, *sysPath) if err != nil { return nil, fmt.Errorf("failed to open sysfs: %w", err) diff --git a/collector/zfs.go b/collector/zfs.go index f26f9921..03bffd11 100644 --- a/collector/zfs.go +++ b/collector/zfs.go @@ -30,7 +30,9 @@ var errZFSNotAvailable = errors.New("ZFS / ZFS statistics are not available") type zfsSysctl string func init() { - registerCollector("zfs", defaultEnabled, NewZFSCollector) + registerCollector("zfs", defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewZFSCollector(logger) + }) } type zfsCollector struct { @@ -43,7 +45,7 @@ type zfsCollector struct { } // NewZFSCollector returns a new Collector exposing ZFS statistics. -func NewZFSCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewZFSCollector(logger log.Logger) (Collector, error) { return &zfsCollector{ linuxProcpathBase: "spl/kstat/zfs", linuxZpoolIoPath: "/*/io", diff --git a/collector/zfs_freebsd.go b/collector/zfs_freebsd.go index e02061cd..77760027 100644 --- a/collector/zfs_freebsd.go +++ b/collector/zfs_freebsd.go @@ -33,10 +33,12 @@ const ( ) func init() { - registerCollector("zfs", defaultEnabled, NewZfsCollector) + registerCollector(zfsCollectorSubsystem, defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewZfsCollector(logger) + }) } -func NewZfsCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewZfsCollector(logger log.Logger) (Collector, error) { return &zfsCollector{ sysctls: []bsdSysctl{ { diff --git a/collector/zfs_solaris.go b/collector/zfs_solaris.go index 3cc3fbb9..d7c2e640 100644 --- a/collector/zfs_solaris.go +++ b/collector/zfs_solaris.go @@ -62,7 +62,9 @@ const ( ) func init() { - registerCollector("zfs", defaultEnabled, NewZfsCollector) + registerCollector(zfsCollectorSubsystem, defaultEnabled, func(config any, logger log.Logger) (Collector, error) { + return NewZfsCollector(logger) + }) } func NewZfsCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { diff --git a/collector/zoneinfo_linux.go b/collector/zoneinfo_linux.go index 9ea23558..4e9045aa 100644 --- a/collector/zoneinfo_linux.go +++ b/collector/zoneinfo_linux.go @@ -33,11 +33,13 @@ type zoneinfoCollector struct { } func init() { - registerCollector("zoneinfo", defaultDisabled, NewZoneinfoCollector) + registerCollector(zoneinfoSubsystem, defaultDisabled, func(config any, logger log.Logger) (Collector, error) { + return NewZoneinfoCollector(logger) + }) } // NewZoneinfoCollector returns a new Collector exposing zone stats. -func NewZoneinfoCollector(config NodeCollectorConfig, logger log.Logger) (Collector, error) { +func NewZoneinfoCollector(logger log.Logger) (Collector, error) { fs, err := procfs.NewFS(*procPath) if err != nil { return nil, fmt.Errorf("failed to open procfs: %w", err) diff --git a/kingpinconfig/flags.go b/kingpinconfig/flags.go index c6302963..3585c893 100644 --- a/kingpinconfig/flags.go +++ b/kingpinconfig/flags.go @@ -118,5 +118,43 @@ func AddFlags(a *kingpin.Application) collector.NodeCollectorConfig { config.Perf.CaProfilerFlag = a.Flag("collector.perf.cache-profilers", "perf cache profilers that should be collected").Strings() config.PowerSupplyClass.IgnoredPowerSupplies = a.Flag("collector.powersupply.ignored-supplies", "Regexp of power supplies to ignore for powersupplyclass collector.").Default("^$").String() + + config.Qdisc.Fixtures = a.Flag("collector.qdisc.fixtures", "test fixtures to use for qdisc collector end-to-end testing").Default("").String() + config.Qdisc.DeviceInclude = a.Flag("collector.qdisk.device-include", "Regexp of qdisk devices to include (mutually exclusive to device-exclude).").String() + config.Qdisc.DeviceExclude = a.Flag("collector.qdisk.device-exclude", "Regexp of qdisk devices to exclude (mutually exclusive to device-include).").String() + + config.Rapl.ZoneLabel = a.Flag("collector.rapl.enable-zone-label", "Enables service unit metric unit_start_time_seconds").Bool() + + config.Runit.ServiceDir = a.Flag("collector.runit.servicedir", "Path to runit service directory.").Default("/etc/service").String() + + config.Stat.Softirq = a.Flag("collector.stat.softirq", "Export softirq calls per vector").Default("false").Bool() + + config.Supervisord.URL = a.Flag("collector.supervisord.url", "XML RPC endpoint.").Default("http://localhost:9001/RPC2").Envar("SUPERVISORD_URL").String() + + config.Sysctl.Include = a.Flag("collector.sysctl.include", "Select sysctl metrics to include").Strings() + config.Sysctl.IncludeInfo = a.Flag("collector.sysctl.include-info", "Select sysctl metrics to include as info metrics").Strings() + + config.Systemd.UnitInclude = a.Flag("collector.systemd.unit-include", "Regexp of systemd units to include. Units must both match include and not match exclude to be included.").Default(".+").PreAction(func(c *kingpin.ParseContext) error { + systemdUnitIncludeSet = true + return nil + }).String() + config.Systemd.UnitExclude = kingpin.Flag("collector.systemd.unit-exclude", "Regexp of systemd units to exclude. Units must both match include and not match exclude to be included.").Default(".+\\.(automount|device|mount|scope|slice)").PreAction(func(c *kingpin.ParseContext) error { + systemdUnitExcludeSet = true + return nil + }).String() + config.Systemd.OldUnitInclude = kingpin.Flag("collector.systemd.unit-whitelist", "DEPRECATED: Use --collector.systemd.unit-include").Hidden().String() + config.Systemd.OldUnitExclude = kingpin.Flag("collector.systemd.unit-blacklist", "DEPRECATED: Use collector.systemd.unit-exclude").Hidden().String() + config.Systemd.Private = kingpin.Flag("collector.systemd.private", "Establish a private, direct connection to systemd without dbus (Strongly discouraged since it requires root. For testing purposes only).").Hidden().Bool() + config.Systemd.EnableTaskMetrics = kingpin.Flag("collector.systemd.enable-task-metrics", "Enables service unit tasks metrics unit_tasks_current and unit_tasks_max").Bool() + config.Systemd.EnableRestartsMetrics = kingpin.Flag("collector.systemd.enable-restarts-metrics", "Enables service unit metric service_restart_total").Bool() + config.Systemd.EnableStartTimeMetrics = kingpin.Flag("collector.systemd.enable-start-time-metrics", "Enables service unit metric unit_start_time_seconds").Bool() + + config.Tapestats.IgnoredDevices = kingpin.Flag("collector.tapestats.ignored-devices", "Regexp of devices to ignore for tapestats.").Default("^$").String() + + config.TextFile.Directory = a.Flag("collector.textfile.directory", "Directory to read text files with metrics from.").Default("").String() + + config.VmStat.Fields = a.Flag("collector.vmstat.fields", "Regexp of fields to return for vmstat collector.").Default("^(oom_kill|pgpg|pswp|pg.*fault).*").String() + + config.Wifi.Fixtures = a.Flag("collector.wifi.fixtures", "test fixtures to use for wifi collector end-to-end testing").Default("").String() return config }