Add sysctl collector

Signed-off-by: Johannes Ziemke <github@5pi.de>
This commit is contained in:
Johannes 'fish' Ziemke 2022-07-14 18:51:55 +02:00 committed by Johannes 'fish' Ziemke
parent 545d70ceb0
commit d962e48ca2
9 changed files with 294 additions and 10 deletions

View file

@ -5,6 +5,7 @@
* [ENHANCEMENT]
* [BUGFIX]
* [FEATURE] Add sysctl collector #2425
* [ENHANCEMENT] Add node_softirqs_total metric #2221
* [ENHANCEMENT] Add device filter flags to arp collector #2254
* [ENHANCEMENT] Add rapl zone name label option #2401

View file

@ -124,6 +124,7 @@ selinux | Exposes SELinux statistics. | Linux
sockstat | Exposes various statistics from `/proc/net/sockstat`. | Linux
softnet | Exposes statistics from `/proc/net/softnet_stat`. | Linux
stat | Exposes various statistics from `/proc/stat`. This includes boot time, forks and interrupts. | Linux
sysctl | Expose sysctl values from `/proc/sys`. Use `--collector.sysctl.include(-info)` to configure. | Linux
tapestats | Exposes statistics from `/sys/class/scsi_tape`. | Linux
textfile | Exposes statistics read from local disk. The `--collector.textfile.directory` flag must be set. | _any_
thermal | Exposes thermal statistics like `pmset -g therm`. | Darwin

View file

@ -3052,6 +3052,7 @@ node_scrape_collector_success{collector="slabinfo"} 1
node_scrape_collector_success{collector="sockstat"} 1
node_scrape_collector_success{collector="softnet"} 1
node_scrape_collector_success{collector="stat"} 1
node_scrape_collector_success{collector="sysctl"} 1
node_scrape_collector_success{collector="tapestats"} 1
node_scrape_collector_success{collector="textfile"} 1
node_scrape_collector_success{collector="thermal_zone"} 1
@ -3185,6 +3186,33 @@ node_softnet_times_squeezed_total{cpu="0"} 1
node_softnet_times_squeezed_total{cpu="1"} 10
node_softnet_times_squeezed_total{cpu="2"} 85
node_softnet_times_squeezed_total{cpu="3"} 50
# HELP node_sysctl_fs_file_nr sysctl fs.file-nr
# TYPE node_sysctl_fs_file_nr untyped
node_sysctl_fs_file_nr{index="0"} 1024
node_sysctl_fs_file_nr{index="1"} 0
node_sysctl_fs_file_nr{index="2"} 1.631329e+06
# HELP node_sysctl_fs_file_nr_current sysctl fs.file-nr, field 1
# TYPE node_sysctl_fs_file_nr_current untyped
node_sysctl_fs_file_nr_current 0
# HELP node_sysctl_fs_file_nr_max sysctl fs.file-nr, field 2
# TYPE node_sysctl_fs_file_nr_max untyped
node_sysctl_fs_file_nr_max 1.631329e+06
# HELP node_sysctl_fs_file_nr_total sysctl fs.file-nr, field 0
# TYPE node_sysctl_fs_file_nr_total untyped
node_sysctl_fs_file_nr_total 1024
# HELP node_sysctl_info sysctl info
# TYPE node_sysctl_info gauge
node_sysctl_info{index="0",name="kernel.seccomp.actions_avail",value="kill_process"} 1
node_sysctl_info{index="1",name="kernel.seccomp.actions_avail",value="kill_thread"} 1
node_sysctl_info{index="2",name="kernel.seccomp.actions_avail",value="trap"} 1
node_sysctl_info{index="3",name="kernel.seccomp.actions_avail",value="errno"} 1
node_sysctl_info{index="4",name="kernel.seccomp.actions_avail",value="user_notif"} 1
node_sysctl_info{index="5",name="kernel.seccomp.actions_avail",value="trace"} 1
node_sysctl_info{index="6",name="kernel.seccomp.actions_avail",value="log"} 1
node_sysctl_info{index="7",name="kernel.seccomp.actions_avail",value="allow"} 1
# HELP node_sysctl_kernel_threads_max sysctl kernel.threads-max
# TYPE node_sysctl_kernel_threads_max untyped
node_sysctl_kernel_threads_max 7801
# HELP node_tape_io_now The number of I/Os currently outstanding to this device.
# TYPE node_tape_io_now gauge
node_tape_io_now{device="st0"} 1

View file

@ -3074,6 +3074,7 @@ node_scrape_collector_success{collector="slabinfo"} 1
node_scrape_collector_success{collector="sockstat"} 1
node_scrape_collector_success{collector="softnet"} 1
node_scrape_collector_success{collector="stat"} 1
node_scrape_collector_success{collector="sysctl"} 1
node_scrape_collector_success{collector="tapestats"} 1
node_scrape_collector_success{collector="textfile"} 1
node_scrape_collector_success{collector="thermal_zone"} 1
@ -3207,6 +3208,33 @@ node_softnet_times_squeezed_total{cpu="0"} 1
node_softnet_times_squeezed_total{cpu="1"} 10
node_softnet_times_squeezed_total{cpu="2"} 85
node_softnet_times_squeezed_total{cpu="3"} 50
# HELP node_sysctl_fs_file_nr sysctl fs.file-nr
# TYPE node_sysctl_fs_file_nr untyped
node_sysctl_fs_file_nr{index="0"} 1024
node_sysctl_fs_file_nr{index="1"} 0
node_sysctl_fs_file_nr{index="2"} 1.631329e+06
# HELP node_sysctl_fs_file_nr_current sysctl fs.file-nr, field 1
# TYPE node_sysctl_fs_file_nr_current untyped
node_sysctl_fs_file_nr_current 0
# HELP node_sysctl_fs_file_nr_max sysctl fs.file-nr, field 2
# TYPE node_sysctl_fs_file_nr_max untyped
node_sysctl_fs_file_nr_max 1.631329e+06
# HELP node_sysctl_fs_file_nr_total sysctl fs.file-nr, field 0
# TYPE node_sysctl_fs_file_nr_total untyped
node_sysctl_fs_file_nr_total 1024
# HELP node_sysctl_info sysctl info
# TYPE node_sysctl_info gauge
node_sysctl_info{index="0",name="kernel.seccomp.actions_avail",value="kill_process"} 1
node_sysctl_info{index="1",name="kernel.seccomp.actions_avail",value="kill_thread"} 1
node_sysctl_info{index="2",name="kernel.seccomp.actions_avail",value="trap"} 1
node_sysctl_info{index="3",name="kernel.seccomp.actions_avail",value="errno"} 1
node_sysctl_info{index="4",name="kernel.seccomp.actions_avail",value="user_notif"} 1
node_sysctl_info{index="5",name="kernel.seccomp.actions_avail",value="trace"} 1
node_sysctl_info{index="6",name="kernel.seccomp.actions_avail",value="log"} 1
node_sysctl_info{index="7",name="kernel.seccomp.actions_avail",value="allow"} 1
# HELP node_sysctl_kernel_threads_max sysctl kernel.threads-max
# TYPE node_sysctl_kernel_threads_max untyped
node_sysctl_kernel_threads_max 7801
# HELP node_tape_io_now The number of I/Os currently outstanding to this device.
# TYPE node_tape_io_now gauge
node_tape_io_now{device="st0"} 1

View file

@ -0,0 +1 @@
kill_process kill_thread trap errno user_notif trace log allow

218
collector/sysctl_linux.go Normal file
View file

@ -0,0 +1,218 @@
// Copyright 2022 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package collector
import (
"fmt"
"strconv"
"strings"
"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/procfs"
"gopkg.in/alecthomas/kingpin.v2"
)
var (
sysctlInclude = kingpin.Flag("collector.sysctl.include", "Select sysctl metrics to include").Strings()
sysctlIncludeInfo = kingpin.Flag("collector.sysctl.include-info", "Select sysctl metrics to include as info metrics").Strings()
sysctlInfoDesc = prometheus.NewDesc(prometheus.BuildFQName(namespace, "sysctl", "info"), "sysctl info", []string{"name", "value", "index"}, nil)
)
type sysctlCollector struct {
fs procfs.FS
logger log.Logger
sysctls []*sysctl
}
func init() {
registerCollector("sysctl", defaultDisabled, NewSysctlCollector)
}
func NewSysctlCollector(logger log.Logger) (Collector, error) {
fs, err := procfs.NewFS(*procPath)
if err != nil {
return nil, fmt.Errorf("failed to open sysfs: %w", err)
}
c := &sysctlCollector{
logger: logger,
fs: fs,
sysctls: []*sysctl{},
}
for _, include := range *sysctlInclude {
sysctl, err := newSysctl(include, true)
if err != nil {
return nil, err
}
c.sysctls = append(c.sysctls, sysctl)
}
for _, include := range *sysctlIncludeInfo {
sysctl, err := newSysctl(include, false)
if err != nil {
return nil, err
}
c.sysctls = append(c.sysctls, sysctl)
}
return c, nil
}
func (c *sysctlCollector) Update(ch chan<- prometheus.Metric) error {
for _, sysctl := range c.sysctls {
metrics, err := c.newMetrics(sysctl)
if err != nil {
return err
}
for _, metric := range metrics {
ch <- metric
}
}
return nil
}
func (c *sysctlCollector) newMetrics(s *sysctl) ([]prometheus.Metric, error) {
var (
values interface{}
length int
err error
)
if s.numeric {
values, err = c.fs.SysctlInts(s.name)
if err != nil {
return nil, fmt.Errorf("error obtaining sysctl info: %w", err)
}
length = len(values.([]int))
} else {
values, err = c.fs.SysctlStrings(s.name)
if err != nil {
return nil, fmt.Errorf("error obtaining sysctl info: %w", err)
}
length = len(values.([]string))
}
switch length {
case 0:
return nil, fmt.Errorf("sysctl %s has no values", s.name)
case 1:
if len(s.keys) > 0 {
return nil, fmt.Errorf("sysctl %s has only one value, but expected %v", s.name, s.keys)
}
return []prometheus.Metric{s.newConstMetric(values)}, nil
default:
if len(s.keys) == 0 {
return s.newIndexedMetrics(values), nil
}
if length != len(s.keys) {
return nil, fmt.Errorf("sysctl %s has %d keys but only %d defined in f lag", s.name, length, len(s.keys))
}
return s.newMappedMetrics(values)
}
}
type sysctl struct {
numeric bool
name string
keys []string
}
func newSysctl(include string, numeric bool) (*sysctl, error) {
parts := strings.SplitN(include, ":", 2)
s := &sysctl{
numeric: numeric,
name: parts[0],
}
if len(parts) == 2 {
s.keys = strings.Split(parts[1], ",")
s.name = parts[0]
}
return s, nil
}
func (s *sysctl) metricName() string {
return SanitizeMetricName(s.name)
}
func (s *sysctl) newConstMetric(v interface{}) prometheus.Metric {
if s.numeric {
return prometheus.MustNewConstMetric(
prometheus.NewDesc(
prometheus.BuildFQName(namespace, "sysctl", s.metricName()),
fmt.Sprintf("sysctl %s", s.name),
nil, nil),
prometheus.UntypedValue,
float64(v.([]int)[0]))
}
return prometheus.MustNewConstMetric(
sysctlInfoDesc,
prometheus.GaugeValue,
1.0,
s.name,
v.([]string)[0],
"0",
)
}
func (s *sysctl) newIndexedMetrics(v interface{}) []prometheus.Metric {
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, "sysctl", s.metricName()),
fmt.Sprintf("sysctl %s", s.name),
[]string{"index"}, nil,
)
switch values := v.(type) {
case []int:
metrics := make([]prometheus.Metric, len(values))
for i, n := range values {
metrics[i] = prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(n), strconv.Itoa(i))
}
return metrics
case []string:
metrics := make([]prometheus.Metric, len(values))
for i, str := range values {
metrics[i] = prometheus.MustNewConstMetric(sysctlInfoDesc, prometheus.GaugeValue, 1.0, s.name, str, strconv.Itoa(i))
}
return metrics
default:
panic(fmt.Sprintf("unexpected type %T", values))
}
}
func (s *sysctl) newMappedMetrics(v interface{}) ([]prometheus.Metric, error) {
switch values := v.(type) {
case []int:
metrics := make([]prometheus.Metric, len(values))
for i, n := range values {
key := s.keys[i]
desc := prometheus.NewDesc(
prometheus.BuildFQName(namespace, "sysctl", s.metricName()+"_"+key),
fmt.Sprintf("sysctl %s, field %d", s.name, i),
nil,
nil,
)
metrics[i] = prometheus.MustNewConstMetric(desc, prometheus.UntypedValue, float64(n))
}
return metrics, nil
case []string:
return nil, fmt.Errorf("mapped sysctl string values not supported")
default:
return nil, fmt.Errorf("unexpected type %T", values)
}
}

View file

@ -43,6 +43,7 @@ enabled_collectors=$(cat << COLLECTORS
slabinfo
sockstat
stat
sysctl
textfile
thermal_zone
udp_queues
@ -135,6 +136,10 @@ fi
--collector.cpu.info.bugs-include="${cpu_info_bugs}" \
--collector.cpu.info.flags-include="${cpu_info_flags}" \
--collector.stat.softirq \
--collector.sysctl.include="kernel.threads-max" \
--collector.sysctl.include="fs.file-nr" \
--collector.sysctl.include="fs.file-nr:total,current,max" \
--collector.sysctl.include-info="kernel.seccomp.actions_avail" \
--web.listen-address "127.0.0.1:${port}" \
--log.level="debug" > "${tmpdir}/node_exporter.log" 2>&1 &

8
go.mod
View file

@ -21,10 +21,10 @@ require (
github.com/prometheus/client_model v0.2.0
github.com/prometheus/common v0.35.0
github.com/prometheus/exporter-toolkit v0.7.1
github.com/prometheus/procfs v0.7.4-0.20211209105546-0f8a320e1e1f
github.com/prometheus/procfs v0.7.4-0.20220719120938-9f9c419a6490
github.com/safchain/ethtool v0.2.0
github.com/soundcloud/go-runit v0.0.0-20150630195641-06ad41a06c4a
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
gopkg.in/alecthomas/kingpin.v2 v2.2.6
)
@ -35,7 +35,7 @@ require (
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/go-cmp v0.5.8 // indirect
github.com/josharian/native v1.0.0 // indirect
github.com/jpillora/backoff v1.0.0 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
@ -49,7 +49,7 @@ require (
golang.org/x/crypto v0.0.0-20220214200702-86341886e292 // indirect
golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
golang.org/x/text v0.3.7 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/protobuf v1.26.0 // indirect

14
go.sum
View file

@ -129,8 +129,9 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@ -239,8 +240,8 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.7.4-0.20211209105546-0f8a320e1e1f h1:tg+5gOV1AkhTfWNi94UA5rvn+Mat/thyB2WASYAzPZo=
github.com/prometheus/procfs v0.7.4-0.20211209105546-0f8a320e1e1f/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.7.4-0.20220719120938-9f9c419a6490 h1:r8tvHkY1WUgbABOocq3OtGE3rFK8pdSA30NZNSxpBIU=
github.com/prometheus/procfs v0.7.4-0.20220719120938-9f9c419a6490/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
@ -363,8 +364,9 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@ -414,8 +416,9 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220307203707-22a9840ba4d7/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f h1:8w7RhxzTVgUzw/AH/9mUV5q0vMgy40SQRursCcfmkCw=
golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@ -472,7 +475,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=