mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-24 21:24:05 -08:00
Add PuppetDB service discovery
We have been Puppet user for 10 years and we are users of https://github.com/camptocamp/prometheus-puppetdb-sd However, that file_sd implementation contains business logic and assumptions around e.g. the modules which you are using. This pull request adds a simple PuppetDB service discovery, which will enable more use cases than the upstream sd. Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>
This commit is contained in:
parent
447d9401fc
commit
8920024323
|
@ -45,6 +45,7 @@ import (
|
||||||
"github.com/prometheus/prometheus/discovery/marathon"
|
"github.com/prometheus/prometheus/discovery/marathon"
|
||||||
"github.com/prometheus/prometheus/discovery/moby"
|
"github.com/prometheus/prometheus/discovery/moby"
|
||||||
"github.com/prometheus/prometheus/discovery/openstack"
|
"github.com/prometheus/prometheus/discovery/openstack"
|
||||||
|
"github.com/prometheus/prometheus/discovery/puppetdb"
|
||||||
"github.com/prometheus/prometheus/discovery/scaleway"
|
"github.com/prometheus/prometheus/discovery/scaleway"
|
||||||
"github.com/prometheus/prometheus/discovery/targetgroup"
|
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||||
"github.com/prometheus/prometheus/discovery/triton"
|
"github.com/prometheus/prometheus/discovery/triton"
|
||||||
|
@ -790,6 +791,34 @@ var expectedConf = &Config{
|
||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
JobName: "service-puppetdb",
|
||||||
|
|
||||||
|
HonorTimestamps: true,
|
||||||
|
ScrapeInterval: model.Duration(15 * time.Second),
|
||||||
|
ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout,
|
||||||
|
|
||||||
|
MetricsPath: DefaultScrapeConfig.MetricsPath,
|
||||||
|
Scheme: DefaultScrapeConfig.Scheme,
|
||||||
|
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||||
|
|
||||||
|
ServiceDiscoveryConfigs: discovery.Configs{&puppetdb.SDConfig{
|
||||||
|
URL: "https://puppetserver/",
|
||||||
|
Query: "resources { type = \"Package\" and title = \"httpd\" }",
|
||||||
|
IncludeParameters: true,
|
||||||
|
Port: 80,
|
||||||
|
RefreshInterval: model.Duration(60 * time.Second),
|
||||||
|
HTTPClientConfig: config.HTTPClientConfig{
|
||||||
|
FollowRedirects: true,
|
||||||
|
TLSConfig: config.TLSConfig{
|
||||||
|
CAFile: "testdata/valid_ca_file",
|
||||||
|
CertFile: "testdata/valid_cert_file",
|
||||||
|
KeyFile: "testdata/valid_key_file",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
{
|
{
|
||||||
JobName: "hetzner",
|
JobName: "hetzner",
|
||||||
HonorTimestamps: true,
|
HonorTimestamps: true,
|
||||||
|
@ -1262,6 +1291,22 @@ var expectedErrors = []struct {
|
||||||
filename: "empty_static_config.bad.yml",
|
filename: "empty_static_config.bad.yml",
|
||||||
errMsg: "empty or null section in static_configs",
|
errMsg: "empty or null section in static_configs",
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
filename: "puppetdb_no_query.bad.yml",
|
||||||
|
errMsg: "query missing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
filename: "puppetdb_no_url.bad.yml",
|
||||||
|
errMsg: "url missing",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
filename: "puppetdb_bad_url.bad.yml",
|
||||||
|
errMsg: "host is missing in URL",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
filename: "puppetdb_no_scheme.bad.yml",
|
||||||
|
errMsg: "url scheme must be http or https",
|
||||||
|
},
|
||||||
{
|
{
|
||||||
filename: "hetzner_role.bad.yml",
|
filename: "hetzner_role.bad.yml",
|
||||||
errMsg: "unknown role",
|
errMsg: "unknown role",
|
||||||
|
|
12
config/testdata/conf.good.yml
vendored
12
config/testdata/conf.good.yml
vendored
|
@ -307,6 +307,18 @@ scrape_configs:
|
||||||
cert_file: valid_cert_file
|
cert_file: valid_cert_file
|
||||||
key_file: valid_key_file
|
key_file: valid_key_file
|
||||||
|
|
||||||
|
- job_name: service-puppetdb
|
||||||
|
puppetdb_sd_configs:
|
||||||
|
- url: https://puppetserver/
|
||||||
|
query: 'resources { type = "Package" and title = "httpd" }'
|
||||||
|
include_parameters: true
|
||||||
|
port: 80
|
||||||
|
refresh_interval: 1m
|
||||||
|
tls_config:
|
||||||
|
ca_file: valid_ca_file
|
||||||
|
cert_file: valid_cert_file
|
||||||
|
key_file: valid_key_file
|
||||||
|
|
||||||
- job_name: hetzner
|
- job_name: hetzner
|
||||||
hetzner_sd_configs:
|
hetzner_sd_configs:
|
||||||
- role: hcloud
|
- role: hcloud
|
||||||
|
|
4
config/testdata/puppetdb_bad_url.bad.yml
vendored
Normal file
4
config/testdata/puppetdb_bad_url.bad.yml
vendored
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
scrape_configs:
|
||||||
|
- puppetdb_sd_configs:
|
||||||
|
- url: http://
|
||||||
|
query: 'resources { type = "Package" and title = "httpd" }'
|
3
config/testdata/puppetdb_no_query.bad.yml
vendored
Normal file
3
config/testdata/puppetdb_no_query.bad.yml
vendored
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
scrape_configs:
|
||||||
|
- puppetdb_sd_configs:
|
||||||
|
- url: http://puppetserver/
|
4
config/testdata/puppetdb_no_scheme.bad.yml
vendored
Normal file
4
config/testdata/puppetdb_no_scheme.bad.yml
vendored
Normal file
|
@ -0,0 +1,4 @@
|
||||||
|
scrape_configs:
|
||||||
|
- puppetdb_sd_configs:
|
||||||
|
- url: ftp://puppet
|
||||||
|
query: 'resources { type = "Package" and title = "httpd" }'
|
3
config/testdata/puppetdb_no_url.bad.yml
vendored
Normal file
3
config/testdata/puppetdb_no_url.bad.yml
vendored
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
scrape_configs:
|
||||||
|
- puppetdb_sd_configs:
|
||||||
|
- query: 'resources { type = "Package" and title = "httpd" }'
|
|
@ -31,6 +31,7 @@ import (
|
||||||
_ "github.com/prometheus/prometheus/discovery/marathon" // register marathon
|
_ "github.com/prometheus/prometheus/discovery/marathon" // register marathon
|
||||||
_ "github.com/prometheus/prometheus/discovery/moby" // register moby
|
_ "github.com/prometheus/prometheus/discovery/moby" // register moby
|
||||||
_ "github.com/prometheus/prometheus/discovery/openstack" // register openstack
|
_ "github.com/prometheus/prometheus/discovery/openstack" // register openstack
|
||||||
|
_ "github.com/prometheus/prometheus/discovery/puppetdb" // register puppetdb
|
||||||
_ "github.com/prometheus/prometheus/discovery/scaleway" // register scaleway
|
_ "github.com/prometheus/prometheus/discovery/scaleway" // register scaleway
|
||||||
_ "github.com/prometheus/prometheus/discovery/triton" // register triton
|
_ "github.com/prometheus/prometheus/discovery/triton" // register triton
|
||||||
_ "github.com/prometheus/prometheus/discovery/xds" // register xds
|
_ "github.com/prometheus/prometheus/discovery/xds" // register xds
|
||||||
|
|
49
discovery/puppetdb/fixtures/vhosts.json
Normal file
49
discovery/puppetdb/fixtures/vhosts.json
Normal file
|
@ -0,0 +1,49 @@
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"certname": "edinburgh.example.com",
|
||||||
|
"environment": "prod",
|
||||||
|
"exported": false,
|
||||||
|
"file": "/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp",
|
||||||
|
"line": 384,
|
||||||
|
"parameters": {
|
||||||
|
"access_log": true,
|
||||||
|
"access_log_file": "ssl_access_log",
|
||||||
|
"additional_includes": [ ],
|
||||||
|
"directoryindex": "",
|
||||||
|
"docroot": "/var/www/html",
|
||||||
|
"ensure": "absent",
|
||||||
|
"options": [
|
||||||
|
"Indexes",
|
||||||
|
"FollowSymLinks",
|
||||||
|
"MultiViews"
|
||||||
|
],
|
||||||
|
"php_flags": { },
|
||||||
|
"labels": {
|
||||||
|
"alias": "edinburgh"
|
||||||
|
},
|
||||||
|
"scriptaliases": [
|
||||||
|
{
|
||||||
|
"alias": "/cgi-bin",
|
||||||
|
"path": "/var/www/cgi-bin"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"resource": "49af83866dc5a1518968b68e58a25319107afe11",
|
||||||
|
"tags": [
|
||||||
|
"roles::hypervisor",
|
||||||
|
"apache",
|
||||||
|
"apache::vhost",
|
||||||
|
"class",
|
||||||
|
"default-ssl",
|
||||||
|
"profile_hypervisor",
|
||||||
|
"vhost",
|
||||||
|
"profile_apache",
|
||||||
|
"hypervisor",
|
||||||
|
"__node_regexp__edinburgh",
|
||||||
|
"roles",
|
||||||
|
"node"
|
||||||
|
],
|
||||||
|
"title": "default-ssl",
|
||||||
|
"type": "Apache::Vhost"
|
||||||
|
}
|
||||||
|
]
|
252
discovery/puppetdb/puppetdb.go
Normal file
252
discovery/puppetdb/puppetdb.go
Normal file
|
@ -0,0 +1,252 @@
|
||||||
|
// Copyright 2021 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package puppetdb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"io/ioutil"
|
||||||
|
"net"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"path"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-kit/log"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/prometheus/common/config"
|
||||||
|
"github.com/prometheus/common/model"
|
||||||
|
"github.com/prometheus/common/version"
|
||||||
|
|
||||||
|
"github.com/prometheus/prometheus/discovery"
|
||||||
|
"github.com/prometheus/prometheus/discovery/refresh"
|
||||||
|
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
pdbLabel = model.MetaLabelPrefix + "puppetdb_"
|
||||||
|
pdbLabelCertname = pdbLabel + "certname"
|
||||||
|
pdbLabelResource = pdbLabel + "resource"
|
||||||
|
pdbLabelType = pdbLabel + "type"
|
||||||
|
pdbLabelTitle = pdbLabel + "title"
|
||||||
|
pdbLabelExported = pdbLabel + "exported"
|
||||||
|
pdbLabelTags = pdbLabel + "tags"
|
||||||
|
pdbLabelFile = pdbLabel + "file"
|
||||||
|
pdbLabelEnvironment = pdbLabel + "environment"
|
||||||
|
pdbLabelParameter = pdbLabel + "parameter_"
|
||||||
|
separator = ","
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
// DefaultSDConfig is the default PuppetDB SD configuration.
|
||||||
|
DefaultSDConfig = SDConfig{
|
||||||
|
RefreshInterval: model.Duration(60 * time.Second),
|
||||||
|
Port: 80,
|
||||||
|
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||||
|
}
|
||||||
|
matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`)
|
||||||
|
userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
|
||||||
|
)
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
discovery.RegisterConfig(&SDConfig{})
|
||||||
|
}
|
||||||
|
|
||||||
|
// SDConfig is the configuration for PuppetDB based discovery.
|
||||||
|
type SDConfig struct {
|
||||||
|
HTTPClientConfig config.HTTPClientConfig `yaml:",inline"`
|
||||||
|
RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
|
||||||
|
URL string `yaml:"url"`
|
||||||
|
Query string `yaml:"query"`
|
||||||
|
IncludeParameters bool `yaml:"include_parameters"`
|
||||||
|
Port int `yaml:"port"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Name returns the name of the Config.
|
||||||
|
func (*SDConfig) Name() string { return "puppetdb" }
|
||||||
|
|
||||||
|
// NewDiscoverer returns a Discoverer for the Config.
|
||||||
|
func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
|
||||||
|
return NewDiscovery(c, opts.Logger)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetDirectory joins any relative file paths with dir.
|
||||||
|
func (c *SDConfig) SetDirectory(dir string) {
|
||||||
|
c.HTTPClientConfig.SetDirectory(dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
// UnmarshalYAML implements the yaml.Unmarshaler interface.
|
||||||
|
func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
|
||||||
|
*c = DefaultSDConfig
|
||||||
|
type plain SDConfig
|
||||||
|
err := unmarshal((*plain)(c))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if c.URL == "" {
|
||||||
|
return fmt.Errorf("URL is missing")
|
||||||
|
}
|
||||||
|
parsedURL, err := url.Parse(c.URL)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
||||||
|
return fmt.Errorf("URL scheme must be 'http' or 'https'")
|
||||||
|
}
|
||||||
|
if parsedURL.Host == "" {
|
||||||
|
return fmt.Errorf("host is missing in URL")
|
||||||
|
}
|
||||||
|
if c.Query == "" {
|
||||||
|
return fmt.Errorf("query missing")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discovery provides service discovery functionality based
|
||||||
|
// on PuppetDB resources.
|
||||||
|
type Discovery struct {
|
||||||
|
*refresh.Discovery
|
||||||
|
url string
|
||||||
|
query string
|
||||||
|
port int
|
||||||
|
includeParameters bool
|
||||||
|
client *http.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewDiscovery returns a new PuppetDB discovery for the given config.
|
||||||
|
func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
|
||||||
|
if logger == nil {
|
||||||
|
logger = log.NewNopLogger()
|
||||||
|
}
|
||||||
|
|
||||||
|
client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "http", config.WithHTTP2Disabled())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
client.Timeout = time.Duration(conf.RefreshInterval)
|
||||||
|
|
||||||
|
u, err := url.Parse(conf.URL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
u.Path = path.Join(u.Path, "pdb/query/v4")
|
||||||
|
|
||||||
|
d := &Discovery{
|
||||||
|
url: u.String(),
|
||||||
|
port: conf.Port,
|
||||||
|
query: conf.Query,
|
||||||
|
includeParameters: conf.IncludeParameters,
|
||||||
|
client: client,
|
||||||
|
}
|
||||||
|
|
||||||
|
d.Discovery = refresh.NewDiscovery(
|
||||||
|
logger,
|
||||||
|
"http",
|
||||||
|
time.Duration(conf.RefreshInterval),
|
||||||
|
d.refresh,
|
||||||
|
)
|
||||||
|
return d, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
|
||||||
|
body := struct {
|
||||||
|
Query string `json:"query"`
|
||||||
|
}{d.query}
|
||||||
|
bodyBytes, err := json.Marshal(body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
req, err := http.NewRequest("POST", d.url, bytes.NewBuffer(bodyBytes))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
req.Header.Set("User-Agent", userAgent)
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
resp, err := d.client.Do(req.WithContext(ctx))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer func() {
|
||||||
|
io.Copy(ioutil.Discard, resp.Body)
|
||||||
|
resp.Body.Close()
|
||||||
|
}()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, errors.Errorf("server returned HTTP status %s", resp.Status)
|
||||||
|
}
|
||||||
|
|
||||||
|
if ct := resp.Header.Get("Content-Type"); !matchContentType.MatchString(ct) {
|
||||||
|
return nil, errors.Errorf("unsupported content type %s", resp.Header.Get("Content-Type"))
|
||||||
|
}
|
||||||
|
|
||||||
|
b, err := ioutil.ReadAll(resp.Body)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
var resources []Resource
|
||||||
|
|
||||||
|
if err := json.Unmarshal(b, &resources); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
tg := &targetgroup.Group{
|
||||||
|
// Use a pseudo-URL as source.
|
||||||
|
Source: d.url + "?query=" + d.query,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, resource := range resources {
|
||||||
|
labels := model.LabelSet{
|
||||||
|
pdbLabelCertname: model.LabelValue(resource.Certname),
|
||||||
|
pdbLabelResource: model.LabelValue(resource.Resource),
|
||||||
|
pdbLabelType: model.LabelValue(resource.Type),
|
||||||
|
pdbLabelTitle: model.LabelValue(resource.Title),
|
||||||
|
pdbLabelExported: model.LabelValue(fmt.Sprintf("%t", resource.Exported)),
|
||||||
|
pdbLabelFile: model.LabelValue(resource.File),
|
||||||
|
pdbLabelEnvironment: model.LabelValue(resource.Environment),
|
||||||
|
}
|
||||||
|
|
||||||
|
addr := net.JoinHostPort(resource.Certname, strconv.FormatUint(uint64(d.port), 10))
|
||||||
|
labels[model.AddressLabel] = model.LabelValue(addr)
|
||||||
|
|
||||||
|
if len(resource.Tags) > 0 {
|
||||||
|
// We surround the separated list with the separator as well. This way regular expressions
|
||||||
|
// in relabeling rules don't have to consider tag positions.
|
||||||
|
tags := separator + strings.Join(resource.Tags, separator) + separator
|
||||||
|
labels[pdbLabelTags] = model.LabelValue(tags)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parameters are not included by default. This should only be enabled
|
||||||
|
// on select resources as it might expose secrets on the Prometheus UI
|
||||||
|
// for certain resources.
|
||||||
|
if d.includeParameters {
|
||||||
|
for k, v := range resource.Parameters.toLabels() {
|
||||||
|
labels[pdbLabelParameter+k] = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
tg.Targets = append(tg.Targets, labels)
|
||||||
|
}
|
||||||
|
|
||||||
|
return []*targetgroup.Group{tg}, nil
|
||||||
|
}
|
195
discovery/puppetdb/puppetdb_test.go
Normal file
195
discovery/puppetdb/puppetdb_test.go
Normal file
|
@ -0,0 +1,195 @@
|
||||||
|
// Copyright 2021 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package puppetdb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/go-kit/log"
|
||||||
|
"github.com/prometheus/common/config"
|
||||||
|
"github.com/prometheus/common/model"
|
||||||
|
"github.com/prometheus/prometheus/discovery/targetgroup"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
func mockServer(t *testing.T) *httptest.Server {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var request struct {
|
||||||
|
Query string `json:"query"`
|
||||||
|
}
|
||||||
|
err := json.NewDecoder(r.Body).Decode(&request)
|
||||||
|
if err != nil {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
http.ServeFile(w, r, "fixtures/"+request.Query+".json")
|
||||||
|
}))
|
||||||
|
t.Cleanup(ts.Close)
|
||||||
|
return ts
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPuppetSlashInURL(t *testing.T) {
|
||||||
|
tests := map[string]string{
|
||||||
|
"https://puppetserver": "https://puppetserver/pdb/query/v4",
|
||||||
|
"https://puppetserver/": "https://puppetserver/pdb/query/v4",
|
||||||
|
"http://puppetserver:8080/": "http://puppetserver:8080/pdb/query/v4",
|
||||||
|
"http://puppetserver:8080": "http://puppetserver:8080/pdb/query/v4",
|
||||||
|
}
|
||||||
|
|
||||||
|
for serverURL, apiURL := range tests {
|
||||||
|
cfg := SDConfig{
|
||||||
|
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||||
|
URL: serverURL,
|
||||||
|
Query: "vhosts", // This is not a valid PuppetDB query, but it is used by the mock.
|
||||||
|
Port: 80,
|
||||||
|
RefreshInterval: model.Duration(30 * time.Second),
|
||||||
|
}
|
||||||
|
d, err := NewDiscovery(&cfg, log.NewNopLogger())
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.Equal(t, apiURL, d.url)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPuppetDBRefresh(t *testing.T) {
|
||||||
|
ts := mockServer(t)
|
||||||
|
|
||||||
|
cfg := SDConfig{
|
||||||
|
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||||
|
URL: ts.URL,
|
||||||
|
Query: "vhosts", // This is not a valid PuppetDB query, but it is used by the mock.
|
||||||
|
Port: 80,
|
||||||
|
RefreshInterval: model.Duration(30 * time.Second),
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewDiscovery(&cfg, log.NewNopLogger())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
tgs, err := d.refresh(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
expectedTargets := []*targetgroup.Group{
|
||||||
|
{
|
||||||
|
Targets: []model.LabelSet{
|
||||||
|
{
|
||||||
|
model.AddressLabel: model.LabelValue("edinburgh.example.com:80"),
|
||||||
|
model.LabelName("__meta_puppetdb_certname"): model.LabelValue("edinburgh.example.com"),
|
||||||
|
model.LabelName("__meta_puppetdb_environment"): model.LabelValue("prod"),
|
||||||
|
model.LabelName("__meta_puppetdb_exported"): model.LabelValue("false"),
|
||||||
|
model.LabelName("__meta_puppetdb_file"): model.LabelValue("/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp"),
|
||||||
|
model.LabelName("__meta_puppetdb_resource"): model.LabelValue("49af83866dc5a1518968b68e58a25319107afe11"),
|
||||||
|
model.LabelName("__meta_puppetdb_tags"): model.LabelValue(",roles::hypervisor,apache,apache::vhost,class,default-ssl,profile_hypervisor,vhost,profile_apache,hypervisor,__node_regexp__edinburgh,roles,node,"),
|
||||||
|
model.LabelName("__meta_puppetdb_title"): model.LabelValue("default-ssl"),
|
||||||
|
model.LabelName("__meta_puppetdb_type"): model.LabelValue("Apache::Vhost"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Source: ts.URL + "/pdb/query/v4?query=vhosts",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
require.Equal(t, tgs, expectedTargets)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPuppetDBRefreshWithParameters(t *testing.T) {
|
||||||
|
ts := mockServer(t)
|
||||||
|
|
||||||
|
cfg := SDConfig{
|
||||||
|
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||||
|
URL: ts.URL,
|
||||||
|
Query: "vhosts", // This is not a valid PuppetDB query, but it is used by the mock.
|
||||||
|
Port: 80,
|
||||||
|
IncludeParameters: true,
|
||||||
|
RefreshInterval: model.Duration(30 * time.Second),
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewDiscovery(&cfg, log.NewNopLogger())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
tgs, err := d.refresh(ctx)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
expectedTargets := []*targetgroup.Group{
|
||||||
|
{
|
||||||
|
Targets: []model.LabelSet{
|
||||||
|
{
|
||||||
|
model.AddressLabel: model.LabelValue("edinburgh.example.com:80"),
|
||||||
|
model.LabelName("__meta_puppetdb_certname"): model.LabelValue("edinburgh.example.com"),
|
||||||
|
model.LabelName("__meta_puppetdb_environment"): model.LabelValue("prod"),
|
||||||
|
model.LabelName("__meta_puppetdb_exported"): model.LabelValue("false"),
|
||||||
|
model.LabelName("__meta_puppetdb_file"): model.LabelValue("/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp"),
|
||||||
|
model.LabelName("__meta_puppetdb_parameter_access_log"): model.LabelValue("true"),
|
||||||
|
model.LabelName("__meta_puppetdb_parameter_access_log_file"): model.LabelValue("ssl_access_log"),
|
||||||
|
model.LabelName("__meta_puppetdb_parameter_docroot"): model.LabelValue("/var/www/html"),
|
||||||
|
model.LabelName("__meta_puppetdb_parameter_ensure"): model.LabelValue("absent"),
|
||||||
|
model.LabelName("__meta_puppetdb_parameter_labels_alias"): model.LabelValue("edinburgh"),
|
||||||
|
model.LabelName("__meta_puppetdb_parameter_options"): model.LabelValue("Indexes,FollowSymLinks,MultiViews"),
|
||||||
|
model.LabelName("__meta_puppetdb_resource"): model.LabelValue("49af83866dc5a1518968b68e58a25319107afe11"),
|
||||||
|
model.LabelName("__meta_puppetdb_tags"): model.LabelValue(",roles::hypervisor,apache,apache::vhost,class,default-ssl,profile_hypervisor,vhost,profile_apache,hypervisor,__node_regexp__edinburgh,roles,node,"),
|
||||||
|
model.LabelName("__meta_puppetdb_title"): model.LabelValue("default-ssl"),
|
||||||
|
model.LabelName("__meta_puppetdb_type"): model.LabelValue("Apache::Vhost"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Source: ts.URL + "/pdb/query/v4?query=vhosts",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
require.Equal(t, tgs, expectedTargets)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPuppetDBInvalidCode(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.WriteHeader(http.StatusBadRequest)
|
||||||
|
}))
|
||||||
|
|
||||||
|
t.Cleanup(ts.Close)
|
||||||
|
|
||||||
|
cfg := SDConfig{
|
||||||
|
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||||
|
URL: ts.URL,
|
||||||
|
RefreshInterval: model.Duration(30 * time.Second),
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewDiscovery(&cfg, log.NewNopLogger())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
_, err = d.refresh(ctx)
|
||||||
|
require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPuppetDBInvalidFormat(t *testing.T) {
|
||||||
|
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
fmt.Fprintln(w, "{}")
|
||||||
|
}))
|
||||||
|
|
||||||
|
t.Cleanup(ts.Close)
|
||||||
|
|
||||||
|
cfg := SDConfig{
|
||||||
|
HTTPClientConfig: config.DefaultHTTPClientConfig,
|
||||||
|
URL: ts.URL,
|
||||||
|
RefreshInterval: model.Duration(30 * time.Second),
|
||||||
|
}
|
||||||
|
|
||||||
|
d, err := NewDiscovery(&cfg, log.NewNopLogger())
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
ctx := context.Background()
|
||||||
|
_, err = d.refresh(ctx)
|
||||||
|
require.EqualError(t, err, "unsupported content type text/plain; charset=utf-8")
|
||||||
|
}
|
82
discovery/puppetdb/resources.go
Normal file
82
discovery/puppetdb/resources.go
Normal file
|
@ -0,0 +1,82 @@
|
||||||
|
// Copyright 2021 The Prometheus Authors
|
||||||
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
// you may not use this file except in compliance with the License.
|
||||||
|
// You may obtain a copy of the License at
|
||||||
|
//
|
||||||
|
// http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
//
|
||||||
|
// Unless required by applicable law or agreed to in writing, software
|
||||||
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
// See the License for the specific language governing permissions and
|
||||||
|
// limitations under the License.
|
||||||
|
|
||||||
|
package puppetdb
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/prometheus/common/model"
|
||||||
|
"github.com/prometheus/prometheus/util/strutil"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Resource struct {
|
||||||
|
Certname string `json:"certname"`
|
||||||
|
Resource string `json:"resource"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
Title string `json:"title"`
|
||||||
|
Exported bool `json:"exported"`
|
||||||
|
Tags []string `json:"tags"`
|
||||||
|
File string `json:"file"`
|
||||||
|
Environment string `json:"environment"`
|
||||||
|
Parameters Parameters `json:"parameters"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type Parameters map[string]interface{}
|
||||||
|
|
||||||
|
func (p *Parameters) toLabels() model.LabelSet {
|
||||||
|
labels := model.LabelSet{}
|
||||||
|
|
||||||
|
for k, v := range *p {
|
||||||
|
var labelValue string
|
||||||
|
switch value := v.(type) {
|
||||||
|
case string:
|
||||||
|
labelValue = value
|
||||||
|
case bool:
|
||||||
|
labelValue = strconv.FormatBool(value)
|
||||||
|
case []string:
|
||||||
|
labelValue = separator + strings.Join(value, separator) + separator
|
||||||
|
case []interface{}:
|
||||||
|
if len(value) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
values := make([]string, len(value))
|
||||||
|
for i, v := range value {
|
||||||
|
switch value := v.(type) {
|
||||||
|
case string:
|
||||||
|
values[i] = value
|
||||||
|
case bool:
|
||||||
|
values[i] = strconv.FormatBool(value)
|
||||||
|
case []string:
|
||||||
|
values[i] = separator + strings.Join(value, separator) + separator
|
||||||
|
}
|
||||||
|
}
|
||||||
|
labelValue = strings.Join(values, separator)
|
||||||
|
case map[string]interface{}:
|
||||||
|
subParameter := Parameters(value)
|
||||||
|
prefix := strutil.SanitizeLabelName(k + "_")
|
||||||
|
for subk, subv := range subParameter.toLabels() {
|
||||||
|
labels[model.LabelName(prefix)+subk] = subv
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if labelValue == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
name := strutil.SanitizeLabelName(k)
|
||||||
|
labels[model.LabelName(name)] = model.LabelValue(labelValue)
|
||||||
|
}
|
||||||
|
return labels
|
||||||
|
}
|
|
@ -272,6 +272,10 @@ nerve_sd_configs:
|
||||||
openstack_sd_configs:
|
openstack_sd_configs:
|
||||||
[ - <openstack_sd_config> ... ]
|
[ - <openstack_sd_config> ... ]
|
||||||
|
|
||||||
|
# List of PuppetDB service discovery configurations.
|
||||||
|
puppetdb_sd_configs:
|
||||||
|
[ - <puppetdb_sd_config> ... ]
|
||||||
|
|
||||||
# List of Scaleway service discovery configurations.
|
# List of Scaleway service discovery configurations.
|
||||||
scaleway_sd_configs:
|
scaleway_sd_configs:
|
||||||
[ - <scaleway_sd_config> ... ]
|
[ - <scaleway_sd_config> ... ]
|
||||||
|
@ -1069,6 +1073,94 @@ tls_config:
|
||||||
[ <tls_config> ]
|
[ <tls_config> ]
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### `<puppetdb_sd_config>`
|
||||||
|
|
||||||
|
PuppetDB SD configurations allow retrieving scrape targets from
|
||||||
|
[PuppetDB](https://puppet.com/docs/puppetdb/latest/index.html) resources.
|
||||||
|
|
||||||
|
This SD discovers resources and will create a target for each resource returned
|
||||||
|
by the API.
|
||||||
|
|
||||||
|
The resource address is the `certname` of the resource and can be changed during
|
||||||
|
[relabeling](#relabel_config).
|
||||||
|
|
||||||
|
The following meta labels are available on targets during [relabeling](#relabel_config):
|
||||||
|
|
||||||
|
* `__meta_puppetdb_certname`: the name of the node associated with the resource
|
||||||
|
* `__meta_puppetdb_resource`: a SHA-1 hash of the resource’s type, title, and parameters, for identification
|
||||||
|
* `__meta_puppetdb_type`: the resource type
|
||||||
|
* `__meta_puppetdb_title`: the resource title
|
||||||
|
* `__meta_puppetdb_exported`: whether the resource is exported (`"true"` or `"false"`)
|
||||||
|
* `__meta_puppetdb_tags`: comma separated list of resource tags
|
||||||
|
* `__meta_puppetdb_file`: the manifest file in which the resource was declared
|
||||||
|
* `__meta_puppetdb_environment`: the environment of the node associated with the resource
|
||||||
|
* `__meta_puppetdb_parameter_<parametername>`: the parameters of the resource
|
||||||
|
|
||||||
|
|
||||||
|
See below for the configuration options for PuppetDB discovery:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# The URL of the PuppetDB root query endpoint.
|
||||||
|
url: <string>
|
||||||
|
|
||||||
|
# Puppet Query Language (PQL) query. Only resources are supported.
|
||||||
|
# https://puppet.com/docs/puppetdb/latest/api/query/v4/pql.html
|
||||||
|
query: <string>
|
||||||
|
|
||||||
|
# Whether to include the parameters as meta labels.
|
||||||
|
# Due to the differences between parameter types and Prometheus labels,
|
||||||
|
# some parameters might not be rendered. The format of the parameters might
|
||||||
|
# also change in future releases.
|
||||||
|
#
|
||||||
|
# Note: Enabling this exposes parameters in the Prometheus UI and API. Make sure
|
||||||
|
# that you don't have secrets exposed as parameters if you enable this.
|
||||||
|
[ include_parameters: <boolean> | default = false ]
|
||||||
|
|
||||||
|
# Refresh interval to re-read the resources list.
|
||||||
|
[ refresh_interval: <duration> | default = 60s ]
|
||||||
|
|
||||||
|
# The port to scrape metrics from.
|
||||||
|
[ port: <int> | default = 80 ]
|
||||||
|
|
||||||
|
# TLS configuration to connect to the PuppetDB.
|
||||||
|
tls_config:
|
||||||
|
[ <tls_config> ]
|
||||||
|
|
||||||
|
# basic_auth, authorization, and oauth2, are mutually exclusive.
|
||||||
|
|
||||||
|
# Optional HTTP basic authentication information.
|
||||||
|
basic_auth:
|
||||||
|
[ username: <string> ]
|
||||||
|
[ password: <secret> ]
|
||||||
|
[ password_file: <string> ]
|
||||||
|
|
||||||
|
# `Authorization` HTTP header configuration.
|
||||||
|
authorization:
|
||||||
|
# Sets the authentication type.
|
||||||
|
[ type: <string> | default: Bearer ]
|
||||||
|
# Sets the credentials. It is mutually exclusive with
|
||||||
|
# `credentials_file`.
|
||||||
|
[ credentials: <secret> ]
|
||||||
|
# Sets the credentials with the credentials read from the configured file.
|
||||||
|
# It is mutually exclusive with `credentials`.
|
||||||
|
[ credentials_file: <filename> ]
|
||||||
|
|
||||||
|
# Optional OAuth 2.0 configuration.
|
||||||
|
# Cannot be used at the same time as basic_auth or authorization.
|
||||||
|
oauth2:
|
||||||
|
[ <oauth2> ]
|
||||||
|
|
||||||
|
# Optional proxy URL.
|
||||||
|
[ proxy_url: <string> ]
|
||||||
|
|
||||||
|
# Configure whether HTTP requests follow HTTP 3xx redirects.
|
||||||
|
[ follow_redirects: <bool> | default = true ]
|
||||||
|
```
|
||||||
|
|
||||||
|
See [this example Prometheus configuration file](/documentation/examples/prometheus-puppetdb.yml)
|
||||||
|
for a detailed example of configuring Prometheus with PuppetDB.
|
||||||
|
|
||||||
|
|
||||||
### `<file_sd_config>`
|
### `<file_sd_config>`
|
||||||
|
|
||||||
File-based service discovery provides a more generic way to configure static targets
|
File-based service discovery provides a more generic way to configure static targets
|
||||||
|
@ -2387,6 +2479,10 @@ nerve_sd_configs:
|
||||||
openstack_sd_configs:
|
openstack_sd_configs:
|
||||||
[ - <openstack_sd_config> ... ]
|
[ - <openstack_sd_config> ... ]
|
||||||
|
|
||||||
|
# List of PuppetDB service discovery configurations.
|
||||||
|
puppetdb_sd_configs:
|
||||||
|
[ - <puppetdb_sd_config> ... ]
|
||||||
|
|
||||||
# List of Scaleway service discovery configurations.
|
# List of Scaleway service discovery configurations.
|
||||||
scaleway_sd_configs:
|
scaleway_sd_configs:
|
||||||
[ - <scaleway_sd_config> ... ]
|
[ - <scaleway_sd_config> ... ]
|
||||||
|
|
40
documentation/examples/prometheus-puppetdb.yml
Normal file
40
documentation/examples/prometheus-puppetdb.yml
Normal file
|
@ -0,0 +1,40 @@
|
||||||
|
# Prometheus example configuration to be used with PuppetDB.
|
||||||
|
|
||||||
|
scrape_configs:
|
||||||
|
- job_name: 'puppetdb-node-exporter'
|
||||||
|
puppetdb_sd_configs:
|
||||||
|
# This example discovers the nodes which have the class Prometheus::Node_exporter.
|
||||||
|
- url: https://puppetdb.example.com
|
||||||
|
query: 'resources { type = "Class" and title = "Prometheus::Node_exporter" }'
|
||||||
|
port: 9100
|
||||||
|
tls_config:
|
||||||
|
cert_file: prometheus-public.pem
|
||||||
|
key_file: prometheus-private.pem
|
||||||
|
ca_file: ca.pem
|
||||||
|
|
||||||
|
- job_name: 'puppetdb-scrape-jobs'
|
||||||
|
puppetdb_sd_configs:
|
||||||
|
# This example uses the Prometheus::Scrape_job
|
||||||
|
# exported resources.
|
||||||
|
# https://github.com/camptocamp/prometheus-puppetdb-sd
|
||||||
|
# This examples is compatible with Prometheus-puppetdb-sd,
|
||||||
|
# if the exported Prometheus::Scrape_job only have at most one target.
|
||||||
|
- url: https://puppetdb.example.com
|
||||||
|
query: 'resources { type = "Prometheus::Scrape_job" and exported = true }'
|
||||||
|
include_parameters: true
|
||||||
|
tls_config:
|
||||||
|
cert_file: prometheus-public.pem
|
||||||
|
key_file: prometheus-private.pem
|
||||||
|
ca_file: ca.pem
|
||||||
|
relabel_configs:
|
||||||
|
- source_labels: [__meta_puppetdb_certname]
|
||||||
|
target_label: certname
|
||||||
|
- source_labels: [__meta_puppetdb_parameter_targets]
|
||||||
|
regex: '(.+),?.*'
|
||||||
|
replacement: $1
|
||||||
|
target_label: __address__
|
||||||
|
- source_labels: [__meta_puppetdb_parameter_job_name]
|
||||||
|
target_label: job
|
||||||
|
- regex: '__meta_puppetdb_parameter_labels_(.+)'
|
||||||
|
replacement: '$1'
|
||||||
|
action: labelmap
|
Loading…
Reference in a new issue