Add PuppetDB service discovery

We have been Puppet user for 10 years and we are users of
https://github.com/camptocamp/prometheus-puppetdb-sd

However, that file_sd implementation contains business logic and
assumptions around e.g. the modules which you are using.

This pull request adds a simple PuppetDB service discovery, which will
enable more use cases than the upstream sd.

Signed-off-by: Julien Pivotto <roidelapluie@inuits.eu>

config/config_test.go

@@ -45,6 +45,7 @@ import (
 	"github.com/prometheus/prometheus/discovery/marathon"
 	"github.com/prometheus/prometheus/discovery/moby"
 	"github.com/prometheus/prometheus/discovery/openstack"
+	"github.com/prometheus/prometheus/discovery/puppetdb"
 	"github.com/prometheus/prometheus/discovery/scaleway"
 	"github.com/prometheus/prometheus/discovery/targetgroup"
 	"github.com/prometheus/prometheus/discovery/triton"
@@ -790,6 +791,34 @@ var expectedConf = &Config{
 				}},
 			},
 		},
+		{
+			JobName: "service-puppetdb",
+
+			HonorTimestamps: true,
+			ScrapeInterval:  model.Duration(15 * time.Second),
+			ScrapeTimeout:   DefaultGlobalConfig.ScrapeTimeout,
+
+			MetricsPath:      DefaultScrapeConfig.MetricsPath,
+			Scheme:           DefaultScrapeConfig.Scheme,
+			HTTPClientConfig: config.DefaultHTTPClientConfig,
+
+			ServiceDiscoveryConfigs: discovery.Configs{&puppetdb.SDConfig{
+				URL:               "https://puppetserver/",
+				Query:             "resources { type = \"Package\" and title = \"httpd\" }",
+				IncludeParameters: true,
+				Port:              80,
+				RefreshInterval:   model.Duration(60 * time.Second),
+				HTTPClientConfig: config.HTTPClientConfig{
+					FollowRedirects: true,
+					TLSConfig: config.TLSConfig{
+						CAFile:   "testdata/valid_ca_file",
+						CertFile: "testdata/valid_cert_file",
+						KeyFile:  "testdata/valid_key_file",
+					},
+				},
+			},
+			},
+		},
 		{
 			JobName:         "hetzner",
 			HonorTimestamps: true,
@@ -1262,6 +1291,22 @@ var expectedErrors = []struct {
 		filename: "empty_static_config.bad.yml",
 		errMsg:   "empty or null section in static_configs",
 	},
+	{
+		filename: "puppetdb_no_query.bad.yml",
+		errMsg:   "query missing",
+	},
+	{
+		filename: "puppetdb_no_url.bad.yml",
+		errMsg:   "url missing",
+	},
+	{
+		filename: "puppetdb_bad_url.bad.yml",
+		errMsg:   "host is missing in URL",
+	},
+	{
+		filename: "puppetdb_no_scheme.bad.yml",
+		errMsg:   "url scheme must be http or https",
+	},
 	{
 		filename: "hetzner_role.bad.yml",
 		errMsg:   "unknown role",

config/testdata/conf.good.yml

@@ -307,6 +307,18 @@ scrape_configs:
           cert_file: valid_cert_file
           key_file: valid_key_file
 
+  - job_name: service-puppetdb
+    puppetdb_sd_configs:
+      - url: https://puppetserver/
+        query: 'resources { type = "Package" and title = "httpd" }'
+        include_parameters: true
+        port: 80
+        refresh_interval: 1m
+        tls_config:
+          ca_file: valid_ca_file
+          cert_file: valid_cert_file
+          key_file: valid_key_file
+
   - job_name: hetzner
     hetzner_sd_configs:
       - role: hcloud

config/testdata/puppetdb_bad_url.bad.yml

@@ -0,0 +1,4 @@
+scrape_configs:
+  - puppetdb_sd_configs:
+      - url: http://
+        query: 'resources { type = "Package" and title = "httpd" }'

config/testdata/puppetdb_no_query.bad.yml

@@ -0,0 +1,3 @@
+scrape_configs:
+  - puppetdb_sd_configs:
+      - url: http://puppetserver/

config/testdata/puppetdb_no_scheme.bad.yml

@@ -0,0 +1,4 @@
+scrape_configs:
+  - puppetdb_sd_configs:
+      - url: ftp://puppet
+        query: 'resources { type = "Package" and title = "httpd" }'

config/testdata/puppetdb_no_url.bad.yml

@@ -0,0 +1,3 @@
+scrape_configs:
+  - puppetdb_sd_configs:
+      - query: 'resources { type = "Package" and title = "httpd" }'

discovery/install/install.go

@@ -31,6 +31,7 @@ import (
 	_ "github.com/prometheus/prometheus/discovery/marathon"     // register marathon
 	_ "github.com/prometheus/prometheus/discovery/moby"         // register moby
 	_ "github.com/prometheus/prometheus/discovery/openstack"    // register openstack
+	_ "github.com/prometheus/prometheus/discovery/puppetdb"     // register puppetdb
 	_ "github.com/prometheus/prometheus/discovery/scaleway"     // register scaleway
 	_ "github.com/prometheus/prometheus/discovery/triton"       // register triton
 	_ "github.com/prometheus/prometheus/discovery/xds"          // register xds

discovery/puppetdb/fixtures/vhosts.json

@@ -0,0 +1,49 @@
+[
+   {
+      "certname": "edinburgh.example.com",
+      "environment": "prod",
+      "exported": false,
+      "file": "/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp",
+      "line": 384,
+      "parameters": {
+         "access_log": true,
+         "access_log_file": "ssl_access_log",
+         "additional_includes": [ ],
+         "directoryindex": "",
+         "docroot": "/var/www/html",
+         "ensure": "absent",
+         "options": [
+            "Indexes",
+            "FollowSymLinks",
+            "MultiViews"
+         ],
+         "php_flags": { },
+         "labels": {
+            "alias": "edinburgh"
+         },
+         "scriptaliases": [
+            {
+               "alias": "/cgi-bin",
+               "path": "/var/www/cgi-bin"
+            }
+         ]
+      },
+      "resource": "49af83866dc5a1518968b68e58a25319107afe11",
+      "tags": [
+         "roles::hypervisor",
+         "apache",
+         "apache::vhost",
+         "class",
+         "default-ssl",
+         "profile_hypervisor",
+         "vhost",
+         "profile_apache",
+         "hypervisor",
+         "__node_regexp__edinburgh",
+         "roles",
+         "node"
+      ],
+      "title": "default-ssl",
+      "type": "Apache::Vhost"
+   }
+]

discovery/puppetdb/puppetdb.go

@@ -0,0 +1,252 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package puppetdb
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"net/url"
+	"path"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/pkg/errors"
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/version"
+
+	"github.com/prometheus/prometheus/discovery"
+	"github.com/prometheus/prometheus/discovery/refresh"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+const (
+	pdbLabel            = model.MetaLabelPrefix + "puppetdb_"
+	pdbLabelCertname    = pdbLabel + "certname"
+	pdbLabelResource    = pdbLabel + "resource"
+	pdbLabelType        = pdbLabel + "type"
+	pdbLabelTitle       = pdbLabel + "title"
+	pdbLabelExported    = pdbLabel + "exported"
+	pdbLabelTags        = pdbLabel + "tags"
+	pdbLabelFile        = pdbLabel + "file"
+	pdbLabelEnvironment = pdbLabel + "environment"
+	pdbLabelParameter   = pdbLabel + "parameter_"
+	separator           = ","
+)
+
+var (
+	// DefaultSDConfig is the default PuppetDB SD configuration.
+	DefaultSDConfig = SDConfig{
+		RefreshInterval:  model.Duration(60 * time.Second),
+		Port:             80,
+		HTTPClientConfig: config.DefaultHTTPClientConfig,
+	}
+	matchContentType = regexp.MustCompile(`^(?i:application\/json(;\s*charset=("utf-8"|utf-8))?)$`)
+	userAgent        = fmt.Sprintf("Prometheus/%s", version.Version)
+)
+
+func init() {
+	discovery.RegisterConfig(&SDConfig{})
+}
+
+// SDConfig is the configuration for PuppetDB based discovery.
+type SDConfig struct {
+	HTTPClientConfig  config.HTTPClientConfig `yaml:",inline"`
+	RefreshInterval   model.Duration          `yaml:"refresh_interval,omitempty"`
+	URL               string                  `yaml:"url"`
+	Query             string                  `yaml:"query"`
+	IncludeParameters bool                    `yaml:"include_parameters"`
+	Port              int                     `yaml:"port"`
+}
+
+// Name returns the name of the Config.
+func (*SDConfig) Name() string { return "puppetdb" }
+
+// NewDiscoverer returns a Discoverer for the Config.
+func (c *SDConfig) NewDiscoverer(opts discovery.DiscovererOptions) (discovery.Discoverer, error) {
+	return NewDiscovery(c, opts.Logger)
+}
+
+// SetDirectory joins any relative file paths with dir.
+func (c *SDConfig) SetDirectory(dir string) {
+	c.HTTPClientConfig.SetDirectory(dir)
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultSDConfig
+	type plain SDConfig
+	err := unmarshal((*plain)(c))
+	if err != nil {
+		return err
+	}
+	if c.URL == "" {
+		return fmt.Errorf("URL is missing")
+	}
+	parsedURL, err := url.Parse(c.URL)
+	if err != nil {
+		return err
+	}
+	if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
+		return fmt.Errorf("URL scheme must be 'http' or 'https'")
+	}
+	if parsedURL.Host == "" {
+		return fmt.Errorf("host is missing in URL")
+	}
+	if c.Query == "" {
+		return fmt.Errorf("query missing")
+	}
+	return nil
+}
+
+// Discovery provides service discovery functionality based
+// on PuppetDB resources.
+type Discovery struct {
+	*refresh.Discovery
+	url               string
+	query             string
+	port              int
+	includeParameters bool
+	client            *http.Client
+}
+
+// NewDiscovery returns a new PuppetDB discovery for the given config.
+func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+	if logger == nil {
+		logger = log.NewNopLogger()
+	}
+
+	client, err := config.NewClientFromConfig(conf.HTTPClientConfig, "http", config.WithHTTP2Disabled())
+	if err != nil {
+		return nil, err
+	}
+	client.Timeout = time.Duration(conf.RefreshInterval)
+
+	u, err := url.Parse(conf.URL)
+	if err != nil {
+		return nil, err
+	}
+	u.Path = path.Join(u.Path, "pdb/query/v4")
+
+	d := &Discovery{
+		url:               u.String(),
+		port:              conf.Port,
+		query:             conf.Query,
+		includeParameters: conf.IncludeParameters,
+		client:            client,
+	}
+
+	d.Discovery = refresh.NewDiscovery(
+		logger,
+		"http",
+		time.Duration(conf.RefreshInterval),
+		d.refresh,
+	)
+	return d, nil
+}
+
+func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+	body := struct {
+		Query string `json:"query"`
+	}{d.query}
+	bodyBytes, err := json.Marshal(body)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequest("POST", d.url, bytes.NewBuffer(bodyBytes))
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", userAgent)
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := d.client.Do(req.WithContext(ctx))
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		io.Copy(ioutil.Discard, resp.Body)
+		resp.Body.Close()
+	}()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, errors.Errorf("server returned HTTP status %s", resp.Status)
+	}
+
+	if ct := resp.Header.Get("Content-Type"); !matchContentType.MatchString(ct) {
+		return nil, errors.Errorf("unsupported content type %s", resp.Header.Get("Content-Type"))
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var resources []Resource
+
+	if err := json.Unmarshal(b, &resources); err != nil {
+		return nil, err
+	}
+
+	tg := &targetgroup.Group{
+		// Use a pseudo-URL as source.
+		Source: d.url + "?query=" + d.query,
+	}
+
+	for _, resource := range resources {
+		labels := model.LabelSet{
+			pdbLabelCertname:    model.LabelValue(resource.Certname),
+			pdbLabelResource:    model.LabelValue(resource.Resource),
+			pdbLabelType:        model.LabelValue(resource.Type),
+			pdbLabelTitle:       model.LabelValue(resource.Title),
+			pdbLabelExported:    model.LabelValue(fmt.Sprintf("%t", resource.Exported)),
+			pdbLabelFile:        model.LabelValue(resource.File),
+			pdbLabelEnvironment: model.LabelValue(resource.Environment),
+		}
+
+		addr := net.JoinHostPort(resource.Certname, strconv.FormatUint(uint64(d.port), 10))
+		labels[model.AddressLabel] = model.LabelValue(addr)
+
+		if len(resource.Tags) > 0 {
+			// We surround the separated list with the separator as well. This way regular expressions
+			// in relabeling rules don't have to consider tag positions.
+			tags := separator + strings.Join(resource.Tags, separator) + separator
+			labels[pdbLabelTags] = model.LabelValue(tags)
+		}
+
+		// Parameters are not included by default. This should only be enabled
+		// on select resources as it might expose secrets on the Prometheus UI
+		// for certain resources.
+		if d.includeParameters {
+			for k, v := range resource.Parameters.toLabels() {
+				labels[pdbLabelParameter+k] = v
+			}
+		}
+
+		tg.Targets = append(tg.Targets, labels)
+	}
+
+	return []*targetgroup.Group{tg}, nil
+}

discovery/puppetdb/puppetdb_test.go

@@ -0,0 +1,195 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package puppetdb
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"github.com/go-kit/log"
+	"github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+	"github.com/stretchr/testify/require"
+)
+
+func mockServer(t *testing.T) *httptest.Server {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		var request struct {
+			Query string `json:"query"`
+		}
+		err := json.NewDecoder(r.Body).Decode(&request)
+		if err != nil {
+			w.WriteHeader(http.StatusBadRequest)
+			return
+		}
+		http.ServeFile(w, r, "fixtures/"+request.Query+".json")
+	}))
+	t.Cleanup(ts.Close)
+	return ts
+}
+
+func TestPuppetSlashInURL(t *testing.T) {
+	tests := map[string]string{
+		"https://puppetserver":      "https://puppetserver/pdb/query/v4",
+		"https://puppetserver/":     "https://puppetserver/pdb/query/v4",
+		"http://puppetserver:8080/": "http://puppetserver:8080/pdb/query/v4",
+		"http://puppetserver:8080":  "http://puppetserver:8080/pdb/query/v4",
+	}
+
+	for serverURL, apiURL := range tests {
+		cfg := SDConfig{
+			HTTPClientConfig: config.DefaultHTTPClientConfig,
+			URL:              serverURL,
+			Query:            "vhosts", // This is not a valid PuppetDB query, but it is used by the mock.
+			Port:             80,
+			RefreshInterval:  model.Duration(30 * time.Second),
+		}
+		d, err := NewDiscovery(&cfg, log.NewNopLogger())
+		require.NoError(t, err)
+		require.Equal(t, apiURL, d.url)
+	}
+}
+
+func TestPuppetDBRefresh(t *testing.T) {
+	ts := mockServer(t)
+
+	cfg := SDConfig{
+		HTTPClientConfig: config.DefaultHTTPClientConfig,
+		URL:              ts.URL,
+		Query:            "vhosts", // This is not a valid PuppetDB query, but it is used by the mock.
+		Port:             80,
+		RefreshInterval:  model.Duration(30 * time.Second),
+	}
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	tgs, err := d.refresh(ctx)
+	require.NoError(t, err)
+
+	expectedTargets := []*targetgroup.Group{
+		{
+			Targets: []model.LabelSet{
+				{
+					model.AddressLabel:                             model.LabelValue("edinburgh.example.com:80"),
+					model.LabelName("__meta_puppetdb_certname"):    model.LabelValue("edinburgh.example.com"),
+					model.LabelName("__meta_puppetdb_environment"): model.LabelValue("prod"),
+					model.LabelName("__meta_puppetdb_exported"):    model.LabelValue("false"),
+					model.LabelName("__meta_puppetdb_file"):        model.LabelValue("/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp"),
+					model.LabelName("__meta_puppetdb_resource"):    model.LabelValue("49af83866dc5a1518968b68e58a25319107afe11"),
+					model.LabelName("__meta_puppetdb_tags"):        model.LabelValue(",roles::hypervisor,apache,apache::vhost,class,default-ssl,profile_hypervisor,vhost,profile_apache,hypervisor,__node_regexp__edinburgh,roles,node,"),
+					model.LabelName("__meta_puppetdb_title"):       model.LabelValue("default-ssl"),
+					model.LabelName("__meta_puppetdb_type"):        model.LabelValue("Apache::Vhost"),
+				},
+			},
+			Source: ts.URL + "/pdb/query/v4?query=vhosts",
+		},
+	}
+	require.Equal(t, tgs, expectedTargets)
+}
+
+func TestPuppetDBRefreshWithParameters(t *testing.T) {
+	ts := mockServer(t)
+
+	cfg := SDConfig{
+		HTTPClientConfig:  config.DefaultHTTPClientConfig,
+		URL:               ts.URL,
+		Query:             "vhosts", // This is not a valid PuppetDB query, but it is used by the mock.
+		Port:              80,
+		IncludeParameters: true,
+		RefreshInterval:   model.Duration(30 * time.Second),
+	}
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	tgs, err := d.refresh(ctx)
+	require.NoError(t, err)
+
+	expectedTargets := []*targetgroup.Group{
+		{
+			Targets: []model.LabelSet{
+				{
+					model.AddressLabel:                                           model.LabelValue("edinburgh.example.com:80"),
+					model.LabelName("__meta_puppetdb_certname"):                  model.LabelValue("edinburgh.example.com"),
+					model.LabelName("__meta_puppetdb_environment"):               model.LabelValue("prod"),
+					model.LabelName("__meta_puppetdb_exported"):                  model.LabelValue("false"),
+					model.LabelName("__meta_puppetdb_file"):                      model.LabelValue("/etc/puppetlabs/code/environments/prod/modules/upstream/apache/manifests/init.pp"),
+					model.LabelName("__meta_puppetdb_parameter_access_log"):      model.LabelValue("true"),
+					model.LabelName("__meta_puppetdb_parameter_access_log_file"): model.LabelValue("ssl_access_log"),
+					model.LabelName("__meta_puppetdb_parameter_docroot"):         model.LabelValue("/var/www/html"),
+					model.LabelName("__meta_puppetdb_parameter_ensure"):          model.LabelValue("absent"),
+					model.LabelName("__meta_puppetdb_parameter_labels_alias"):    model.LabelValue("edinburgh"),
+					model.LabelName("__meta_puppetdb_parameter_options"):         model.LabelValue("Indexes,FollowSymLinks,MultiViews"),
+					model.LabelName("__meta_puppetdb_resource"):                  model.LabelValue("49af83866dc5a1518968b68e58a25319107afe11"),
+					model.LabelName("__meta_puppetdb_tags"):                      model.LabelValue(",roles::hypervisor,apache,apache::vhost,class,default-ssl,profile_hypervisor,vhost,profile_apache,hypervisor,__node_regexp__edinburgh,roles,node,"),
+					model.LabelName("__meta_puppetdb_title"):                     model.LabelValue("default-ssl"),
+					model.LabelName("__meta_puppetdb_type"):                      model.LabelValue("Apache::Vhost"),
+				},
+			},
+			Source: ts.URL + "/pdb/query/v4?query=vhosts",
+		},
+	}
+	require.Equal(t, tgs, expectedTargets)
+}
+
+func TestPuppetDBInvalidCode(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusBadRequest)
+	}))
+
+	t.Cleanup(ts.Close)
+
+	cfg := SDConfig{
+		HTTPClientConfig: config.DefaultHTTPClientConfig,
+		URL:              ts.URL,
+		RefreshInterval:  model.Duration(30 * time.Second),
+	}
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	_, err = d.refresh(ctx)
+	require.EqualError(t, err, "server returned HTTP status 400 Bad Request")
+}
+
+func TestPuppetDBInvalidFormat(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, "{}")
+	}))
+
+	t.Cleanup(ts.Close)
+
+	cfg := SDConfig{
+		HTTPClientConfig: config.DefaultHTTPClientConfig,
+		URL:              ts.URL,
+		RefreshInterval:  model.Duration(30 * time.Second),
+	}
+
+	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	require.NoError(t, err)
+
+	ctx := context.Background()
+	_, err = d.refresh(ctx)
+	require.EqualError(t, err, "unsupported content type text/plain; charset=utf-8")
+}

discovery/puppetdb/resources.go

@@ -0,0 +1,82 @@
+// Copyright 2021 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package puppetdb
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/util/strutil"
+)
+
+type Resource struct {
+	Certname    string     `json:"certname"`
+	Resource    string     `json:"resource"`
+	Type        string     `json:"type"`
+	Title       string     `json:"title"`
+	Exported    bool       `json:"exported"`
+	Tags        []string   `json:"tags"`
+	File        string     `json:"file"`
+	Environment string     `json:"environment"`
+	Parameters  Parameters `json:"parameters"`
+}
+
+type Parameters map[string]interface{}
+
+func (p *Parameters) toLabels() model.LabelSet {
+	labels := model.LabelSet{}
+
+	for k, v := range *p {
+		var labelValue string
+		switch value := v.(type) {
+		case string:
+			labelValue = value
+		case bool:
+			labelValue = strconv.FormatBool(value)
+		case []string:
+			labelValue = separator + strings.Join(value, separator) + separator
+		case []interface{}:
+			if len(value) == 0 {
+				continue
+			}
+			values := make([]string, len(value))
+			for i, v := range value {
+				switch value := v.(type) {
+				case string:
+					values[i] = value
+				case bool:
+					values[i] = strconv.FormatBool(value)
+				case []string:
+					values[i] = separator + strings.Join(value, separator) + separator
+				}
+			}
+			labelValue = strings.Join(values, separator)
+		case map[string]interface{}:
+			subParameter := Parameters(value)
+			prefix := strutil.SanitizeLabelName(k + "_")
+			for subk, subv := range subParameter.toLabels() {
+				labels[model.LabelName(prefix)+subk] = subv
+			}
+		default:
+			continue
+		}
+		if labelValue == "" {
+			continue
+		}
+		name := strutil.SanitizeLabelName(k)
+		labels[model.LabelName(name)] = model.LabelValue(labelValue)
+	}
+	return labels
+}

docs/configuration/configuration.md

@@ -272,6 +272,10 @@ nerve_sd_configs:
 openstack_sd_configs:
   [ - <openstack_sd_config> ... ]
 
+# List of PuppetDB service discovery configurations.
+puppetdb_sd_configs:
+  [ - <puppetdb_sd_config> ... ]
+
 # List of Scaleway service discovery configurations.
 scaleway_sd_configs:
   [ - <scaleway_sd_config> ... ]
@@ -1069,6 +1073,94 @@ tls_config:
   [ <tls_config> ]
 ```
 
+### `<puppetdb_sd_config>`
+
+PuppetDB SD configurations allow retrieving scrape targets from
+[PuppetDB](https://puppet.com/docs/puppetdb/latest/index.html) resources.
+
+This SD discovers resources and will create a target for each resource returned
+by the API.
+
+The resource address is the `certname` of the resource and can be changed during
+[relabeling](#relabel_config).
+
+The following meta labels are available on targets during [relabeling](#relabel_config):
+
+* `__meta_puppetdb_certname`: the name of the node associated with the resource
+* `__meta_puppetdb_resource`: a SHA-1 hash of the resource’s type, title, and parameters, for identification
+* `__meta_puppetdb_type`: the resource type
+* `__meta_puppetdb_title`: the resource title
+* `__meta_puppetdb_exported`: whether the resource is exported (`"true"` or `"false"`)
+* `__meta_puppetdb_tags`: comma separated list of resource tags
+* `__meta_puppetdb_file`: the manifest file in which the resource was declared
+* `__meta_puppetdb_environment`: the environment of the node associated with the resource
+* `__meta_puppetdb_parameter_<parametername>`: the parameters of the resource
+
+
+See below for the configuration options for PuppetDB discovery:
+
+```yaml
+# The URL of the PuppetDB root query endpoint.
+url: <string>
+
+# Puppet Query Language (PQL) query. Only resources are supported.
+# https://puppet.com/docs/puppetdb/latest/api/query/v4/pql.html
+query: <string>
+
+# Whether to include the parameters as meta labels.
+# Due to the differences between parameter types and Prometheus labels,
+# some parameters might not be rendered. The format of the parameters might
+# also change in future releases.
+#
+# Note: Enabling this exposes parameters in the Prometheus UI and API. Make sure
+# that you don't have secrets exposed as parameters if you enable this.
+[ include_parameters: <boolean> | default = false ]
+
+# Refresh interval to re-read the resources list.
+[ refresh_interval: <duration> | default = 60s ]
+
+# The port to scrape metrics from.
+[ port: <int> | default = 80 ]
+
+# TLS configuration to connect to the PuppetDB.
+tls_config:
+  [ <tls_config> ]
+
+# basic_auth, authorization, and oauth2, are mutually exclusive.
+
+# Optional HTTP basic authentication information.
+basic_auth:
+  [ username: <string> ]
+  [ password: <secret> ]
+  [ password_file: <string> ]
+
+# `Authorization` HTTP header configuration.
+authorization:
+  # Sets the authentication type.
+  [ type: <string> | default: Bearer ]
+  # Sets the credentials. It is mutually exclusive with
+  # `credentials_file`.
+  [ credentials: <secret> ]
+  # Sets the credentials with the credentials read from the configured file.
+  # It is mutually exclusive with `credentials`.
+  [ credentials_file: <filename> ]
+
+# Optional OAuth 2.0 configuration.
+# Cannot be used at the same time as basic_auth or authorization.
+oauth2:
+  [ <oauth2> ]
+
+# Optional proxy URL.
+[ proxy_url: <string> ]
+
+# Configure whether HTTP requests follow HTTP 3xx redirects.
+[ follow_redirects: <bool> | default = true ]
+```
+
+See [this example Prometheus configuration file](/documentation/examples/prometheus-puppetdb.yml)
+for a detailed example of configuring Prometheus with PuppetDB.
+
+
 ### `<file_sd_config>`
 
 File-based service discovery provides a more generic way to configure static targets
@@ -2387,6 +2479,10 @@ nerve_sd_configs:
 openstack_sd_configs:
   [ - <openstack_sd_config> ... ]
 
+# List of PuppetDB service discovery configurations.
+puppetdb_sd_configs:
+  [ - <puppetdb_sd_config> ... ]
+
 # List of Scaleway service discovery configurations.
 scaleway_sd_configs:
   [ - <scaleway_sd_config> ... ]

documentation/examples/prometheus-puppetdb.yml

@@ -0,0 +1,40 @@
+# Prometheus example configuration to be used with PuppetDB.
+
+scrape_configs:
+  - job_name: 'puppetdb-node-exporter'
+    puppetdb_sd_configs:
+    # This example discovers the nodes which have the class Prometheus::Node_exporter.
+      - url: https://puppetdb.example.com
+        query: 'resources { type = "Class" and title = "Prometheus::Node_exporter" }'
+        port: 9100
+        tls_config:
+          cert_file: prometheus-public.pem
+          key_file: prometheus-private.pem
+          ca_file: ca.pem
+
+  - job_name: 'puppetdb-scrape-jobs'
+    puppetdb_sd_configs:
+    # This example uses the Prometheus::Scrape_job
+    # exported resources.
+    # https://github.com/camptocamp/prometheus-puppetdb-sd
+    # This examples is compatible with Prometheus-puppetdb-sd,
+    # if the exported Prometheus::Scrape_job only have at most one target.
+      - url: https://puppetdb.example.com
+        query: 'resources { type = "Prometheus::Scrape_job" and exported = true }'
+        include_parameters: true
+        tls_config:
+          cert_file: prometheus-public.pem
+          key_file: prometheus-private.pem
+          ca_file: ca.pem
+    relabel_configs:
+      - source_labels: [__meta_puppetdb_certname]
+        target_label: certname
+      - source_labels: [__meta_puppetdb_parameter_targets]
+        regex: '(.+),?.*'
+        replacement: $1
+        target_label: __address__
+      - source_labels: [__meta_puppetdb_parameter_job_name]
+        target_label: job
+      - regex: '__meta_puppetdb_parameter_labels_(.+)'
+        replacement: '$1'
+        action: labelmap