Kubernetes SD example: separate out cluster level components & services

documentation/examples/prometheus-kubernetes.yml

@@ -1,9 +1,21 @@
-# A scrape configuration for running prometheus in cluster on kubernetes. It
-# will create endpoints for node and master roles, as well as any service
-# which is annotated with `prometheus_io_scrape=true`
+# A scrape configuration for running Prometheus on a Kubernetes cluster.
+# This uses separate scrape configs for cluster components (i.e. master, node)
+# and services to allow each to use different authentication configs.
+#
+# Kubernetes labels will be added as Prometheus labels on metrics via the
+# `labelmap` relabeling action.
 
+# Scrape config for cluster components.
 scrape_configs:
-- job_name: 'kubernetes'
+- job_name: 'kubernetes-cluster'
+
+  # This TLS & bearer token file config is used to connect to the actual scrape
+  # endpoints for cluster components. This is separate to discovery auth
+  # configuration (`in_cluster` below) because discovery & scraping are two
+  # separate concerns in Prometheus.
+  tls_config:
+    ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
+  bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
 
   kubernetes_sd_configs:
   - masters:
@@ -11,9 +23,40 @@ scrape_configs:
     in_cluster: true
 
   relabel_configs:
-  - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_scrape]
+  - source_labels: [__meta_kubernetes_role]
     action: keep
-    regex: ^(?:(?:master|node);.*|.*;true)$
+    regex: ^(?:master|node)$
+  - action: labelmap
+    regex: ^__meta_kubernetes_node_label_(.+)$
+    replacement: $1
+  - source_labels: [__meta_kubernetes_role]
+    action: replace
+    regex: ^(.+)$
+    target_label: kubernetes_role
+    replacement: $1
+
+# Scrape config for services.
+#
+# The relabeling allows the actual service scrape endpoint to be configured
+# via the following annotations:
+#
+# * `prometheus.io/scrape`: Only scrape services that have a value of `true`
+# * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need
+# to set this to `https` & most likely set the `tls_config` of the scrape config.
+# * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
+# * `prometheus.io/port`: If the metrics are exposed on a different port to the
+# service then set this appropriately.
+- job_name: 'kubernetes-services'
+
+  kubernetes_sd_configs:
+  - masters:
+    - 'https://kubernetes.default.svc'
+    in_cluster: true
+
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+    action: keep
+    regex: ^true$
   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
     action: replace
     target_label: __scheme__
@@ -32,3 +75,8 @@ scrape_configs:
   - action: labelmap
     regex: ^__meta_kubernetes_service_label_(.+)$
     replacement: $1
+  - source_labels: [__meta_kubernetes_role]
+    action: replace
+    regex: ^(.+)$
+    target_label: kubernetes_role
+    replacement: $1

retrieval/discovery/kubernetes/discovery.go

@@ -584,7 +584,7 @@ func newKubernetesHTTPClient(conf *config.KubernetesSDConfig) (*http.Client, err
 			bearerTokenFile = serviceAccountToken
 		}
 		if len(caFile) == 0 {
-			// With recent versions, the CA certificate is provided as a token
+			// With recent versions, the CA certificate is mounted as a secret
 			// but we need to handle older versions too. In this case, don't
 			// set the CAFile & the configuration will have to use InsecureSkipVerify.
 			if _, err := os.Stat(serviceAccountCACert); err == nil {