KubernetesSD: Create targets for services as well as service endpoints

documentation/examples/prometheus-kubernetes.yml

@@ -35,7 +35,7 @@ scrape_configs:
     target_label: kubernetes_role
     replacement: $1
 
-# Scrape config for services.
+# Scrape config for service endpoints.
 #
 # The relabeling allows the actual service scrape endpoint to be configured
 # via the following annotations:
@@ -46,7 +46,7 @@ scrape_configs:
 # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
 # * `prometheus.io/port`: If the metrics are exposed on a different port to the
 # service then set this appropriately.
-- job_name: 'kubernetes-services'
+- job_name: 'kubernetes-service-endpoints'
 
   kubernetes_sd_configs:
   - api_servers:
@@ -54,9 +54,9 @@ scrape_configs:
     in_cluster: true
 
   relabel_configs:
-  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+  - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_scrape]
     action: keep
-    regex: true
+    regex: endpoint;true
   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
     action: replace
     target_label: __scheme__
@@ -90,3 +90,55 @@ scrape_configs:
     regex: (.+)
     target_label: kubernetes_name
     replacement: $1
+
+# Example scrape config for probing services via the Blackbox Exporter.
+#
+# The relabeling allows the actual service scrape endpoint to be configured
+# via the following annotations:
+#
+# * `prometheus.io/probe`: Only probe services that have a value of `true`
+- job_name: 'kubernetes-services'
+
+  metrics_path: /probe
+  params:
+    module: [http_2xx]
+
+  kubernetes_sd_configs:
+  - api_servers:
+    - 'https://kubernetes.default.svc'
+    in_cluster: true
+
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_probe]
+    action: keep
+    regex: service;true
+  - source_labels: []
+    regex: .*
+    target_label: __address__
+    replacement: prom-blackbox.default.svc:9115
+  - source_labels: [__address__]
+    regex: (.*)(:80)?
+    target_label: __param_target
+    replacement: ${1}
+  - source_labels: [__param_target]
+    regex: (.*)
+    target_label: instance
+    replacement: ${1}
+  - action: labelmap
+    regex: __meta_kubernetes_service_label_(.+)
+    replacement: $1
+  - source_labels: [__meta_kubernetes_role]
+    action: replace
+    regex: (.+)
+    target_label: kubernetes_role
+    replacement: $1
+  - source_labels: [__meta_kubernetes_service_namespace]
+    action: replace
+    regex: (.+)
+    target_label: kubernetes_namespace
+    replacement: $1
+  - source_labels: [__meta_kubernetes_service_name]
+    action: replace
+    regex: (.+)
+    target_label: kubernetes_name
+    replacement: $1

retrieval/discovery/kubernetes/discovery.go

@@ -501,7 +501,6 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints)
 		Labels: model.LabelSet{
 			serviceNamespaceLabel: model.LabelValue(service.ObjectMeta.Namespace),
 			serviceNameLabel:      model.LabelValue(service.ObjectMeta.Name),
-			roleLabel:             model.LabelValue("service"),
 		},
 	}
 
@@ -515,6 +514,12 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints)
 		tg.Labels[model.LabelName(labelName)] = model.LabelValue(v)
 	}
 
+	t := model.LabelSet{
+		model.AddressLabel: model.LabelValue(service.ObjectMeta.Name + "." + service.ObjectMeta.Namespace + ".svc"),
+		roleLabel:          model.LabelValue("service"),
+	}
+	tg.Targets = append(tg.Targets, t)
+
 	// Now let's loop through the endpoints & add them to the target group with appropriate labels.
 	for _, ss := range eps.Subsets {
 		epPort := ss.Ports[0].Port
@@ -526,7 +531,10 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints)
 			}
 			address := fmt.Sprintf("%s:%d", ipAddr, epPort)
 
-			t := model.LabelSet{model.AddressLabel: model.LabelValue(address)}
+			t := model.LabelSet{
+				model.AddressLabel: model.LabelValue(address),
+				roleLabel:          model.LabelValue("endpoint"),
+			}
 
 			tg.Targets = append(tg.Targets, t)
 		}