Merge pull request #1225 from fabric8io/kubernetes-discovery

KubernetesSD: Create targets for services as well as service endpoints

documentation/examples/prometheus-kubernetes.yml

@@ -28,14 +28,11 @@ scrape_configs:
     regex: (?:apiserver|node)
   - action: labelmap
     regex: __meta_kubernetes_node_label_(.+)
-    replacement: $1
   - source_labels: [__meta_kubernetes_role]
     action: replace
-    regex: (.+)
     target_label: kubernetes_role
-    replacement: $1
 
-# Scrape config for services.
+# Scrape config for service endpoints.
 #
 # The relabeling allows the actual service scrape endpoint to be configured
 # via the following annotations:
@@ -46,7 +43,7 @@ scrape_configs:
 # * `prometheus.io/path`: If the metrics path is not `/metrics` override this.
 # * `prometheus.io/port`: If the metrics are exposed on a different port to the
 # service then set this appropriately.
-- job_name: 'kubernetes-services'
+- job_name: 'kubernetes-service-endpoints'
 
   kubernetes_sd_configs:
   - api_servers:
@@ -54,19 +51,16 @@ scrape_configs:
     in_cluster: true
 
   relabel_configs:
-  - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scrape]
+  - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_scrape]
     action: keep
-    regex: true
+    regex: endpoint;true
   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
     action: replace
     target_label: __scheme__
     regex: (https?)
-    replacement: $1
   - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
     action: replace
     target_label: __metrics_path__
-    regex: (.+)
-    replacement: $1
   - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
     action: replace
     target_label: __address__
@@ -74,19 +68,53 @@ scrape_configs:
     replacement: $1:$2
   - action: labelmap
     regex: __meta_kubernetes_service_label_(.+)
-    replacement: $1
   - source_labels: [__meta_kubernetes_role]
     action: replace
-    regex: (.+)
     target_label: kubernetes_role
-    replacement: $1
   - source_labels: [__meta_kubernetes_service_namespace]
     action: replace
-    regex: (.+)
     target_label: kubernetes_namespace
-    replacement: $1
   - source_labels: [__meta_kubernetes_service_name]
     action: replace
-    regex: (.+)
     target_label: kubernetes_name
-    replacement: $1
+
+# Example scrape config for probing services via the Blackbox Exporter.
+#
+# The relabeling allows the actual service scrape endpoint to be configured
+# via the following annotations:
+#
+# * `prometheus.io/probe`: Only probe services that have a value of `true`
+- job_name: 'kubernetes-services'
+
+  metrics_path: /probe
+  params:
+    module: [http_2xx]
+
+  kubernetes_sd_configs:
+  - api_servers:
+    - 'https://kubernetes.default.svc'
+    in_cluster: true
+
+  relabel_configs:
+  - source_labels: [__meta_kubernetes_role, __meta_kubernetes_service_annotation_prometheus_io_probe]
+    action: keep
+    regex: service;true
+  - source_labels: []
+    target_label: __address__
+    replacement: prom-blackbox.default.svc:9115
+  - source_labels: [__address__]
+    regex: (.*)(:80)?
+    target_label: __param_target
+  - source_labels: [__param_target]
+    target_label: instance
+  - action: labelmap
+    regex: __meta_kubernetes_service_label_(.+)
+  - source_labels: [__meta_kubernetes_role]
+    action: replace
+    target_label: kubernetes_role
+  - source_labels: [__meta_kubernetes_service_namespace]
+    action: replace
+    target_label: kubernetes_namespace
+  - source_labels: [__meta_kubernetes_service_name]
+    action: replace
+    target_label: kubernetes_name

retrieval/discovery/kubernetes/discovery.go

@@ -501,7 +501,6 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints)
 		Labels: model.LabelSet{
 			serviceNamespaceLabel: model.LabelValue(service.ObjectMeta.Namespace),
 			serviceNameLabel:      model.LabelValue(service.ObjectMeta.Name),
-			roleLabel:             model.LabelValue("service"),
 		},
 	}
 
@@ -515,6 +514,12 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints)
 		tg.Labels[model.LabelName(labelName)] = model.LabelValue(v)
 	}
 
+	t := model.LabelSet{
+		model.AddressLabel: model.LabelValue(service.ObjectMeta.Name + "." + service.ObjectMeta.Namespace + ".svc"),
+		roleLabel:          model.LabelValue("service"),
+	}
+	tg.Targets = append(tg.Targets, t)
+
 	// Now let's loop through the endpoints & add them to the target group with appropriate labels.
 	for _, ss := range eps.Subsets {
 		epPort := ss.Ports[0].Port
@@ -526,7 +531,10 @@ func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints)
 			}
 			address := fmt.Sprintf("%s:%d", ipAddr, epPort)
 
-			t := model.LabelSet{model.AddressLabel: model.LabelValue(address)}
+			t := model.LabelSet{
+				model.AddressLabel: model.LabelValue(address),
+				roleLabel:          model.LabelValue("endpoint"),
+			}
 
 			tg.Targets = append(tg.Targets, t)
 		}