mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-29 07:29:42 -08:00
1067 lines
32 KiB
Go
1067 lines
32 KiB
Go
// Copyright 2015 The Prometheus Authors
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package kubernetes
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/prometheus/common/log"
|
|
"github.com/prometheus/common/model"
|
|
"golang.org/x/net/context"
|
|
|
|
"github.com/prometheus/prometheus/config"
|
|
"github.com/prometheus/prometheus/util/httputil"
|
|
"github.com/prometheus/prometheus/util/strutil"
|
|
)
|
|
|
|
const (
|
|
// kubernetesMetaLabelPrefix is the meta prefix used for all meta labels.
|
|
// in this discovery.
|
|
metaLabelPrefix = model.MetaLabelPrefix + "kubernetes_"
|
|
|
|
// roleLabel is the name for the label containing a target's role.
|
|
roleLabel = metaLabelPrefix + "role"
|
|
|
|
sourcePodPrefix = "pods"
|
|
// podsTargetGroupNAme is the name given to the target group for pods
|
|
podsTargetGroupName = "pods"
|
|
// podNamespaceLabel is the name for the label containing a target pod's namespace
|
|
podNamespaceLabel = metaLabelPrefix + "pod_namespace"
|
|
// podNameLabel is the name for the label containing a target pod's name
|
|
podNameLabel = metaLabelPrefix + "pod_name"
|
|
// podAddressLabel is the name for the label containing a target pod's IP address (the PodIP)
|
|
podAddressLabel = metaLabelPrefix + "pod_address"
|
|
// podContainerNameLabel is the name for the label containing a target's container name
|
|
podContainerNameLabel = metaLabelPrefix + "pod_container_name"
|
|
// podContainerPortNameLabel is the name for the label containing the name of the port selected for a target
|
|
podContainerPortNameLabel = metaLabelPrefix + "pod_container_port_name"
|
|
// PodContainerPortListLabel is the name for the label containing a list of all TCP ports on the target container
|
|
podContainerPortListLabel = metaLabelPrefix + "pod_container_port_list"
|
|
// PodContainerPortMapPrefix is the prefix used to create the names of labels that associate container port names to port values
|
|
// Such labels will be named (podContainerPortMapPrefix)_(PortName) = (ContainerPort)
|
|
podContainerPortMapPrefix = metaLabelPrefix + "pod_container_port_map_"
|
|
// podReadyLabel is the name for the label containing the 'Ready' status (true/false/unknown) for a target
|
|
podReadyLabel = metaLabelPrefix + "pod_ready"
|
|
// podLabelPrefix is the prefix for prom label names corresponding to k8s labels for a target pod
|
|
podLabelPrefix = metaLabelPrefix + "pod_label_"
|
|
// podAnnotationPrefix is the prefix for prom label names corresponding to k8s annotations for a target pod
|
|
podAnnotationPrefix = metaLabelPrefix + "pod_annotation_"
|
|
|
|
sourceServicePrefix = "services"
|
|
// serviceNamespaceLabel is the name for the label containing a target's service namespace.
|
|
serviceNamespaceLabel = metaLabelPrefix + "service_namespace"
|
|
// serviceNameLabel is the name for the label containing a target's service name.
|
|
serviceNameLabel = metaLabelPrefix + "service_name"
|
|
// serviceLabelPrefix is the prefix for the service labels.
|
|
serviceLabelPrefix = metaLabelPrefix + "service_label_"
|
|
// serviceAnnotationPrefix is the prefix for the service annotations.
|
|
serviceAnnotationPrefix = metaLabelPrefix + "service_annotation_"
|
|
|
|
// nodesTargetGroupName is the name given to the target group for nodes.
|
|
nodesTargetGroupName = "nodes"
|
|
// nodeLabelPrefix is the prefix for the node labels.
|
|
nodeLabelPrefix = metaLabelPrefix + "node_label_"
|
|
// nodeAddressPrefix is the prefix for the node addresses.
|
|
nodeAddressPrefix = metaLabelPrefix + "node_address_"
|
|
// nodePortLabel is the name of the label for the node port.
|
|
nodePortLabel = metaLabelPrefix + "node_port"
|
|
|
|
// apiServersTargetGroupName is the name given to the target group for API servers.
|
|
apiServersTargetGroupName = "apiServers"
|
|
|
|
serviceAccountToken = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
|
serviceAccountCACert = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
|
|
|
apiVersion = "v1"
|
|
apiPrefix = "/api/" + apiVersion
|
|
nodesURL = apiPrefix + "/nodes"
|
|
podsURL = apiPrefix + "/pods"
|
|
servicesURL = apiPrefix + "/services"
|
|
endpointsURL = apiPrefix + "/endpoints"
|
|
serviceEndpointsURL = apiPrefix + "/namespaces/%s/endpoints/%s"
|
|
)
|
|
|
|
// Discovery implements a TargetProvider for Kubernetes services.
|
|
type Discovery struct {
|
|
client *http.Client
|
|
Conf *config.KubernetesSDConfig
|
|
|
|
apiServers []config.URL
|
|
apiServersMu sync.RWMutex
|
|
nodes map[string]*Node
|
|
services map[string]map[string]*Service
|
|
// map of namespace to (map of pod name to pod)
|
|
pods map[string]map[string]*Pod
|
|
nodesMu sync.RWMutex
|
|
servicesMu sync.RWMutex
|
|
podsMu sync.RWMutex
|
|
runDone chan struct{}
|
|
}
|
|
|
|
// Initialize sets up the discovery for usage.
|
|
func (kd *Discovery) Initialize() error {
|
|
client, err := newKubernetesHTTPClient(kd.Conf)
|
|
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
kd.apiServers = kd.Conf.APIServers
|
|
kd.client = client
|
|
kd.runDone = make(chan struct{})
|
|
|
|
return nil
|
|
}
|
|
|
|
// Run implements the TargetProvider interface.
|
|
func (kd *Discovery) Run(ctx context.Context, ch chan<- []*config.TargetGroup) {
|
|
log.Debugf("Kubernetes Discovery.Run beginning")
|
|
defer close(ch)
|
|
|
|
// Send an initial full view.
|
|
// TODO(fabxc): this does not include all available services and service
|
|
// endpoints yet. Service endpoints were also missing in the previous Sources() method.
|
|
var all []*config.TargetGroup
|
|
|
|
all = append(all, kd.updateAPIServersTargetGroup())
|
|
all = append(all, kd.updateNodesTargetGroup())
|
|
|
|
pods, _, err := kd.getPods()
|
|
if err != nil {
|
|
log.Errorf("Cannot initialize pods collection: %s", err)
|
|
return
|
|
}
|
|
kd.podsMu.Lock()
|
|
kd.pods = pods
|
|
kd.podsMu.Unlock()
|
|
|
|
all = append(all, kd.updatePodsTargetGroup())
|
|
for _, ns := range kd.pods {
|
|
for _, pod := range ns {
|
|
all = append(all, kd.updatePodTargetGroup(pod))
|
|
}
|
|
}
|
|
|
|
select {
|
|
case ch <- all:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
|
|
retryInterval := time.Duration(kd.Conf.RetryInterval)
|
|
|
|
update := make(chan interface{}, 10)
|
|
|
|
go kd.watchNodes(update, ctx.Done(), retryInterval)
|
|
go kd.startServiceWatch(update, ctx.Done(), retryInterval)
|
|
go kd.watchPods(update, ctx.Done(), retryInterval)
|
|
|
|
for {
|
|
tg := []*config.TargetGroup{}
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case event := <-update:
|
|
switch obj := event.(type) {
|
|
case *nodeEvent:
|
|
log.Debugf("k8s discovery received node event (EventType=%s, Node Name=%s)", obj.EventType, obj.Node.ObjectMeta.Name)
|
|
kd.updateNode(obj.Node, obj.EventType)
|
|
tg = append(tg, kd.updateNodesTargetGroup())
|
|
case *serviceEvent:
|
|
log.Debugf("k8s discovery received service event (EventType=%s, Service Name=%s)", obj.EventType, obj.Service.ObjectMeta.Name)
|
|
tg = append(tg, kd.updateService(obj.Service, obj.EventType))
|
|
case *endpointsEvent:
|
|
log.Debugf("k8s discovery received endpoint event (EventType=%s, Endpoint Name=%s)", obj.EventType, obj.Endpoints.ObjectMeta.Name)
|
|
tg = append(tg, kd.updateServiceEndpoints(obj.Endpoints, obj.EventType))
|
|
case *podEvent:
|
|
log.Debugf("k8s discovery received pod event (EventType=%s, Pod Name=%s)", obj.EventType, obj.Pod.ObjectMeta.Name)
|
|
// Update the per-pod target group
|
|
kd.updatePod(obj.Pod, obj.EventType)
|
|
tg = append(tg, kd.updatePodTargetGroup(obj.Pod))
|
|
// ...and update the all pods target group
|
|
tg = append(tg, kd.updatePodsTargetGroup())
|
|
}
|
|
}
|
|
|
|
if tg == nil {
|
|
continue
|
|
}
|
|
|
|
for _, t := range tg {
|
|
select {
|
|
case ch <- []*config.TargetGroup{t}:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func (kd *Discovery) queryAPIServerPath(path string) (*http.Response, error) {
|
|
req, err := http.NewRequest("GET", path, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return kd.queryAPIServerReq(req)
|
|
}
|
|
|
|
func (kd *Discovery) queryAPIServerReq(req *http.Request) (*http.Response, error) {
|
|
// Lock in case we need to rotate API servers to request.
|
|
kd.apiServersMu.Lock()
|
|
defer kd.apiServersMu.Unlock()
|
|
var lastErr error
|
|
for i := 0; i < len(kd.apiServers); i++ {
|
|
cloneReq := *req
|
|
cloneReq.URL.Host = kd.apiServers[0].Host
|
|
cloneReq.URL.Scheme = kd.apiServers[0].Scheme
|
|
res, err := kd.client.Do(&cloneReq)
|
|
if err == nil {
|
|
return res, nil
|
|
}
|
|
lastErr = err
|
|
kd.rotateAPIServers()
|
|
}
|
|
return nil, fmt.Errorf("unable to query any API servers: %v", lastErr)
|
|
}
|
|
|
|
func (kd *Discovery) rotateAPIServers() {
|
|
if len(kd.apiServers) > 1 {
|
|
kd.apiServers = append(kd.apiServers[1:], kd.apiServers[0])
|
|
}
|
|
}
|
|
|
|
func (kd *Discovery) updateAPIServersTargetGroup() *config.TargetGroup {
|
|
tg := &config.TargetGroup{
|
|
Source: apiServersTargetGroupName,
|
|
Labels: model.LabelSet{
|
|
roleLabel: model.LabelValue("apiserver"),
|
|
},
|
|
}
|
|
|
|
for _, apiServer := range kd.apiServers {
|
|
apiServerAddress := apiServer.Host
|
|
_, _, err := net.SplitHostPort(apiServerAddress)
|
|
// If error then no port is specified - use default for scheme.
|
|
if err != nil {
|
|
switch apiServer.Scheme {
|
|
case "http":
|
|
apiServerAddress = net.JoinHostPort(apiServerAddress, "80")
|
|
case "https":
|
|
apiServerAddress = net.JoinHostPort(apiServerAddress, "443")
|
|
}
|
|
}
|
|
|
|
t := model.LabelSet{
|
|
model.AddressLabel: model.LabelValue(apiServerAddress),
|
|
model.SchemeLabel: model.LabelValue(apiServer.Scheme),
|
|
}
|
|
tg.Targets = append(tg.Targets, t)
|
|
}
|
|
|
|
return tg
|
|
}
|
|
|
|
func (kd *Discovery) updateNodesTargetGroup() *config.TargetGroup {
|
|
kd.nodesMu.RLock()
|
|
defer kd.nodesMu.RUnlock()
|
|
|
|
tg := &config.TargetGroup{
|
|
Source: nodesTargetGroupName,
|
|
Labels: model.LabelSet{
|
|
roleLabel: model.LabelValue("node"),
|
|
},
|
|
}
|
|
|
|
// Now let's loop through the nodes & add them to the target group with appropriate labels.
|
|
for nodeName, node := range kd.nodes {
|
|
defaultNodeAddress, nodeAddressMap, err := nodeAddresses(node)
|
|
if err != nil {
|
|
log.Debugf("Skipping node %s: %s", node.Name, err)
|
|
continue
|
|
}
|
|
|
|
kubeletPort := int(node.Status.DaemonEndpoints.KubeletEndpoint.Port)
|
|
|
|
address := fmt.Sprintf("%s:%d", defaultNodeAddress.String(), kubeletPort)
|
|
|
|
t := model.LabelSet{
|
|
model.AddressLabel: model.LabelValue(address),
|
|
model.InstanceLabel: model.LabelValue(nodeName),
|
|
}
|
|
|
|
for addrType, ip := range nodeAddressMap {
|
|
labelName := strutil.SanitizeLabelName(nodeAddressPrefix + string(addrType))
|
|
t[model.LabelName(labelName)] = model.LabelValue(ip[0].String())
|
|
}
|
|
|
|
t[model.LabelName(nodePortLabel)] = model.LabelValue(strconv.Itoa(kubeletPort))
|
|
|
|
for k, v := range node.ObjectMeta.Labels {
|
|
labelName := strutil.SanitizeLabelName(nodeLabelPrefix + k)
|
|
t[model.LabelName(labelName)] = model.LabelValue(v)
|
|
}
|
|
tg.Targets = append(tg.Targets, t)
|
|
}
|
|
|
|
return tg
|
|
}
|
|
|
|
func (kd *Discovery) updateNode(node *Node, eventType EventType) {
|
|
kd.nodesMu.Lock()
|
|
defer kd.nodesMu.Unlock()
|
|
updatedNodeName := node.ObjectMeta.Name
|
|
switch eventType {
|
|
case Deleted:
|
|
// Deleted - remove from nodes map.
|
|
delete(kd.nodes, updatedNodeName)
|
|
case Added, Modified:
|
|
// Added/Modified - update the node in the nodes map.
|
|
kd.nodes[updatedNodeName] = node
|
|
}
|
|
}
|
|
|
|
func (kd *Discovery) getNodes() (map[string]*Node, string, error) {
|
|
res, err := kd.queryAPIServerPath(nodesURL)
|
|
if err != nil {
|
|
// If we can't list nodes then we can't watch them. Assume this is a misconfiguration
|
|
// & return error.
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes nodes: %s", err)
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes nodes; unexpected response: %d %s", res.StatusCode, res.Status)
|
|
}
|
|
|
|
var nodes NodeList
|
|
if err := json.NewDecoder(res.Body).Decode(&nodes); err != nil {
|
|
body, _ := ioutil.ReadAll(res.Body)
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes nodes; unexpected response body: %s", string(body))
|
|
}
|
|
|
|
nodeMap := map[string]*Node{}
|
|
for idx, node := range nodes.Items {
|
|
nodeMap[node.ObjectMeta.Name] = &nodes.Items[idx]
|
|
}
|
|
|
|
return nodeMap, nodes.ResourceVersion, nil
|
|
}
|
|
|
|
func (kd *Discovery) getServices() (map[string]map[string]*Service, string, error) {
|
|
res, err := kd.queryAPIServerPath(servicesURL)
|
|
if err != nil {
|
|
// If we can't list services then we can't watch them. Assume this is a misconfiguration
|
|
// & return error.
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes services: %s", err)
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes services; unexpected response: %d %s", res.StatusCode, res.Status)
|
|
}
|
|
var services ServiceList
|
|
if err := json.NewDecoder(res.Body).Decode(&services); err != nil {
|
|
body, _ := ioutil.ReadAll(res.Body)
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes services; unexpected response body: %s", string(body))
|
|
}
|
|
|
|
serviceMap := map[string]map[string]*Service{}
|
|
for idx, service := range services.Items {
|
|
namespace, ok := serviceMap[service.ObjectMeta.Namespace]
|
|
if !ok {
|
|
namespace = map[string]*Service{}
|
|
serviceMap[service.ObjectMeta.Namespace] = namespace
|
|
}
|
|
namespace[service.ObjectMeta.Name] = &services.Items[idx]
|
|
}
|
|
|
|
return serviceMap, services.ResourceVersion, nil
|
|
}
|
|
|
|
// watchNodes watches nodes as they come & go.
|
|
func (kd *Discovery) watchNodes(events chan interface{}, done <-chan struct{}, retryInterval time.Duration) {
|
|
until(func() {
|
|
nodes, resourceVersion, err := kd.getNodes()
|
|
if err != nil {
|
|
log.Errorf("Cannot initialize nodes collection: %s", err)
|
|
return
|
|
}
|
|
|
|
// Reset the known nodes.
|
|
kd.nodesMu.Lock()
|
|
kd.nodes = map[string]*Node{}
|
|
kd.nodesMu.Unlock()
|
|
|
|
for _, node := range nodes {
|
|
events <- &nodeEvent{Added, node}
|
|
}
|
|
|
|
req, err := http.NewRequest("GET", nodesURL, nil)
|
|
if err != nil {
|
|
log.Errorf("Cannot create nodes request: %s", err)
|
|
return
|
|
}
|
|
values := req.URL.Query()
|
|
values.Add("watch", "true")
|
|
values.Add("resourceVersion", resourceVersion)
|
|
req.URL.RawQuery = values.Encode()
|
|
res, err := kd.queryAPIServerReq(req)
|
|
if err != nil {
|
|
log.Errorf("Failed to watch nodes: %s", err)
|
|
return
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
log.Errorf("Failed to watch nodes: %d", res.StatusCode)
|
|
return
|
|
}
|
|
|
|
d := json.NewDecoder(res.Body)
|
|
|
|
for {
|
|
var event nodeEvent
|
|
if err := d.Decode(&event); err != nil {
|
|
log.Errorf("Watch nodes unexpectedly closed: %s", err)
|
|
return
|
|
}
|
|
|
|
select {
|
|
case events <- &event:
|
|
case <-done:
|
|
}
|
|
}
|
|
}, retryInterval, done)
|
|
}
|
|
|
|
// watchServices watches services as they come & go.
|
|
func (kd *Discovery) startServiceWatch(events chan<- interface{}, done <-chan struct{}, retryInterval time.Duration) {
|
|
until(func() {
|
|
// We use separate target groups for each discovered service so we'll need to clean up any if they've been deleted
|
|
// in Kubernetes while we couldn't connect - small chance of this, but worth dealing with.
|
|
existingServices := kd.services
|
|
|
|
// Reset the known services.
|
|
kd.servicesMu.Lock()
|
|
kd.services = map[string]map[string]*Service{}
|
|
kd.servicesMu.Unlock()
|
|
|
|
services, resourceVersion, err := kd.getServices()
|
|
if err != nil {
|
|
log.Errorf("Cannot initialize services collection: %s", err)
|
|
return
|
|
}
|
|
|
|
// Now let's loop through the old services & see if they still exist in here
|
|
for oldNSName, oldNS := range existingServices {
|
|
if ns, ok := services[oldNSName]; !ok {
|
|
for _, service := range existingServices[oldNSName] {
|
|
events <- &serviceEvent{Deleted, service}
|
|
}
|
|
} else {
|
|
for oldServiceName, oldService := range oldNS {
|
|
if _, ok := ns[oldServiceName]; !ok {
|
|
events <- &serviceEvent{Deleted, oldService}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Discard the existing services map for GC.
|
|
existingServices = nil
|
|
|
|
for _, ns := range services {
|
|
for _, service := range ns {
|
|
events <- &serviceEvent{Added, service}
|
|
}
|
|
}
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(2)
|
|
|
|
go func() {
|
|
kd.watchServices(resourceVersion, events, done)
|
|
wg.Done()
|
|
}()
|
|
go func() {
|
|
kd.watchServiceEndpoints(resourceVersion, events, done)
|
|
wg.Done()
|
|
}()
|
|
|
|
wg.Wait()
|
|
}, retryInterval, done)
|
|
}
|
|
|
|
func (kd *Discovery) watchServices(resourceVersion string, events chan<- interface{}, done <-chan struct{}) {
|
|
req, err := http.NewRequest("GET", servicesURL, nil)
|
|
if err != nil {
|
|
log.Errorf("Failed to create services request: %s", err)
|
|
return
|
|
}
|
|
values := req.URL.Query()
|
|
values.Add("watch", "true")
|
|
values.Add("resourceVersion", resourceVersion)
|
|
req.URL.RawQuery = values.Encode()
|
|
|
|
res, err := kd.queryAPIServerReq(req)
|
|
if err != nil {
|
|
log.Errorf("Failed to watch services: %s", err)
|
|
return
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
log.Errorf("Failed to watch services: %d", res.StatusCode)
|
|
return
|
|
}
|
|
|
|
d := json.NewDecoder(res.Body)
|
|
|
|
for {
|
|
var event serviceEvent
|
|
if err := d.Decode(&event); err != nil {
|
|
log.Errorf("Watch services unexpectedly closed: %s", err)
|
|
return
|
|
}
|
|
|
|
select {
|
|
case events <- &event:
|
|
case <-done:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// watchServiceEndpoints watches service endpoints as they come & go.
|
|
func (kd *Discovery) watchServiceEndpoints(resourceVersion string, events chan<- interface{}, done <-chan struct{}) {
|
|
req, err := http.NewRequest("GET", endpointsURL, nil)
|
|
if err != nil {
|
|
log.Errorf("Failed to create service endpoints request: %s", err)
|
|
return
|
|
}
|
|
values := req.URL.Query()
|
|
values.Add("watch", "true")
|
|
values.Add("resourceVersion", resourceVersion)
|
|
req.URL.RawQuery = values.Encode()
|
|
|
|
res, err := kd.queryAPIServerReq(req)
|
|
if err != nil {
|
|
log.Errorf("Failed to watch service endpoints: %s", err)
|
|
return
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
log.Errorf("Failed to watch service endpoints: %d", res.StatusCode)
|
|
return
|
|
}
|
|
|
|
d := json.NewDecoder(res.Body)
|
|
|
|
for {
|
|
var event endpointsEvent
|
|
if err := d.Decode(&event); err != nil {
|
|
log.Errorf("Watch service endpoints unexpectedly closed: %s", err)
|
|
return
|
|
}
|
|
|
|
select {
|
|
case events <- &event:
|
|
case <-done:
|
|
}
|
|
}
|
|
}
|
|
|
|
func (kd *Discovery) updateService(service *Service, eventType EventType) *config.TargetGroup {
|
|
kd.servicesMu.Lock()
|
|
defer kd.servicesMu.Unlock()
|
|
|
|
switch eventType {
|
|
case Deleted:
|
|
return kd.deleteService(service)
|
|
case Added, Modified:
|
|
return kd.addService(service)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (kd *Discovery) deleteService(service *Service) *config.TargetGroup {
|
|
tg := &config.TargetGroup{Source: serviceSource(service)}
|
|
|
|
delete(kd.services[service.ObjectMeta.Namespace], service.ObjectMeta.Name)
|
|
if len(kd.services[service.ObjectMeta.Namespace]) == 0 {
|
|
delete(kd.services, service.ObjectMeta.Namespace)
|
|
}
|
|
|
|
return tg
|
|
}
|
|
|
|
func (kd *Discovery) addService(service *Service) *config.TargetGroup {
|
|
namespace, ok := kd.services[service.ObjectMeta.Namespace]
|
|
if !ok {
|
|
namespace = map[string]*Service{}
|
|
kd.services[service.ObjectMeta.Namespace] = namespace
|
|
}
|
|
|
|
namespace[service.ObjectMeta.Name] = service
|
|
endpointURL := fmt.Sprintf(serviceEndpointsURL, service.ObjectMeta.Namespace, service.ObjectMeta.Name)
|
|
|
|
res, err := kd.queryAPIServerPath(endpointURL)
|
|
if err != nil {
|
|
log.Errorf("Error getting service endpoints: %s", err)
|
|
return nil
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
log.Errorf("Failed to get service endpoints: %d", res.StatusCode)
|
|
return nil
|
|
}
|
|
|
|
var eps Endpoints
|
|
if err := json.NewDecoder(res.Body).Decode(&eps); err != nil {
|
|
log.Errorf("Error getting service endpoints: %s", err)
|
|
return nil
|
|
}
|
|
|
|
return kd.updateServiceTargetGroup(service, &eps)
|
|
}
|
|
|
|
func (kd *Discovery) updateServiceTargetGroup(service *Service, eps *Endpoints) *config.TargetGroup {
|
|
tg := &config.TargetGroup{
|
|
Source: serviceSource(service),
|
|
Labels: model.LabelSet{
|
|
serviceNamespaceLabel: model.LabelValue(service.ObjectMeta.Namespace),
|
|
serviceNameLabel: model.LabelValue(service.ObjectMeta.Name),
|
|
},
|
|
}
|
|
|
|
for k, v := range service.ObjectMeta.Labels {
|
|
labelName := strutil.SanitizeLabelName(serviceLabelPrefix + k)
|
|
tg.Labels[model.LabelName(labelName)] = model.LabelValue(v)
|
|
}
|
|
|
|
for k, v := range service.ObjectMeta.Annotations {
|
|
labelName := strutil.SanitizeLabelName(serviceAnnotationPrefix + k)
|
|
tg.Labels[model.LabelName(labelName)] = model.LabelValue(v)
|
|
}
|
|
|
|
serviceAddress := service.ObjectMeta.Name + "." + service.ObjectMeta.Namespace + ".svc"
|
|
|
|
// Append the first TCP service port if one exists.
|
|
for _, port := range service.Spec.Ports {
|
|
if port.Protocol == ProtocolTCP {
|
|
serviceAddress += fmt.Sprintf(":%d", port.Port)
|
|
break
|
|
}
|
|
}
|
|
|
|
t := model.LabelSet{
|
|
model.AddressLabel: model.LabelValue(serviceAddress),
|
|
roleLabel: model.LabelValue("service"),
|
|
}
|
|
tg.Targets = append(tg.Targets, t)
|
|
|
|
// Now let's loop through the endpoints & add them to the target group with appropriate labels.
|
|
for _, ss := range eps.Subsets {
|
|
epPort := ss.Ports[0].Port
|
|
|
|
for _, addr := range ss.Addresses {
|
|
ipAddr := addr.IP
|
|
if len(ipAddr) == net.IPv6len {
|
|
ipAddr = "[" + ipAddr + "]"
|
|
}
|
|
address := fmt.Sprintf("%s:%d", ipAddr, epPort)
|
|
|
|
t := model.LabelSet{
|
|
model.AddressLabel: model.LabelValue(address),
|
|
roleLabel: model.LabelValue("endpoint"),
|
|
}
|
|
|
|
tg.Targets = append(tg.Targets, t)
|
|
}
|
|
}
|
|
|
|
return tg
|
|
}
|
|
|
|
func (kd *Discovery) updateServiceEndpoints(endpoints *Endpoints, eventType EventType) *config.TargetGroup {
|
|
kd.servicesMu.Lock()
|
|
defer kd.servicesMu.Unlock()
|
|
|
|
serviceNamespace := endpoints.ObjectMeta.Namespace
|
|
serviceName := endpoints.ObjectMeta.Name
|
|
|
|
if service, ok := kd.services[serviceNamespace][serviceName]; ok {
|
|
return kd.updateServiceTargetGroup(service, endpoints)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func newKubernetesHTTPClient(conf *config.KubernetesSDConfig) (*http.Client, error) {
|
|
bearerTokenFile := conf.BearerTokenFile
|
|
caFile := conf.TLSConfig.CAFile
|
|
if conf.InCluster {
|
|
if len(bearerTokenFile) == 0 {
|
|
bearerTokenFile = serviceAccountToken
|
|
}
|
|
if len(caFile) == 0 {
|
|
// With recent versions, the CA certificate is mounted as a secret
|
|
// but we need to handle older versions too. In this case, don't
|
|
// set the CAFile & the configuration will have to use InsecureSkipVerify.
|
|
if _, err := os.Stat(serviceAccountCACert); err == nil {
|
|
caFile = serviceAccountCACert
|
|
}
|
|
}
|
|
}
|
|
|
|
tlsOpts := httputil.TLSOptions{
|
|
InsecureSkipVerify: conf.TLSConfig.InsecureSkipVerify,
|
|
CAFile: caFile,
|
|
CertFile: conf.TLSConfig.CertFile,
|
|
KeyFile: conf.TLSConfig.KeyFile,
|
|
}
|
|
tlsConfig, err := httputil.NewTLSConfig(tlsOpts)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var rt http.RoundTripper = &http.Transport{
|
|
Dial: func(netw, addr string) (c net.Conn, err error) {
|
|
c, err = net.DialTimeout(netw, addr, time.Duration(conf.RequestTimeout))
|
|
return
|
|
},
|
|
TLSClientConfig: tlsConfig,
|
|
}
|
|
|
|
// If a bearer token is provided, create a round tripper that will set the
|
|
// Authorization header correctly on each request.
|
|
bearerToken := conf.BearerToken
|
|
if len(bearerToken) == 0 && len(bearerTokenFile) > 0 {
|
|
b, err := ioutil.ReadFile(bearerTokenFile)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("unable to read bearer token file %s: %s", bearerTokenFile, err)
|
|
}
|
|
bearerToken = string(b)
|
|
}
|
|
if len(bearerToken) > 0 {
|
|
rt = httputil.NewBearerAuthRoundTripper(bearerToken, rt)
|
|
}
|
|
|
|
if conf.BasicAuth != nil {
|
|
rt = httputil.NewBasicAuthRoundTripper(conf.BasicAuth.Username, conf.BasicAuth.Password, rt)
|
|
}
|
|
|
|
return &http.Client{
|
|
Transport: rt,
|
|
}, nil
|
|
}
|
|
|
|
func serviceSource(service *Service) string {
|
|
return sourceServicePrefix + ":" + service.ObjectMeta.Namespace + "/" + service.ObjectMeta.Name
|
|
}
|
|
|
|
// Until loops until stop channel is closed, running f every period.
|
|
// f may not be invoked if stop channel is already closed.
|
|
func until(f func(), period time.Duration, stopCh <-chan struct{}) {
|
|
select {
|
|
case <-stopCh:
|
|
return
|
|
default:
|
|
f()
|
|
}
|
|
for {
|
|
select {
|
|
case <-stopCh:
|
|
return
|
|
case <-time.After(period):
|
|
f()
|
|
}
|
|
}
|
|
}
|
|
|
|
// nodeAddresses returns the provided node's address, based on the priority:
|
|
// 1. NodeInternalIP
|
|
// 2. NodeExternalIP
|
|
// 3. NodeLegacyHostIP
|
|
//
|
|
// Copied from k8s.io/kubernetes/pkg/util/node/node.go
|
|
func nodeAddresses(node *Node) (net.IP, map[NodeAddressType][]net.IP, error) {
|
|
addresses := node.Status.Addresses
|
|
addressMap := map[NodeAddressType][]net.IP{}
|
|
for _, addr := range addresses {
|
|
ip := net.ParseIP(addr.Address)
|
|
// All addresses should be valid IPs.
|
|
if ip == nil {
|
|
continue
|
|
}
|
|
addressMap[addr.Type] = append(addressMap[addr.Type], ip)
|
|
}
|
|
if addresses, ok := addressMap[NodeInternalIP]; ok {
|
|
return addresses[0], addressMap, nil
|
|
}
|
|
if addresses, ok := addressMap[NodeExternalIP]; ok {
|
|
return addresses[0], addressMap, nil
|
|
}
|
|
if addresses, ok := addressMap[NodeLegacyHostIP]; ok {
|
|
return addresses[0], addressMap, nil
|
|
}
|
|
return nil, nil, fmt.Errorf("host IP unknown; known addresses: %v", addresses)
|
|
}
|
|
|
|
func (kd *Discovery) updatePod(pod *Pod, eventType EventType) {
|
|
kd.podsMu.Lock()
|
|
defer kd.podsMu.Unlock()
|
|
|
|
switch eventType {
|
|
case Deleted:
|
|
if _, ok := kd.pods[pod.ObjectMeta.Namespace]; ok {
|
|
delete(kd.pods[pod.ObjectMeta.Namespace], pod.ObjectMeta.Name)
|
|
if len(kd.pods[pod.ObjectMeta.Namespace]) == 0 {
|
|
delete(kd.pods, pod.ObjectMeta.Namespace)
|
|
}
|
|
}
|
|
case Added, Modified:
|
|
if _, ok := kd.pods[pod.ObjectMeta.Namespace]; !ok {
|
|
kd.pods[pod.ObjectMeta.Namespace] = map[string]*Pod{}
|
|
}
|
|
kd.pods[pod.ObjectMeta.Namespace][pod.ObjectMeta.Name] = pod
|
|
}
|
|
}
|
|
|
|
func (kd *Discovery) getPods() (map[string]map[string]*Pod, string, error) {
|
|
res, err := kd.queryAPIServerPath(podsURL)
|
|
if err != nil {
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes pods: %s", err)
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes pods; unexpected response: %d %s", res.StatusCode, res.Status)
|
|
}
|
|
|
|
var pods PodList
|
|
if err := json.NewDecoder(res.Body).Decode(&pods); err != nil {
|
|
body, _ := ioutil.ReadAll(res.Body)
|
|
return nil, "", fmt.Errorf("unable to list Kubernetes pods; unexpected response body: %s", string(body))
|
|
}
|
|
|
|
podMap := map[string]map[string]*Pod{}
|
|
for idx, pod := range pods.Items {
|
|
if _, ok := podMap[pod.ObjectMeta.Namespace]; !ok {
|
|
podMap[pod.ObjectMeta.Namespace] = map[string]*Pod{}
|
|
}
|
|
log.Debugf("Got pod %s in namespace %s", pod.ObjectMeta.Name, pod.ObjectMeta.Namespace)
|
|
podMap[pod.ObjectMeta.Namespace][pod.ObjectMeta.Name] = &pods.Items[idx]
|
|
}
|
|
|
|
return podMap, pods.ResourceVersion, nil
|
|
}
|
|
|
|
func (kd *Discovery) watchPods(events chan interface{}, done <-chan struct{}, retryInterval time.Duration) {
|
|
until(func() {
|
|
pods, resourceVersion, err := kd.getPods()
|
|
if err != nil {
|
|
log.Errorf("Cannot initialize pods collection: %s", err)
|
|
return
|
|
}
|
|
kd.podsMu.Lock()
|
|
kd.pods = pods
|
|
kd.podsMu.Unlock()
|
|
|
|
req, err := http.NewRequest("GET", podsURL, nil)
|
|
if err != nil {
|
|
log.Errorf("Cannot create pods request: %s", err)
|
|
return
|
|
}
|
|
|
|
values := req.URL.Query()
|
|
values.Add("watch", "true")
|
|
values.Add("resourceVersion", resourceVersion)
|
|
req.URL.RawQuery = values.Encode()
|
|
res, err := kd.queryAPIServerReq(req)
|
|
if err != nil {
|
|
log.Errorf("Failed to watch pods: %s", err)
|
|
return
|
|
}
|
|
defer res.Body.Close()
|
|
if res.StatusCode != http.StatusOK {
|
|
log.Errorf("Failed to watch pods: %d", res.StatusCode)
|
|
return
|
|
}
|
|
|
|
d := json.NewDecoder(res.Body)
|
|
|
|
for {
|
|
var event podEvent
|
|
if err := d.Decode(&event); err != nil {
|
|
log.Errorf("Watch pods unexpectedly closed: %s", err)
|
|
return
|
|
}
|
|
|
|
select {
|
|
case events <- &event:
|
|
case <-done:
|
|
}
|
|
}
|
|
}, retryInterval, done)
|
|
}
|
|
|
|
func podSource(pod *Pod) string {
|
|
return sourcePodPrefix + ":" + pod.ObjectMeta.Namespace + ":" + pod.ObjectMeta.Name
|
|
}
|
|
|
|
type ByContainerPort []ContainerPort
|
|
|
|
func (a ByContainerPort) Len() int { return len(a) }
|
|
func (a ByContainerPort) Less(i, j int) bool { return a[i].ContainerPort < a[j].ContainerPort }
|
|
func (a ByContainerPort) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
|
|
type ByContainerName []Container
|
|
|
|
func (a ByContainerName) Len() int { return len(a) }
|
|
func (a ByContainerName) Less(i, j int) bool { return a[i].Name < a[j].Name }
|
|
func (a ByContainerName) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
|
|
|
|
func updatePodTargets(pod *Pod, allContainers bool) []model.LabelSet {
|
|
var targets []model.LabelSet = make([]model.LabelSet, 0, len(pod.PodSpec.Containers))
|
|
if pod.PodStatus.PodIP == "" {
|
|
log.Debugf("skipping pod %s -- PodStatus.PodIP is empty", pod.ObjectMeta.Name)
|
|
return targets
|
|
}
|
|
|
|
if pod.PodStatus.Phase != "Running" {
|
|
log.Debugf("skipping pod %s -- status is not `Running`", pod.ObjectMeta.Name)
|
|
return targets
|
|
}
|
|
|
|
ready := "unknown"
|
|
for _, cond := range pod.PodStatus.Conditions {
|
|
if strings.ToLower(cond.Type) == "ready" {
|
|
ready = strings.ToLower(cond.Status)
|
|
}
|
|
}
|
|
|
|
sort.Sort(ByContainerName(pod.PodSpec.Containers))
|
|
|
|
for _, container := range pod.PodSpec.Containers {
|
|
// Collect a list of TCP ports
|
|
// Sort by port number, ascending
|
|
// Product a target pointed at the first port
|
|
// Include a label containing all ports (portName=port,PortName=port,...,)
|
|
var tcpPorts []ContainerPort
|
|
var portLabel *bytes.Buffer = bytes.NewBufferString(",")
|
|
|
|
for _, port := range container.Ports {
|
|
if port.Protocol == "TCP" {
|
|
tcpPorts = append(tcpPorts, port)
|
|
}
|
|
}
|
|
|
|
if len(tcpPorts) == 0 {
|
|
log.Debugf("skipping container %s with no TCP ports", container.Name)
|
|
continue
|
|
}
|
|
|
|
sort.Sort(ByContainerPort(tcpPorts))
|
|
|
|
t := model.LabelSet{
|
|
model.AddressLabel: model.LabelValue(net.JoinHostPort(pod.PodIP, strconv.FormatInt(int64(tcpPorts[0].ContainerPort), 10))),
|
|
podNameLabel: model.LabelValue(pod.ObjectMeta.Name),
|
|
podAddressLabel: model.LabelValue(pod.PodStatus.PodIP),
|
|
podNamespaceLabel: model.LabelValue(pod.ObjectMeta.Namespace),
|
|
podContainerNameLabel: model.LabelValue(container.Name),
|
|
podContainerPortNameLabel: model.LabelValue(tcpPorts[0].Name),
|
|
podReadyLabel: model.LabelValue(ready),
|
|
}
|
|
|
|
for _, port := range tcpPorts {
|
|
portLabel.WriteString(port.Name)
|
|
portLabel.WriteString("=")
|
|
portLabel.WriteString(strconv.FormatInt(int64(port.ContainerPort), 10))
|
|
portLabel.WriteString(",")
|
|
t[model.LabelName(podContainerPortMapPrefix+port.Name)] = model.LabelValue(strconv.FormatInt(int64(port.ContainerPort), 10))
|
|
}
|
|
|
|
t[model.LabelName(podContainerPortListLabel)] = model.LabelValue(portLabel.String())
|
|
|
|
for k, v := range pod.ObjectMeta.Labels {
|
|
labelName := strutil.SanitizeLabelName(podLabelPrefix + k)
|
|
t[model.LabelName(labelName)] = model.LabelValue(v)
|
|
}
|
|
|
|
for k, v := range pod.ObjectMeta.Annotations {
|
|
labelName := strutil.SanitizeLabelName(podAnnotationPrefix + k)
|
|
t[model.LabelName(labelName)] = model.LabelValue(v)
|
|
}
|
|
|
|
targets = append(targets, t)
|
|
|
|
if !allContainers {
|
|
break
|
|
}
|
|
}
|
|
|
|
if len(targets) == 0 {
|
|
log.Debugf("no targets for pod %s", pod.ObjectMeta.Name)
|
|
}
|
|
|
|
return targets
|
|
}
|
|
|
|
func (kd *Discovery) updatePodTargetGroup(pod *Pod) *config.TargetGroup {
|
|
kd.podsMu.RLock()
|
|
defer kd.podsMu.RUnlock()
|
|
|
|
tg := &config.TargetGroup{
|
|
Source: podSource(pod),
|
|
}
|
|
|
|
// If this pod doesn't exist, return an empty target group
|
|
if _, ok := kd.pods[pod.ObjectMeta.Namespace]; !ok {
|
|
return tg
|
|
}
|
|
if _, ok := kd.pods[pod.ObjectMeta.Namespace][pod.ObjectMeta.Name]; !ok {
|
|
return tg
|
|
}
|
|
|
|
tg.Labels = model.LabelSet{
|
|
roleLabel: model.LabelValue("container"),
|
|
}
|
|
tg.Targets = updatePodTargets(pod, true)
|
|
|
|
return tg
|
|
}
|
|
|
|
func (kd *Discovery) updatePodsTargetGroup() *config.TargetGroup {
|
|
tg := &config.TargetGroup{
|
|
Source: podsTargetGroupName,
|
|
Labels: model.LabelSet{
|
|
roleLabel: model.LabelValue("pod"),
|
|
},
|
|
}
|
|
|
|
for _, namespace := range kd.pods {
|
|
for _, pod := range namespace {
|
|
tg.Targets = append(tg.Targets, updatePodTargets(pod, false)...)
|
|
}
|
|
}
|
|
|
|
return tg
|
|
}
|