prometheus/retrieval/target.go

// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package retrieval

import (
	"encoding/json"
	"fmt"
	"github.com/matttproud/golang_instrumentation/metrics"
	"github.com/matttproud/prometheus/model"
	"io/ioutil"
	"log"
	"math"
	"net/http"
	"strconv"
	"time"
)

type TargetState int

const (
	UNKNOWN TargetState = iota
	ALIVE
	UNREACHABLE
)

const (
	MAXIMUM_BACKOFF = time.Minute * 30
)

type Target struct {
	scheduledFor     time.Time
	unreachableCount int
	state            TargetState

	Address  string
	Deadline time.Duration

	// XXX: Move this to a field with the target manager initialization instead of here.
	Interval time.Duration
}

type Result struct {
	Err     error
	Samples []model.Sample
	Target  Target
}

func (t *Target) reschedule(s TargetState) {
	currentState := t.state

	switch currentState {
	case UNKNOWN, UNREACHABLE:
		switch s {
		case ALIVE:
			t.unreachableCount = 0
			targetsHealthy.Increment()
		case UNREACHABLE:
			backoff := MAXIMUM_BACKOFF
			exponential := time.Duration(math.Pow(2, float64(t.unreachableCount))) * time.Second
			if backoff > exponential {
				backoff = exponential
			}

			t.scheduledFor = time.Now().Add(backoff)
			t.unreachableCount++
			log.Printf("%s unavailable %s times deferred for %s.", t, t.unreachableCount, backoff)
		default:
		}
	case ALIVE:
		switch s {
		case UNREACHABLE:
			t.unreachableCount++
			targetsUnhealthy.Increment()
		}
	default:
	}

	if s != currentState {
		log.Printf("%s transitioning from %s to %s.", t, currentState, s)
	}

	t.state = s
}

func (t *Target) Scrape(results chan Result) (err error) {
	result := Result{}

	defer func() {
		futureState := t.state

		switch err {
		case nil:
			futureState = ALIVE
		default:
			futureState = UNREACHABLE
		}

		t.reschedule(futureState)

		result.Err = err
		results <- result
	}()

	done := make(chan bool)

	request := func() {
		ti := time.Now()
		resp, err := http.Get(t.Address)
		if err != nil {
			return
		}

		defer resp.Body.Close()

		raw, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return
		}

		intermediate := make(map[string]interface{})
		err = json.Unmarshal(raw, &intermediate)
		if err != nil {
			return
		}

		baseLabels := map[string]string{"instance": t.Address}

		for name, v := range intermediate {
			asMap, ok := v.(map[string]interface{})

			if !ok {
				continue
			}

			switch asMap["type"] {
			case "counter":
				m := model.Metric{}
				m["name"] = model.LabelValue(name)
				asFloat, ok := asMap["value"].(float64)
				if !ok {
					continue
				}

				s := model.Sample{
					Metric:    m,
					Value:     model.SampleValue(asFloat),
					Timestamp: ti,
				}

				for baseK, baseV := range baseLabels {
					m[model.LabelName(baseK)] = model.LabelValue(baseV)
				}

				result.Samples = append(result.Samples, s)
			case "histogram":
				values, ok := asMap["value"].(map[string]interface{})
				if !ok {
					continue
				}

				for p, pValue := range values {
					asString, ok := pValue.(string)
					if !ok {
						continue
					}

					float, err := strconv.ParseFloat(asString, 64)
					if err != nil {
						continue
					}

					m := model.Metric{}
					m["name"] = model.LabelValue(name)
					m["percentile"] = model.LabelValue(p)

					s := model.Sample{
						Metric:    m,
						Value:     model.SampleValue(float),
						Timestamp: ti,
					}

					for baseK, baseV := range baseLabels {
						m[model.LabelName(baseK)] = model.LabelValue(baseV)
					}

					result.Samples = append(result.Samples, s)
				}
			}
		}

		done <- true
	}

	accumulator := func(d time.Duration) {
		ms := float64(d) / float64(time.Millisecond)
		if err == nil {
			scrapeLatencyHealthy.Add(ms)
		} else {
			scrapeLatencyUnhealthy.Add(ms)
		}
	}

	go metrics.InstrumentCall(request, accumulator)

	select {
	case <-done:
		break
	case <-time.After(t.Deadline):
		err = fmt.Errorf("Target %s exceeded %s deadline.", t, t.Deadline)
	}

	return
}