mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2025-02-21 03:16:00 -08:00
Code Issues Actions 5 Packages Projects Releases Wiki Activity

Make scraping offset consistent.

Browse source 
To evenly distribute scraping load we currently rely on random
jittering. This commit hashes over the target's identity and calculates
a consistent offset. This also ensures that scrape intervals
are constantly spaced between config/target changes.
This commit is contained in:
Fabian Reinartz 2016-02-15 15:22:57 +01:00
parent 938ebe78c2
commit fe7e91e2eb
3 changed files with 123 additions and 56 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
retrieval/target.go
View file

@ -18,7 +18,6 @@ import (
"fmt" "fmt"
"io" "io"
"io/ioutil" "io/ioutil"
"math/rand"
"net/http" "net/http"
"net/url" "net/url"
"strings" "strings"
@ -267,6 +266,30 @@ func (t *Target) String() string {
return t.host() return t.host()
} }
// fingerprint returns an identifying hash for the target.
func (t *Target) fingerprint() model.Fingerprint {
t.RLock()
defer t.RUnlock()
return t.labels.Fingerprint()
}
// offset returns the time until the next scrape cycle for the target.
func (t *Target) offset(interval time.Duration) time.Duration {
now := time.Now().UnixNano()
var (
base = now % int64(interval)
offset = uint64(t.fingerprint()) % uint64(interval)
next = base + int64(offset)
)
if next > int64(interval) {
next -= int64(interval)
}
return time.Duration(next)
}
func (t *Target) client() (*http.Client, error) { func (t *Target) client() (*http.Client, error) {
t.RLock() t.RLock()
defer t.RUnlock() defer t.RUnlock()
@ -366,14 +389,12 @@ func (t *Target) RunScraper(sampleAppender storage.SampleAppender) {
log.Debugf("Starting scraper for target %v...", t) log.Debugf("Starting scraper for target %v...", t)
jitterTimer := time.NewTimer(time.Duration(float64(lastScrapeInterval) * rand.Float64()))
select { select {
case <-jitterTimer.C: case <-time.After(t.offset(lastScrapeInterval)):
// Continue after scraping offset.
case <-t.scraperStopping: case <-t.scraperStopping:
jitterTimer.Stop()
return return
} }
jitterTimer.Stop()
ticker := time.NewTicker(lastScrapeInterval) ticker := time.NewTicker(lastScrapeInterval)
defer ticker.Stop() defer ticker.Stop()

46
retrieval/target_test.go
View file

@ -45,6 +45,52 @@ func TestTargetLabels(t *testing.T) {
} }
} }
func TestTargetOffset(t *testing.T) {
interval := 10 * time.Second
offsets := make([]time.Duration, 10000)
// Calculate offsets for 10000 different targets.
for i := range offsets {
target := newTestTarget("example.com:80", 0, model.LabelSet{
"label": model.LabelValue(fmt.Sprintf("%d", i)),
})
offsets[i] = target.offset(interval)
}
// Put the offsets into buckets and validate that they are all
// within bounds.
bucketSize := 1 * time.Second
buckets := make([]int, interval/bucketSize)
for _, offset := range offsets {
if offset < 0 || offset >= interval {
t.Fatalf("Offset %v out of bounds", offset)
}
bucket := offset / bucketSize
buckets[bucket]++
}
t.Log(buckets)
// Calculate whether the the number of targets per bucket
// does not differ more than a given tolerance.
avg := len(offsets) / len(buckets)
tolerance := 0.15
for _, bucket := range buckets {
diff := bucket - avg
if diff < 0 {
diff = -diff
}
if float64(diff)/float64(avg) > tolerance {
t.Fatalf("Bucket out of tolerance bounds")
}
}
}
func TestOverwriteLabels(t *testing.T) { func TestOverwriteLabels(t *testing.T) {
type test struct { type test struct {
metric string metric string

102
web/ui/bindata.go
View file

@ -791,38 +791,38 @@ func AssetNames() []string {
// _bindata is a table, holding each asset generator, mapped to its name. // _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){ var _bindata = map[string]func() (*asset, error){
"web/ui/templates/_base.html": webUiTemplates_baseHtml, "web/ui/templates/_base.html": webUiTemplates_baseHtml,
"web/ui/templates/alerts.html": webUiTemplatesAlertsHtml, "web/ui/templates/alerts.html": webUiTemplatesAlertsHtml,
"web/ui/templates/graph.html": webUiTemplatesGraphHtml, "web/ui/templates/graph.html": webUiTemplatesGraphHtml,
"web/ui/templates/status.html": webUiTemplatesStatusHtml, "web/ui/templates/status.html": webUiTemplatesStatusHtml,
"web/ui/static/css/alerts.css": webUiStaticCssAlertsCss, "web/ui/static/css/alerts.css": webUiStaticCssAlertsCss,
"web/ui/static/css/graph.css": webUiStaticCssGraphCss, "web/ui/static/css/graph.css": webUiStaticCssGraphCss,
"web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss, "web/ui/static/css/prom_console.css": webUiStaticCssProm_consoleCss,
"web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss, "web/ui/static/css/prometheus.css": webUiStaticCssPrometheusCss,
"web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif, "web/ui/static/img/ajax-loader.gif": webUiStaticImgAjaxLoaderGif,
"web/ui/static/js/alerts.js": webUiStaticJsAlertsJs, "web/ui/static/js/alerts.js": webUiStaticJsAlertsJs,
"web/ui/static/js/graph.js": webUiStaticJsGraphJs, "web/ui/static/js/graph.js": webUiStaticJsGraphJs,
"web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar, "web/ui/static/js/graph_template.handlebar": webUiStaticJsGraph_templateHandlebar,
"web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs, "web/ui/static/js/prom_console.js": webUiStaticJsProm_consoleJs,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap-theme.min.css": webUiStaticVendorBootstrap331CssBootstrapThemeMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss, "web/ui/static/vendor/bootstrap-3.3.1/css/bootstrap.min.css": webUiStaticVendorBootstrap331CssBootstrapMinCss,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.eot": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.svg": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.ttf": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf,
"web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, "web/ui/static/vendor/bootstrap-3.3.1/fonts/glyphicons-halflings-regular.woff": webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff,
"web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs, "web/ui/static/vendor/bootstrap-3.3.1/js/bootstrap.min.js": webUiStaticVendorBootstrap331JsBootstrapMinJs,
"web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs, "web/ui/static/vendor/bootstrap-3.3.1/js/npm.js": webUiStaticVendorBootstrap331JsNpmJs,
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, "web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.js": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs,
"web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.min.css": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, "web/ui/static/vendor/bootstrap-datetimepicker/bootstrap-datetimepicker.min.css": webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss,
"web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, "web/ui/static/vendor/bootstrap3-typeahead/bootstrap3-typeahead.min.js": webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs,
"web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs, "web/ui/static/vendor/js/handlebars.js": webUiStaticVendorJsHandlebarsJs,
"web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs, "web/ui/static/vendor/js/jquery.hotkeys.js": webUiStaticVendorJsJqueryHotkeysJs,
"web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs, "web/ui/static/vendor/js/jquery.min.js": webUiStaticVendorJsJqueryMinJs,
"web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs, "web/ui/static/vendor/js/jquery.selection.js": webUiStaticVendorJsJquerySelectionJs,
"web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss, "web/ui/static/vendor/rickshaw/rickshaw.min.css": webUiStaticVendorRickshawRickshawMinCss,
"web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs, "web/ui/static/vendor/rickshaw/rickshaw.min.js": webUiStaticVendorRickshawRickshawMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs, "web/ui/static/vendor/rickshaw/vendor/d3.layout.min.js": webUiStaticVendorRickshawVendorD3LayoutMinJs,
"web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js, "web/ui/static/vendor/rickshaw/vendor/d3.v3.js": webUiStaticVendorRickshawVendorD3V3Js,
} }
// AssetDir returns the file names below a certain // AssetDir returns the file names below a certain
@ -864,69 +864,70 @@ type bintree struct {
Func func() (*asset, error) Func func() (*asset, error)
Children map[string]*bintree Children map[string]*bintree
} }
var _bintree = &bintree{nil, map[string]*bintree{ var _bintree = &bintree{nil, map[string]*bintree{
"web": &bintree{nil, map[string]*bintree{ "web": &bintree{nil, map[string]*bintree{
"ui": &bintree{nil, map[string]*bintree{ "ui": &bintree{nil, map[string]*bintree{
"static": &bintree{nil, map[string]*bintree{ "static": &bintree{nil, map[string]*bintree{
"css": &bintree{nil, map[string]*bintree{ "css": &bintree{nil, map[string]*bintree{
"alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}}, "alerts.css": &bintree{webUiStaticCssAlertsCss, map[string]*bintree{}},
"graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}}, "graph.css": &bintree{webUiStaticCssGraphCss, map[string]*bintree{}},
"prom_console.css": &bintree{webUiStaticCssProm_consoleCss, map[string]*bintree{}}, "prom_console.css": &bintree{webUiStaticCssProm_consoleCss, map[string]*bintree{}},
"prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}}, "prometheus.css": &bintree{webUiStaticCssPrometheusCss, map[string]*bintree{}},
}}, }},
"img": &bintree{nil, map[string]*bintree{ "img": &bintree{nil, map[string]*bintree{
"ajax-loader.gif": &bintree{webUiStaticImgAjaxLoaderGif, map[string]*bintree{}}, "ajax-loader.gif": &bintree{webUiStaticImgAjaxLoaderGif, map[string]*bintree{}},
}}, }},
"js": &bintree{nil, map[string]*bintree{ "js": &bintree{nil, map[string]*bintree{
"alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}}, "alerts.js": &bintree{webUiStaticJsAlertsJs, map[string]*bintree{}},
"graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}}, "graph.js": &bintree{webUiStaticJsGraphJs, map[string]*bintree{}},
"graph_template.handlebar": &bintree{webUiStaticJsGraph_templateHandlebar, map[string]*bintree{}}, "graph_template.handlebar": &bintree{webUiStaticJsGraph_templateHandlebar, map[string]*bintree{}},
"prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}}, "prom_console.js": &bintree{webUiStaticJsProm_consoleJs, map[string]*bintree{}},
}}, }},
"vendor": &bintree{nil, map[string]*bintree{ "vendor": &bintree{nil, map[string]*bintree{
"bootstrap-3.3.1": &bintree{nil, map[string]*bintree{ "bootstrap-3.3.1": &bintree{nil, map[string]*bintree{
"css": &bintree{nil, map[string]*bintree{ "css": &bintree{nil, map[string]*bintree{
"bootstrap-theme.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, map[string]*bintree{}}, "bootstrap-theme.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapThemeMinCss, map[string]*bintree{}},
"bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}}, "bootstrap.min.css": &bintree{webUiStaticVendorBootstrap331CssBootstrapMinCss, map[string]*bintree{}},
}}, }},
"fonts": &bintree{nil, map[string]*bintree{ "fonts": &bintree{nil, map[string]*bintree{
"glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}}, "glyphicons-halflings-regular.eot": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularEot, map[string]*bintree{}},
"glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}}, "glyphicons-halflings-regular.svg": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularSvg, map[string]*bintree{}},
"glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}}, "glyphicons-halflings-regular.ttf": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularTtf, map[string]*bintree{}},
"glyphicons-halflings-regular.woff": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, map[string]*bintree{}}, "glyphicons-halflings-regular.woff": &bintree{webUiStaticVendorBootstrap331FontsGlyphiconsHalflingsRegularWoff, map[string]*bintree{}},
}}, }},
"js": &bintree{nil, map[string]*bintree{ "js": &bintree{nil, map[string]*bintree{
"bootstrap.min.js": &bintree{webUiStaticVendorBootstrap331JsBootstrapMinJs, map[string]*bintree{}}, "bootstrap.min.js": &bintree{webUiStaticVendorBootstrap331JsBootstrapMinJs, map[string]*bintree{}},
"npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}}, "npm.js": &bintree{webUiStaticVendorBootstrap331JsNpmJs, map[string]*bintree{}},
}}, }},
}}, }},
"bootstrap-datetimepicker": &bintree{nil, map[string]*bintree{ "bootstrap-datetimepicker": &bintree{nil, map[string]*bintree{
"bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}}, "bootstrap-datetimepicker.js": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerJs, map[string]*bintree{}},
"bootstrap-datetimepicker.min.css": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, map[string]*bintree{}}, "bootstrap-datetimepicker.min.css": &bintree{webUiStaticVendorBootstrapDatetimepickerBootstrapDatetimepickerMinCss, map[string]*bintree{}},
}}, }},
"bootstrap3-typeahead": &bintree{nil, map[string]*bintree{ "bootstrap3-typeahead": &bintree{nil, map[string]*bintree{
"bootstrap3-typeahead.min.js": &bintree{webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, map[string]*bintree{}}, "bootstrap3-typeahead.min.js": &bintree{webUiStaticVendorBootstrap3TypeaheadBootstrap3TypeaheadMinJs, map[string]*bintree{}},
}}, }},
"js": &bintree{nil, map[string]*bintree{ "js": &bintree{nil, map[string]*bintree{
"handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}}, "handlebars.js": &bintree{webUiStaticVendorJsHandlebarsJs, map[string]*bintree{}},
"jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}}, "jquery.hotkeys.js": &bintree{webUiStaticVendorJsJqueryHotkeysJs, map[string]*bintree{}},
"jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}}, "jquery.min.js": &bintree{webUiStaticVendorJsJqueryMinJs, map[string]*bintree{}},
"jquery.selection.js": &bintree{webUiStaticVendorJsJquerySelectionJs, map[string]*bintree{}}, "jquery.selection.js": &bintree{webUiStaticVendorJsJquerySelectionJs, map[string]*bintree{}},
}}, }},
"rickshaw": &bintree{nil, map[string]*bintree{ "rickshaw": &bintree{nil, map[string]*bintree{
"rickshaw.min.css": &bintree{webUiStaticVendorRickshawRickshawMinCss, map[string]*bintree{}}, "rickshaw.min.css": &bintree{webUiStaticVendorRickshawRickshawMinCss, map[string]*bintree{}},
"rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}}, "rickshaw.min.js": &bintree{webUiStaticVendorRickshawRickshawMinJs, map[string]*bintree{}},
"vendor": &bintree{nil, map[string]*bintree{ "vendor": &bintree{nil, map[string]*bintree{
"d3.layout.min.js": &bintree{webUiStaticVendorRickshawVendorD3LayoutMinJs, map[string]*bintree{}}, "d3.layout.min.js": &bintree{webUiStaticVendorRickshawVendorD3LayoutMinJs, map[string]*bintree{}},
"d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}}, "d3.v3.js": &bintree{webUiStaticVendorRickshawVendorD3V3Js, map[string]*bintree{}},
}}, }},
}}, }},
}}, }},
}}, }},
"templates": &bintree{nil, map[string]*bintree{ "templates": &bintree{nil, map[string]*bintree{
"_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}}, "_base.html": &bintree{webUiTemplates_baseHtml, map[string]*bintree{}},
"alerts.html": &bintree{webUiTemplatesAlertsHtml, map[string]*bintree{}}, "alerts.html": &bintree{webUiTemplatesAlertsHtml, map[string]*bintree{}},
"graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}}, "graph.html": &bintree{webUiTemplatesGraphHtml, map[string]*bintree{}},
"status.html": &bintree{webUiTemplatesStatusHtml, map[string]*bintree{}}, "status.html": &bintree{webUiTemplatesStatusHtml, map[string]*bintree{}},
}}, }},
}}, }},
@ -979,4 +980,3 @@ func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1) cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
} }