diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index 1f3b26468..4e05657a7 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -29,23 +29,23 @@ }, { "ImportPath": "github.com/prometheus/client_golang/extraction", - "Comment": "0.4.0-1-g692492e", - "Rev": "692492e54b553a81013254cc1fba4b6dd76fad30" + "Comment": "0.5.0", + "Rev": "b0bd7e1be33327b85cb4853e7011156e3cedd657" }, { "ImportPath": "github.com/prometheus/client_golang/model", - "Comment": "0.4.0-1-g692492e", - "Rev": "692492e54b553a81013254cc1fba4b6dd76fad30" + "Comment": "0.5.0", + "Rev": "b0bd7e1be33327b85cb4853e7011156e3cedd657" }, { "ImportPath": "github.com/prometheus/client_golang/prometheus", - "Comment": "0.4.0-1-g692492e", - "Rev": "692492e54b553a81013254cc1fba4b6dd76fad30" + "Comment": "0.5.0", + "Rev": "b0bd7e1be33327b85cb4853e7011156e3cedd657" }, { "ImportPath": "github.com/prometheus/client_golang/text", - "Comment": "0.4.0-1-g692492e", - "Rev": "692492e54b553a81013254cc1fba4b6dd76fad30" + "Comment": "0.5.0", + "Rev": "b0bd7e1be33327b85cb4853e7011156e3cedd657" }, { "ImportPath": "github.com/prometheus/client_model/go", diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/labelname.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/labelname.go index 75b2e79da..a283132c5 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/labelname.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/labelname.go @@ -26,14 +26,30 @@ const ( // timeseries. MetricNameLabel LabelName = "__name__" + // AddressLabel is the name of the label that holds the address of + // a scrape target. + AddressLabel LabelName = "__address__" + + // MetricsPathLabel is the name of the label that holds the path on which to + // scrape a target. + MetricsPathLabel LabelName = "__metrics_path__" + // ReservedLabelPrefix is a prefix which is not legal in user-supplied // label names. ReservedLabelPrefix = "__" + // MetaLabelPrefix is a prefix for labels that provide meta information. + // Labels with this prefix are used for intermediate label processing and + // will not be attached to time series. + MetaLabelPrefix = "__meta_" + // JobLabel is the label name indicating the job from which a timeseries // was scraped. JobLabel LabelName = "job" + // InstanceLabel is the label name used for the instance label. + InstanceLabel LabelName = "instance" + // BucketLabel is used for the label that defines the upper bound of a // bucket of a histogram ("le" -> "less or equal"). BucketLabel = "le" diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go index 32f9d7fbc..0870f2368 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric.go @@ -26,14 +26,68 @@ var separator = []byte{0} // a singleton and refers to one and only one stream of samples. type Metric map[LabelName]LabelValue -// Equal compares the fingerprints of both metrics. +// Equal compares the metrics. func (m Metric) Equal(o Metric) bool { - return m.Fingerprint().Equal(o.Fingerprint()) + if len(m) != len(o) { + return false + } + for ln, lv := range m { + olv, ok := o[ln] + if !ok { + return false + } + if olv != lv { + return false + } + } + return true } -// Before compares the fingerprints of both metrics. +// Before compares the metrics, using the following criteria: +// +// If m has fewer labels than o, it is before o. If it has more, it is not. +// +// If the number of labels is the same, the superset of all label names is +// sorted alphanumerically. The first differing label pair found in that order +// determines the outcome: If the label does not exist at all in m, then m is +// before o, and vice versa. Otherwise the label value is compared +// alphanumerically. +// +// If m and o are equal, the method returns false. func (m Metric) Before(o Metric) bool { - return m.Fingerprint().Less(o.Fingerprint()) + if len(m) < len(o) { + return true + } + if len(m) > len(o) { + return false + } + + lns := make(LabelNames, 0, len(m)+len(o)) + for ln := range m { + lns = append(lns, ln) + } + for ln := range o { + lns = append(lns, ln) + } + // It's probably not worth it to de-dup lns. + sort.Sort(lns) + for _, ln := range lns { + mlv, ok := m[ln] + if !ok { + return true + } + olv, ok := o[ln] + if !ok { + return false + } + if mlv < olv { + return true + } + if mlv > olv { + return false + } + } + return false } // String implements Stringer. @@ -67,6 +121,12 @@ func (m Metric) Fingerprint() Fingerprint { return metricToFingerprint(m) } +// Fingerprint returns a Metric's Fingerprint calculated by a faster hashing +// algorithm, which is, however, more susceptible to hash collisions. +func (m Metric) FastFingerprint() Fingerprint { + return metricToFastFingerprint(m) +} + // Clone returns a copy of the Metric. func (m Metric) Clone() Metric { clone := Metric{} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go index d51b1842f..5dbc02376 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/metric_test.go @@ -17,12 +17,14 @@ import "testing" func testMetric(t testing.TB) { var scenarios = []struct { - input Metric - fingerprint Fingerprint + input Metric + fingerprint Fingerprint + fastFingerprint Fingerprint }{ { - input: Metric{}, - fingerprint: 14695981039346656037, + input: Metric{}, + fingerprint: 14695981039346656037, + fastFingerprint: 14695981039346656037, }, { input: Metric{ @@ -30,27 +32,31 @@ func testMetric(t testing.TB) { "occupation": "robot", "manufacturer": "westinghouse", }, - fingerprint: 11310079640881077873, + fingerprint: 5911716720268894962, + fastFingerprint: 11310079640881077873, }, { input: Metric{ "x": "y", }, - fingerprint: 13948396922932177635, + fingerprint: 8241431561484471700, + fastFingerprint: 13948396922932177635, }, { input: Metric{ "a": "bb", "b": "c", }, - fingerprint: 3198632812309449502, + fingerprint: 3016285359649981711, + fastFingerprint: 3198632812309449502, }, { input: Metric{ "a": "b", "bb": "c", }, - fingerprint: 5774953389407657638, + fingerprint: 7122421792099404749, + fastFingerprint: 5774953389407657638, }, } @@ -58,6 +64,9 @@ func testMetric(t testing.TB) { if scenario.fingerprint != scenario.input.Fingerprint() { t.Errorf("%d. expected %d, got %d", i, scenario.fingerprint, scenario.input.Fingerprint()) } + if scenario.fastFingerprint != scenario.input.FastFingerprint() { + t.Errorf("%d. expected %d, got %d", i, scenario.fastFingerprint, scenario.input.FastFingerprint()) + } } } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/sample_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/sample_test.go index 3dc4ad251..d5e065d6c 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/sample_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/sample_test.go @@ -21,42 +21,36 @@ import ( func TestSamplesSort(t *testing.T) { input := Samples{ &Sample{ - // Fingerprint: 81f9c9ed24563f8f. Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 1, }, &Sample{ - // Fingerprint: 81f9c9ed24563f8f. Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 2, }, &Sample{ - // Fingerprint: 1bf6c9ed24543f8f. Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 1, }, &Sample{ - // Fingerprint: 1bf6c9ed24543f8f. Metric: Metric{ MetricNameLabel: "C", }, Timestamp: 2, }, &Sample{ - // Fingerprint: 68f4c9ed24533f8f. Metric: Metric{ MetricNameLabel: "B", }, Timestamp: 1, }, &Sample{ - // Fingerprint: 68f4c9ed24533f8f. Metric: Metric{ MetricNameLabel: "B", }, @@ -66,47 +60,41 @@ func TestSamplesSort(t *testing.T) { expected := Samples{ &Sample{ - // Fingerprint: 1bf6c9ed24543f8f. - Metric: Metric{ - MetricNameLabel: "C", - }, - Timestamp: 1, - }, - &Sample{ - // Fingerprint: 1bf6c9ed24543f8f. - Metric: Metric{ - MetricNameLabel: "C", - }, - Timestamp: 2, - }, - &Sample{ - // Fingerprint: 68f4c9ed24533f8f. - Metric: Metric{ - MetricNameLabel: "B", - }, - Timestamp: 1, - }, - &Sample{ - // Fingerprint: 68f4c9ed24533f8f. - Metric: Metric{ - MetricNameLabel: "B", - }, - Timestamp: 2, - }, - &Sample{ - // Fingerprint: 81f9c9ed24563f8f. Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 1, }, &Sample{ - // Fingerprint: 81f9c9ed24563f8f. Metric: Metric{ MetricNameLabel: "A", }, Timestamp: 2, }, + &Sample{ + Metric: Metric{ + MetricNameLabel: "B", + }, + Timestamp: 1, + }, + &Sample{ + Metric: Metric{ + MetricNameLabel: "B", + }, + Timestamp: 2, + }, + &Sample{ + Metric: Metric{ + MetricNameLabel: "C", + }, + Timestamp: 1, + }, + &Sample{ + Metric: Metric{ + MetricNameLabel: "C", + }, + Timestamp: 2, + }, } sort.Sort(input) diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature.go index cc77b192d..7bd58f4b0 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature.go @@ -17,6 +17,7 @@ import ( "bytes" "hash" "hash/fnv" + "sort" "sync" ) @@ -46,30 +47,37 @@ func getHashAndBuf() *hashAndBuf { } func putHashAndBuf(hb *hashAndBuf) { + hb.h.Reset() + hb.b.Reset() hashAndBufPool.Put(hb) } -// LabelsToSignature returns a unique signature (i.e., fingerprint) for a given -// label set. +// LabelsToSignature returns a quasi-unique signature (i.e., fingerprint) for a +// given label set. (Collisions are possible but unlikely if the number of label +// sets the function is applied to is small.) func LabelsToSignature(labels map[string]string) uint64 { if len(labels) == 0 { return emptyLabelSignature } - var result uint64 + labelNames := make([]string, 0, len(labels)) + for labelName := range labels { + labelNames = append(labelNames, labelName) + } + sort.Strings(labelNames) + hb := getHashAndBuf() defer putHashAndBuf(hb) - for labelName, labelValue := range labels { + for _, labelName := range labelNames { hb.b.WriteString(labelName) hb.b.WriteByte(SeparatorByte) - hb.b.WriteString(labelValue) + hb.b.WriteString(labels[labelName]) + hb.b.WriteByte(SeparatorByte) hb.h.Write(hb.b.Bytes()) - result ^= hb.h.Sum64() - hb.h.Reset() hb.b.Reset() } - return result + return hb.h.Sum64() } // metricToFingerprint works exactly as LabelsToSignature but takes a Metric as @@ -79,6 +87,34 @@ func metricToFingerprint(m Metric) Fingerprint { return Fingerprint(emptyLabelSignature) } + labelNames := make(LabelNames, 0, len(m)) + for labelName := range m { + labelNames = append(labelNames, labelName) + } + sort.Sort(labelNames) + + hb := getHashAndBuf() + defer putHashAndBuf(hb) + + for _, labelName := range labelNames { + hb.b.WriteString(string(labelName)) + hb.b.WriteByte(SeparatorByte) + hb.b.WriteString(string(m[labelName])) + hb.b.WriteByte(SeparatorByte) + hb.h.Write(hb.b.Bytes()) + hb.b.Reset() + } + return Fingerprint(hb.h.Sum64()) +} + +// metricToFastFingerprint works similar to metricToFingerprint but uses a +// faster and less allocation-heavy hash function, which is more susceptible to +// create hash collisions. Therefore, collision detection should be applied. +func metricToFastFingerprint(m Metric) Fingerprint { + if len(m) == 0 { + return Fingerprint(emptyLabelSignature) + } + var result uint64 hb := getHashAndBuf() defer putHashAndBuf(hb) @@ -97,13 +133,15 @@ func metricToFingerprint(m Metric) Fingerprint { // SignatureForLabels works like LabelsToSignature but takes a Metric as // parameter (rather than a label map) and only includes the labels with the -// specified LabelNames into the signature calculation. +// specified LabelNames into the signature calculation. The labels passed in +// will be sorted by this function. func SignatureForLabels(m Metric, labels LabelNames) uint64 { if len(m) == 0 || len(labels) == 0 { return emptyLabelSignature } - var result uint64 + sort.Sort(labels) + hb := getHashAndBuf() defer putHashAndBuf(hb) @@ -111,12 +149,11 @@ func SignatureForLabels(m Metric, labels LabelNames) uint64 { hb.b.WriteString(string(label)) hb.b.WriteByte(SeparatorByte) hb.b.WriteString(string(m[label])) + hb.b.WriteByte(SeparatorByte) hb.h.Write(hb.b.Bytes()) - result ^= hb.h.Sum64() - hb.h.Reset() hb.b.Reset() } - return result + return hb.h.Sum64() } // SignatureWithoutLabels works like LabelsToSignature but takes a Metric as @@ -127,24 +164,27 @@ func SignatureWithoutLabels(m Metric, labels map[LabelName]struct{}) uint64 { return emptyLabelSignature } - var result uint64 + labelNames := make(LabelNames, 0, len(m)) + for labelName := range m { + if _, exclude := labels[labelName]; !exclude { + labelNames = append(labelNames, labelName) + } + } + if len(labelNames) == 0 { + return emptyLabelSignature + } + sort.Sort(labelNames) + hb := getHashAndBuf() defer putHashAndBuf(hb) - for labelName, labelValue := range m { - if _, exclude := labels[labelName]; exclude { - continue - } + for _, labelName := range labelNames { hb.b.WriteString(string(labelName)) hb.b.WriteByte(SeparatorByte) - hb.b.WriteString(string(labelValue)) + hb.b.WriteString(string(m[labelName])) + hb.b.WriteByte(SeparatorByte) hb.h.Write(hb.b.Bytes()) - result ^= hb.h.Sum64() - hb.h.Reset() hb.b.Reset() } - if result == 0 { - return emptyLabelSignature - } - return result + return hb.h.Sum64() } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature_test.go index 7b3327d44..01db531d0 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/model/signature_test.go @@ -30,7 +30,7 @@ func TestLabelsToSignature(t *testing.T) { }, { in: map[string]string{"name": "garland, briggs", "fear": "love is not enough"}, - out: 12952432476264840823, + out: 5799056148416392346, }, } @@ -54,7 +54,7 @@ func TestMetricToFingerprint(t *testing.T) { }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, - out: 12952432476264840823, + out: 5799056148416392346, }, } @@ -67,6 +67,30 @@ func TestMetricToFingerprint(t *testing.T) { } } +func TestMetricToFastFingerprint(t *testing.T) { + var scenarios = []struct { + in Metric + out Fingerprint + }{ + { + in: Metric{}, + out: 14695981039346656037, + }, + { + in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, + out: 12952432476264840823, + }, + } + + for i, scenario := range scenarios { + actual := metricToFastFingerprint(scenario.in) + + if actual != scenario.out { + t.Errorf("%d. expected %d, got %d", i, scenario.out, actual) + } + } +} + func TestSignatureForLabels(t *testing.T) { var scenarios = []struct { in Metric @@ -81,12 +105,12 @@ func TestSignatureForLabels(t *testing.T) { { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: LabelNames{"fear", "name"}, - out: 12952432476264840823, + out: 5799056148416392346, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough", "foo": "bar"}, labels: LabelNames{"fear", "name"}, - out: 12952432476264840823, + out: 5799056148416392346, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, @@ -128,17 +152,17 @@ func TestSignatureWithoutLabels(t *testing.T) { { in: Metric{"name": "garland, briggs", "fear": "love is not enough", "foo": "bar"}, labels: map[LabelName]struct{}{"foo": struct{}{}}, - out: 12952432476264840823, + out: 5799056148416392346, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: map[LabelName]struct{}{}, - out: 12952432476264840823, + out: 5799056148416392346, }, { in: Metric{"name": "garland, briggs", "fear": "love is not enough"}, labels: nil, - out: 12952432476264840823, + out: 5799056148416392346, }, } @@ -164,15 +188,15 @@ func BenchmarkLabelToSignatureScalar(b *testing.B) { } func BenchmarkLabelToSignatureSingle(b *testing.B) { - benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value"}, 5147259542624943964) + benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value"}, 5146282821936882169) } func BenchmarkLabelToSignatureDouble(b *testing.B) { - benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value"}, 18269973311206963528) + benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value"}, 3195800080984914717) } func BenchmarkLabelToSignatureTriple(b *testing.B) { - benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676) + benchmarkLabelToSignature(b, map[string]string{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 13843036195897128121) } func benchmarkMetricToFingerprint(b *testing.B, m Metric, e Fingerprint) { @@ -188,15 +212,39 @@ func BenchmarkMetricToFingerprintScalar(b *testing.B) { } func BenchmarkMetricToFingerprintSingle(b *testing.B) { - benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value"}, 5147259542624943964) + benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value"}, 5146282821936882169) } func BenchmarkMetricToFingerprintDouble(b *testing.B) { - benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value"}, 18269973311206963528) + benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value"}, 3195800080984914717) } func BenchmarkMetricToFingerprintTriple(b *testing.B) { - benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676) + benchmarkMetricToFingerprint(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 13843036195897128121) +} + +func benchmarkMetricToFastFingerprint(b *testing.B, m Metric, e Fingerprint) { + for i := 0; i < b.N; i++ { + if a := metricToFastFingerprint(m); a != e { + b.Fatalf("expected signature of %d for %s, got %d", e, m, a) + } + } +} + +func BenchmarkMetricToFastFingerprintScalar(b *testing.B) { + benchmarkMetricToFastFingerprint(b, nil, 14695981039346656037) +} + +func BenchmarkMetricToFastFingerprintSingle(b *testing.B) { + benchmarkMetricToFastFingerprint(b, Metric{"first-label": "first-label-value"}, 5147259542624943964) +} + +func BenchmarkMetricToFastFingerprintDouble(b *testing.B) { + benchmarkMetricToFastFingerprint(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value"}, 18269973311206963528) +} + +func BenchmarkMetricToFastFingerprintTriple(b *testing.B) { + benchmarkMetricToFastFingerprint(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676) } func TestEmptyLabelSignature(t *testing.T) { @@ -218,7 +266,7 @@ func TestEmptyLabelSignature(t *testing.T) { } } -func benchmarkMetricToFingerprintConc(b *testing.B, m Metric, e Fingerprint, concLevel int) { +func benchmarkMetricToFastFingerprintConc(b *testing.B, m Metric, e Fingerprint, concLevel int) { var start, end sync.WaitGroup start.Add(1) end.Add(concLevel) @@ -227,7 +275,7 @@ func benchmarkMetricToFingerprintConc(b *testing.B, m Metric, e Fingerprint, con go func() { start.Wait() for j := b.N / concLevel; j >= 0; j-- { - if a := metricToFingerprint(m); a != e { + if a := metricToFastFingerprint(m); a != e { b.Fatalf("expected signature of %d for %s, got %d", e, m, a) } } @@ -239,18 +287,18 @@ func benchmarkMetricToFingerprintConc(b *testing.B, m Metric, e Fingerprint, con end.Wait() } -func BenchmarkMetricToFingerprintTripleConc1(b *testing.B) { - benchmarkMetricToFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 1) +func BenchmarkMetricToFastFingerprintTripleConc1(b *testing.B) { + benchmarkMetricToFastFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 1) } -func BenchmarkMetricToFingerprintTripleConc2(b *testing.B) { - benchmarkMetricToFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 2) +func BenchmarkMetricToFastFingerprintTripleConc2(b *testing.B) { + benchmarkMetricToFastFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 2) } -func BenchmarkMetricToFingerprintTripleConc4(b *testing.B) { - benchmarkMetricToFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 4) +func BenchmarkMetricToFastFingerprintTripleConc4(b *testing.B) { + benchmarkMetricToFastFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 4) } -func BenchmarkMetricToFingerprintTripleConc8(b *testing.B) { - benchmarkMetricToFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 8) +func BenchmarkMetricToFastFingerprintTripleConc8(b *testing.B) { + benchmarkMetricToFastFingerprintConc(b, Metric{"first-label": "first-label-value", "second-label": "second-label-value", "third-label": "third-label-value"}, 15738406913934009676, 8) } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/counter.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/counter.go index f8d633fbd..a2952d1c8 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/counter.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/counter.go @@ -33,7 +33,7 @@ type Counter interface { // Set is used to set the Counter to an arbitrary value. It is only used // if you have to transfer a value from an external counter into this - // Prometheus metrics. Do not use it for regular handling of a + // Prometheus metric. Do not use it for regular handling of a // Prometheus counter (as it can be used to break the contract of // monotonically increasing values). Set(float64) diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go index d106c42c6..a28a80125 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/examples_test.go @@ -455,6 +455,56 @@ func ExampleSummaryVec() { // ] } +func ExampleConstSummary() { + desc := prometheus.NewDesc( + "http_request_duration_seconds", + "A summary of the HTTP request durations.", + []string{"code", "method"}, + prometheus.Labels{"owner": "example"}, + ) + + // Create a constant summary from values we got from a 3rd party telemetry system. + s := prometheus.MustNewConstSummary( + desc, + 4711, 403.34, + map[float64]float64{0.5: 42.3, 0.9: 323.3}, + "200", "get", + ) + + // Just for demonstration, let's check the state of the summary by + // (ab)using its Write method (which is usually only used by Prometheus + // internally). + metric := &dto.Metric{} + s.Write(metric) + fmt.Println(proto.MarshalTextString(metric)) + + // Output: + // label: < + // name: "code" + // value: "200" + // > + // label: < + // name: "method" + // value: "get" + // > + // label: < + // name: "owner" + // value: "example" + // > + // summary: < + // sample_count: 4711 + // sample_sum: 403.34 + // quantile: < + // quantile: 0.5 + // value: 42.3 + // > + // quantile: < + // quantile: 0.9 + // value: 323.3 + // > + // > +} + func ExampleHistogram() { temps := prometheus.NewHistogram(prometheus.HistogramOpts{ Name: "pond_temperature_celsius", @@ -501,6 +551,64 @@ func ExampleHistogram() { // > } +func ExampleConstHistogram() { + desc := prometheus.NewDesc( + "http_request_duration_seconds", + "A histogram of the HTTP request durations.", + []string{"code", "method"}, + prometheus.Labels{"owner": "example"}, + ) + + // Create a constant histogram from values we got from a 3rd party telemetry system. + h := prometheus.MustNewConstHistogram( + desc, + 4711, 403.34, + map[float64]uint64{25: 121, 50: 2403, 100: 3221, 200: 4233}, + "200", "get", + ) + + // Just for demonstration, let's check the state of the histogram by + // (ab)using its Write method (which is usually only used by Prometheus + // internally). + metric := &dto.Metric{} + h.Write(metric) + fmt.Println(proto.MarshalTextString(metric)) + + // Output: + // label: < + // name: "code" + // value: "200" + // > + // label: < + // name: "method" + // value: "get" + // > + // label: < + // name: "owner" + // value: "example" + // > + // histogram: < + // sample_count: 4711 + // sample_sum: 403.34 + // bucket: < + // cumulative_count: 121 + // upper_bound: 25 + // > + // bucket: < + // cumulative_count: 2403 + // upper_bound: 50 + // > + // bucket: < + // cumulative_count: 3221 + // upper_bound: 100 + // > + // bucket: < + // cumulative_count: 4233 + // upper_bound: 200 + // > + // > +} + func ExamplePushCollectors() { hostname, _ := os.Hostname() completionTime := prometheus.NewGauge(prometheus.GaugeOpts{ diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/go_collector.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/go_collector.go index d7b7a20a1..dbf4c306f 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/go_collector.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/go_collector.go @@ -2,10 +2,13 @@ package prometheus import ( "runtime" + "runtime/debug" + "time" ) type goCollector struct { goroutines Gauge + gcDesc *Desc } // NewGoCollector returns a collector which exports metrics about the current @@ -16,16 +19,32 @@ func NewGoCollector() *goCollector { Name: "process_goroutines", Help: "Number of goroutines that currently exist.", }), + gcDesc: NewDesc( + "go_gc_duration_seconds", + "A summary of the GC invocation durations.", + nil, nil), } } // Describe returns all descriptions of the collector. func (c *goCollector) Describe(ch chan<- *Desc) { ch <- c.goroutines.Desc() + ch <- c.gcDesc } // Collect returns the current state of all metrics of the collector. func (c *goCollector) Collect(ch chan<- Metric) { c.goroutines.Set(float64(runtime.NumGoroutine())) ch <- c.goroutines + + var stats debug.GCStats + stats.PauseQuantiles = make([]time.Duration, 5) + debug.ReadGCStats(&stats) + + quantiles := make(map[float64]float64) + for idx, pq := range stats.PauseQuantiles[1:] { + quantiles[float64(idx+1)/float64(len(stats.PauseQuantiles)-1)] = pq.Seconds() + } + quantiles[0.0] = stats.PauseQuantiles[0].Seconds() + ch <- MustNewConstSummary(c.gcDesc, uint64(stats.NumGC), float64(stats.PauseTotal.Seconds()), quantiles) } diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/go_collector_test.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/go_collector_test.go index b0582d1b9..b75d28e59 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/go_collector_test.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/go_collector_test.go @@ -1,7 +1,7 @@ package prometheus import ( - "reflect" + "runtime" "testing" "time" @@ -35,6 +35,9 @@ func TestGoCollector(t *testing.T) { case Gauge: pb := &dto.Metric{} m.Write(pb) + if pb.GetGauge() == nil { + continue + } if old == -1 { old = int(pb.GetGauge().GetValue()) @@ -48,8 +51,66 @@ func TestGoCollector(t *testing.T) { } return - default: - t.Errorf("want type Gauge, got %s", reflect.TypeOf(metric)) + } + case <-time.After(1 * time.Second): + t.Fatalf("expected collect timed out") + } + } +} + +func TestGCCollector(t *testing.T) { + var ( + c = NewGoCollector() + ch = make(chan Metric) + waitc = make(chan struct{}) + closec = make(chan struct{}) + oldGC uint64 + oldPause float64 + ) + defer close(closec) + + go func() { + c.Collect(ch) + // force GC + runtime.GC() + <-waitc + c.Collect(ch) + }() + + first := true + for { + select { + case metric := <-ch: + switch m := metric.(type) { + case *constSummary, *value: + pb := &dto.Metric{} + m.Write(pb) + if pb.GetSummary() == nil { + continue + } + + if len(pb.GetSummary().Quantile) != 5 { + t.Errorf("expected 4 buckets, got %d", len(pb.GetSummary().Quantile)) + } + for idx, want := range []float64{0.0, 0.25, 0.5, 0.75, 1.0} { + if *pb.GetSummary().Quantile[idx].Quantile != want { + t.Errorf("bucket #%d is off, got %f, want %f", idx, *pb.GetSummary().Quantile[idx].Quantile, want) + } + } + if first { + first = false + oldGC = *pb.GetSummary().SampleCount + oldPause = *pb.GetSummary().SampleSum + close(waitc) + continue + } + if diff := *pb.GetSummary().SampleCount - oldGC; diff != 1 { + t.Errorf("want 1 new garbage collection run, got %d", diff) + } + if diff := *pb.GetSummary().SampleSum - oldPause; diff <= 0 { + t.Errorf("want moar pause, got %f", diff) + } + return } case <-time.After(1 * time.Second): t.Fatalf("expected collect timed out") diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram.go index 9b36a6115..27e9c5f7d 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/histogram.go @@ -342,3 +342,102 @@ func (m *HistogramVec) WithLabelValues(lvs ...string) Histogram { func (m *HistogramVec) With(labels Labels) Histogram { return m.MetricVec.With(labels).(Histogram) } + +type constHistogram struct { + desc *Desc + count uint64 + sum float64 + buckets map[float64]uint64 + labelPairs []*dto.LabelPair +} + +func (h *constHistogram) Desc() *Desc { + return h.desc +} + +func (h *constHistogram) Write(out *dto.Metric) error { + his := &dto.Histogram{} + buckets := make([]*dto.Bucket, 0, len(h.buckets)) + + his.SampleCount = proto.Uint64(h.count) + his.SampleSum = proto.Float64(h.sum) + + for upperBound, count := range h.buckets { + buckets = append(buckets, &dto.Bucket{ + CumulativeCount: proto.Uint64(count), + UpperBound: proto.Float64(upperBound), + }) + } + + if len(buckets) > 0 { + sort.Sort(buckSort(buckets)) + } + his.Bucket = buckets + + out.Histogram = his + out.Label = h.labelPairs + + return nil +} + +// NewConstHistogram returns a metric representing a Prometheus histogram with +// fixed values for the count, sum, and bucket counts. As those parameters +// cannot be changed, the returned value does not implement the Histogram +// interface (but only the Metric interface). Users of this package will not +// have much use for it in regular operations. However, when implementing custom +// Collectors, it is useful as a throw-away metric that is generated on the fly +// to send it to Prometheus in the Collect method. +// +// buckets is a map of upper bounds to cumulative counts, excluding the +Inf +// bucket. +// +// NewConstHistogram returns an error if the length of labelValues is not +// consistent with the variable labels in Desc. +func NewConstHistogram( + desc *Desc, + count uint64, + sum float64, + buckets map[float64]uint64, + labelValues ...string, +) (Metric, error) { + if len(desc.variableLabels) != len(labelValues) { + return nil, errInconsistentCardinality + } + return &constHistogram{ + desc: desc, + count: count, + sum: sum, + buckets: buckets, + labelPairs: makeLabelPairs(desc, labelValues), + }, nil +} + +// MustNewConstHistogram is a version of NewConstHistogram that panics where +// NewConstMetric would have returned an error. +func MustNewConstHistogram( + desc *Desc, + count uint64, + sum float64, + buckets map[float64]uint64, + labelValues ...string, +) Metric { + m, err := NewConstHistogram(desc, count, sum, buckets, labelValues...) + if err != nil { + panic(err) + } + return m +} + +type buckSort []*dto.Bucket + +func (s buckSort) Len() int { + return len(s) +} + +func (s buckSort) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +} + +func (s buckSort) Less(i, j int) bool { + return s[i].GetUpperBound() < s[j].GetUpperBound() +} diff --git a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go index dd336c519..67fe43cd7 100644 --- a/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go +++ b/Godeps/_workspace/src/github.com/prometheus/client_golang/prometheus/summary.go @@ -448,3 +448,89 @@ func (m *SummaryVec) WithLabelValues(lvs ...string) Summary { func (m *SummaryVec) With(labels Labels) Summary { return m.MetricVec.With(labels).(Summary) } + +type constSummary struct { + desc *Desc + count uint64 + sum float64 + quantiles map[float64]float64 + labelPairs []*dto.LabelPair +} + +func (s *constSummary) Desc() *Desc { + return s.desc +} + +func (s *constSummary) Write(out *dto.Metric) error { + sum := &dto.Summary{} + qs := make([]*dto.Quantile, 0, len(s.quantiles)) + + sum.SampleCount = proto.Uint64(s.count) + sum.SampleSum = proto.Float64(s.sum) + + for rank, q := range s.quantiles { + qs = append(qs, &dto.Quantile{ + Quantile: proto.Float64(rank), + Value: proto.Float64(q), + }) + } + + if len(qs) > 0 { + sort.Sort(quantSort(qs)) + } + sum.Quantile = qs + + out.Summary = sum + out.Label = s.labelPairs + + return nil +} + +// NewConstSummary returns a metric representing a Prometheus summary with fixed +// values for the count, sum, and quantiles. As those parameters cannot be +// changed, the returned value does not implement the Summary interface (but +// only the Metric interface). Users of this package will not have much use for +// it in regular operations. However, when implementing custom Collectors, it is +// useful as a throw-away metric that is generated on the fly to send it to +// Prometheus in the Collect method. +// +// quantiles maps ranks to quantile values. For example, a median latency of +// 0.23s and a 99th percentile latency of 0.56s would be expressed as: +// map[float64]float64{0.5: 0.23, 0.99: 0.56} +// +// NewConstSummary returns an error if the length of labelValues is not +// consistent with the variable labels in Desc. +func NewConstSummary( + desc *Desc, + count uint64, + sum float64, + quantiles map[float64]float64, + labelValues ...string, +) (Metric, error) { + if len(desc.variableLabels) != len(labelValues) { + return nil, errInconsistentCardinality + } + return &constSummary{ + desc: desc, + count: count, + sum: sum, + quantiles: quantiles, + labelPairs: makeLabelPairs(desc, labelValues), + }, nil +} + +// MustNewConstSummary is a version of NewConstSummary that panics where +// NewConstMetric would have returned an error. +func MustNewConstSummary( + desc *Desc, + count uint64, + sum float64, + quantiles map[float64]float64, + labelValues ...string, +) Metric { + m, err := NewConstSummary(desc, count, sum, quantiles, labelValues...) + if err != nil { + panic(err) + } + return m +} diff --git a/storage/local/persistence_test.go b/storage/local/persistence_test.go index fc21b8d24..61abf1b19 100644 --- a/storage/local/persistence_test.go +++ b/storage/local/persistence_test.go @@ -50,9 +50,9 @@ func newTestPersistence(t *testing.T, encoding chunkEncoding) (*persistence, tes func buildTestChunks(encoding chunkEncoding) map[clientmodel.Fingerprint][]chunk { fps := clientmodel.Fingerprints{ - m1.Fingerprint(), - m2.Fingerprint(), - m3.Fingerprint(), + m1.FastFingerprint(), + m2.FastFingerprint(), + m3.FastFingerprint(), } fpToChunks := map[clientmodel.Fingerprint][]chunk{} @@ -375,11 +375,11 @@ func testCheckpointAndLoadSeriesMapAndHeads(t *testing.T, encoding chunkEncoding s5.persistWatermark = 3 chunkCountS4 := len(s4.chunkDescs) chunkCountS5 := len(s5.chunkDescs) - sm.put(m1.Fingerprint(), s1) - sm.put(m2.Fingerprint(), s2) - sm.put(m3.Fingerprint(), s3) - sm.put(m4.Fingerprint(), s4) - sm.put(m5.Fingerprint(), s5) + sm.put(m1.FastFingerprint(), s1) + sm.put(m2.FastFingerprint(), s2) + sm.put(m3.FastFingerprint(), s3) + sm.put(m4.FastFingerprint(), s4) + sm.put(m5.FastFingerprint(), s5) if err := p.checkpointSeriesMapAndHeads(sm, fpLocker); err != nil { t.Fatal(err) @@ -392,7 +392,7 @@ func testCheckpointAndLoadSeriesMapAndHeads(t *testing.T, encoding chunkEncoding if loadedSM.length() != 4 { t.Errorf("want 4 series in map, got %d", loadedSM.length()) } - if loadedS1, ok := loadedSM.get(m1.Fingerprint()); ok { + if loadedS1, ok := loadedSM.get(m1.FastFingerprint()); ok { if !reflect.DeepEqual(loadedS1.metric, m1) { t.Errorf("want metric %v, got %v", m1, loadedS1.metric) } @@ -408,7 +408,7 @@ func testCheckpointAndLoadSeriesMapAndHeads(t *testing.T, encoding chunkEncoding } else { t.Errorf("couldn't find %v in loaded map", m1) } - if loadedS3, ok := loadedSM.get(m3.Fingerprint()); ok { + if loadedS3, ok := loadedSM.get(m3.FastFingerprint()); ok { if !reflect.DeepEqual(loadedS3.metric, m3) { t.Errorf("want metric %v, got %v", m3, loadedS3.metric) } @@ -424,7 +424,7 @@ func testCheckpointAndLoadSeriesMapAndHeads(t *testing.T, encoding chunkEncoding } else { t.Errorf("couldn't find %v in loaded map", m3) } - if loadedS4, ok := loadedSM.get(m4.Fingerprint()); ok { + if loadedS4, ok := loadedSM.get(m4.FastFingerprint()); ok { if !reflect.DeepEqual(loadedS4.metric, m4) { t.Errorf("want metric %v, got %v", m4, loadedS4.metric) } @@ -449,7 +449,7 @@ func testCheckpointAndLoadSeriesMapAndHeads(t *testing.T, encoding chunkEncoding } else { t.Errorf("couldn't find %v in loaded map", m4) } - if loadedS5, ok := loadedSM.get(m5.Fingerprint()); ok { + if loadedS5, ok := loadedSM.get(m5.FastFingerprint()); ok { if !reflect.DeepEqual(loadedS5.metric, m5) { t.Errorf("want metric %v, got %v", m5, loadedS5.metric) } diff --git a/storage/local/series.go b/storage/local/series.go index df76c9712..5e9c1e5e3 100644 --- a/storage/local/series.go +++ b/storage/local/series.go @@ -318,7 +318,7 @@ func (s *memorySeries) preloadChunks(indexes []int, mss *memorySeriesStorage) ([ if s.chunkDescsOffset == -1 { panic("requested loading chunks from persistence in a situation where we must not have persisted data for chunk descriptors in memory") } - fp := s.metric.Fingerprint() + fp := s.metric.FastFingerprint() // TODO(beorn): Handle collisions. chunks, err := mss.loadChunks(fp, loadIndexes, s.chunkDescsOffset) if err != nil { // Unpin the chunks since we won't return them as pinned chunks now. diff --git a/storage/local/storage.go b/storage/local/storage.go index e741dae61..3782359b6 100644 --- a/storage/local/storage.go +++ b/storage/local/storage.go @@ -382,7 +382,7 @@ func (s *memorySeriesStorage) Append(sample *clientmodel.Sample) { } glog.Warning("Sample ingestion resumed.") } - fp := sample.Metric.Fingerprint() + fp := sample.Metric.FastFingerprint() // TODO(beorn): Handle collisions. s.fpLocker.Lock(fp) series := s.getOrCreateSeries(fp, sample.Metric) completedChunksCount := series.add(&metric.SamplePair{ diff --git a/storage/local/storage_test.go b/storage/local/storage_test.go index 9801d6b0e..330a4eafe 100644 --- a/storage/local/storage_test.go +++ b/storage/local/storage_test.go @@ -46,7 +46,7 @@ func TestGetFingerprintsForLabelMatchers(t *testing.T) { Timestamp: clientmodel.Timestamp(i), Value: clientmodel.SampleValue(i), } - fingerprints[i] = metric.Fingerprint() + fingerprints[i] = metric.FastFingerprint() } for _, s := range samples { storage.Append(s) @@ -172,7 +172,7 @@ func TestLoop(t *testing.T) { storage.Append(s) } storage.WaitForIndexing() - series, _ := storage.(*memorySeriesStorage).fpToSeries.get(clientmodel.Metric{}.Fingerprint()) + series, _ := storage.(*memorySeriesStorage).fpToSeries.get(clientmodel.Metric{}.FastFingerprint()) cdsBefore := len(series.chunkDescs) time.Sleep(fpMaxWaitDuration + time.Second) // TODO(beorn7): Ugh, need to wait for maintenance to kick in. cdsAfter := len(series.chunkDescs) @@ -251,7 +251,7 @@ func testGetValueAtTime(t *testing.T, encoding chunkEncoding) { } s.WaitForIndexing() - fp := clientmodel.Metric{}.Fingerprint() + fp := clientmodel.Metric{}.FastFingerprint() it := s.NewIterator(fp) @@ -344,7 +344,7 @@ func testGetRangeValues(t *testing.T, encoding chunkEncoding) { } s.WaitForIndexing() - fp := clientmodel.Metric{}.Fingerprint() + fp := clientmodel.Metric{}.FastFingerprint() it := s.NewIterator(fp) @@ -498,7 +498,7 @@ func testEvictAndPurgeSeries(t *testing.T, encoding chunkEncoding) { } s.WaitForIndexing() - fp := clientmodel.Metric{}.Fingerprint() + fp := clientmodel.Metric{}.FastFingerprint() // Drop ~half of the chunks. ms.maintainMemorySeries(fp, 1000) @@ -896,7 +896,7 @@ func verifyStorage(t testing.TB, s Storage, samples clientmodel.Samples, maxAge // retention period, we can verify here that no results // are returned. } - fp := sample.Metric.Fingerprint() + fp := sample.Metric.FastFingerprint() p := s.NewPreloader() p.PreloadRange(fp, sample.Timestamp, sample.Timestamp, time.Hour) found := s.NewIterator(fp).GetValueAtTime(sample.Timestamp)