From 873411ffce0cf8b4867d6ae7ce03c52fa2d3830e Mon Sep 17 00:00:00 2001 From: Arthur Silva Sens Date: Mon, 12 Jun 2023 12:17:20 -0300 Subject: [PATCH] Add feature flag to squash metadata from /api/v1/metadata (#12391) Signed-off-by: ArthurSens --- docs/querying/api.md | 27 +++++++ web/api/v1/api.go | 16 ++++- web/api/v1/api_test.go | 160 ++++++++++++++++++++++++++++++++++++++--- 3 files changed, 193 insertions(+), 10 deletions(-) diff --git a/docs/querying/api.md b/docs/querying/api.md index b2b72d9cd..ca7f64f62 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -863,6 +863,7 @@ GET /api/v1/metadata URL query parameters: - `limit=`: Maximum number of metrics to return. +- `limit_per_metric=`: Maximum number of metadata to return per metric. - `metric=`: A metric name to filter metadata for. All metric metadata is retrieved if left empty. The `data` section of the query result consists of an object where each key is a metric name and each value is a list of unique metadata objects, as exposed for that metric name across all targets. @@ -898,6 +899,32 @@ curl -G http://localhost:9090/api/v1/metadata?limit=2 } ``` +The following example returns only one metadata entry for each metric. + +```json +curl -G http://localhost:9090/api/v1/metadata?limit_per_metric=1 + +{ + "status": "success", + "data": { + "cortex_ring_tokens": [ + { + "type": "gauge", + "help": "Number of tokens in the ring", + "unit": "" + } + ], + "http_requests_total": [ + { + "type": "counter", + "help": "Number of HTTP requests", + "unit": "" + } + ] + } +} +``` + The following example returns metadata only for the metric `http_requests_total`. ```json diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 6c8db32c7..a3eafb623 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -1202,16 +1202,26 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult { return apiFuncResult{nil, &apiError{errorBadData, errors.New("limit must be a number")}, nil, nil} } } + limitPerMetric := -1 + if s := r.FormValue("limit_per_metric"); s != "" { + var err error + if limitPerMetric, err = strconv.Atoi(s); err != nil { + return apiFuncResult{nil, &apiError{errorBadData, errors.New("limit_per_metric must be a number")}, nil, nil} + } + } metric := r.FormValue("metric") for _, tt := range api.targetRetriever(r.Context()).TargetsActive() { for _, t := range tt { - if metric == "" { for _, mm := range t.MetadataList() { m := metadata{Type: mm.Type, Help: mm.Help, Unit: mm.Unit} ms, ok := metrics[mm.Metric] + if limitPerMetric > 0 && len(ms) >= limitPerMetric { + continue + } + if !ok { ms = map[metadata]struct{}{} metrics[mm.Metric] = ms @@ -1225,6 +1235,10 @@ func (api *API) metricMetadata(r *http.Request) apiFuncResult { m := metadata{Type: md.Type, Help: md.Help, Unit: md.Unit} ms, ok := metrics[md.Metric] + if limitPerMetric > 0 && len(ms) >= limitPerMetric { + continue + } + if !ok { ms = map[metadata]struct{}{} metrics[md.Metric] = ms diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 2d7455bf2..cef610769 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -1021,15 +1021,16 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E } type test struct { - endpoint apiFunc - params map[string]string - query url.Values - response interface{} - responseLen int - errType errorType - sorter func(interface{}) - metadata []targetMetadata - exemplars []exemplar.QueryResult + endpoint apiFunc + params map[string]string + query url.Values + response interface{} + responseLen int + responseMetadataTotal int + errType errorType + sorter func(interface{}) + metadata []targetMetadata + exemplars []exemplar.QueryResult } tests := []test{ @@ -1774,6 +1775,126 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E }, responseLen: 2, }, + // With a limit for the number of metadata per metric. + { + endpoint: api.metricMetadata, + query: url.Values{"limit_per_metric": []string{"1"}}, + metadata: []targetMetadata{ + { + identifier: "test", + metadata: []scrape.MetricMetadata{ + { + Metric: "go_threads", + Type: textparse.MetricTypeGauge, + Help: "Number of OS threads created", + Unit: "", + }, + { + Metric: "go_threads", + Type: textparse.MetricTypeGauge, + Help: "Repeated metadata", + Unit: "", + }, + { + Metric: "go_gc_duration_seconds", + Type: textparse.MetricTypeSummary, + Help: "A summary of the GC invocation durations.", + Unit: "", + }, + }, + }, + }, + response: map[string][]metadata{ + "go_threads": { + {textparse.MetricTypeGauge, "Number of OS threads created", ""}, + }, + "go_gc_duration_seconds": { + {textparse.MetricTypeSummary, "A summary of the GC invocation durations.", ""}, + }, + }, + }, + // With a limit for the number of metadata per metric and per metric. + { + endpoint: api.metricMetadata, + query: url.Values{"limit_per_metric": []string{"1"}, "limit": []string{"1"}}, + metadata: []targetMetadata{ + { + identifier: "test", + metadata: []scrape.MetricMetadata{ + { + Metric: "go_threads", + Type: textparse.MetricTypeGauge, + Help: "Number of OS threads created", + Unit: "", + }, + { + Metric: "go_threads", + Type: textparse.MetricTypeGauge, + Help: "Repeated metadata", + Unit: "", + }, + { + Metric: "go_gc_duration_seconds", + Type: textparse.MetricTypeSummary, + Help: "A summary of the GC invocation durations.", + Unit: "", + }, + }, + }, + }, + responseLen: 1, + responseMetadataTotal: 1, + }, + + // With a limit for the number of metadata per metric and per metric, while having multiple targets. + { + endpoint: api.metricMetadata, + query: url.Values{"limit_per_metric": []string{"1"}, "limit": []string{"1"}}, + metadata: []targetMetadata{ + { + identifier: "test", + metadata: []scrape.MetricMetadata{ + { + Metric: "go_threads", + Type: textparse.MetricTypeGauge, + Help: "Number of OS threads created", + Unit: "", + }, + { + Metric: "go_threads", + Type: textparse.MetricTypeGauge, + Help: "Repeated metadata", + Unit: "", + }, + { + Metric: "go_gc_duration_seconds", + Type: textparse.MetricTypeSummary, + Help: "A summary of the GC invocation durations.", + Unit: "", + }, + }, + }, + { + identifier: "secondTarget", + metadata: []scrape.MetricMetadata{ + { + Metric: "go_threads", + Type: textparse.MetricTypeGauge, + Help: "Number of OS threads created, but from a different target", + Unit: "", + }, + { + Metric: "go_gc_duration_seconds", + Type: textparse.MetricTypeSummary, + Help: "A summary of the GC invocation durations, but from a different target.", + Unit: "", + }, + }, + }, + }, + responseLen: 1, + responseMetadataTotal: 1, + }, // When requesting a specific metric that is present. { endpoint: api.metricMetadata, @@ -2563,6 +2684,9 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E if test.responseLen != 0 { assertAPIResponseLength(t, res.data, test.responseLen) + if test.responseMetadataTotal != 0 { + assertAPIResponseMetadataLen(t, res.data, test.responseMetadataTotal) + } } else { assertAPIResponse(t, res.data, test.response) } @@ -2613,6 +2737,24 @@ func assertAPIResponseLength(t *testing.T, got interface{}, expLen int) { } } +func assertAPIResponseMetadataLen(t *testing.T, got interface{}, expLen int) { + t.Helper() + + var gotLen int + response := got.(map[string][]metadata) + for _, m := range response { + gotLen += len(m) + } + + if gotLen != expLen { + t.Fatalf( + "Amount of metadata in the response does not match, expected:\n%d\ngot:\n%d", + expLen, + gotLen, + ) + } +} + type fakeDB struct { err error }