From 52662e81053be6daa3abe3b6ae91213ccc531144 Mon Sep 17 00:00:00 2001 From: David N Perkins Date: Fri, 7 Jan 2022 15:31:37 -0500 Subject: [PATCH 001/180] Added metric name to histogram quartile binning and associated unit test Signed-off-by: David N Perkins --- promql/functions.go | 7 +++++-- promql/testdata/histograms.test | 9 +++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/promql/functions.go b/promql/functions.go index fbafc7864f..3fad20d15c 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -791,7 +791,7 @@ func funcPredictLinear(vals []parser.Value, args parser.Expressions, enh *EvalNo func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *EvalNodeHelper) Vector { q := vals[0].(Vector)[0].V inVec := vals[1].(Vector) - sigf := signatureFunc(false, enh.lblBuf, excludedLabels...) + sigf := signatureFunc(false, enh.lblBuf, labels.BucketLabel) if enh.signatureToMetricWithBuckets == nil { enh.signatureToMetricWithBuckets = map[string]*metricWithBuckets{} @@ -810,11 +810,14 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev continue } l := sigf(el.Metric) + // Add the metric name (which is always removed) to the signature to prevent combining multiple histograms + // with the same label set + l = l + el.Metric.Get(model.MetricNameLabel) mb, ok := enh.signatureToMetricWithBuckets[l] if !ok { el.Metric = labels.NewBuilder(el.Metric). - Del(labels.BucketLabel, labels.MetricName). + Del(excludedLabels...). Labels() mb = &metricWithBuckets{el.Metric, nil} diff --git a/promql/testdata/histograms.test b/promql/testdata/histograms.test index 07e9a9225f..508f1eface 100644 --- a/promql/testdata/histograms.test +++ b/promql/testdata/histograms.test @@ -219,3 +219,12 @@ load 5m eval instant at 50m histogram_quantile(0.2, rate(empty_bucket[5m])) {instance="ins1", job="job1"} NaN + +# Load a duplicate histogram with a different name to test failure scenario on multiple histograms with the same label set +# https://github.com/prometheus/prometheus/issues/9910 +load 5m + request_duration_seconds2_bucket{job="job1", instance="ins1", le="0.1"} 0+1x10 + request_duration_seconds2_bucket{job="job1", instance="ins1", le="0.2"} 0+3x10 + request_duration_seconds2_bucket{job="job1", instance="ins1", le="+Inf"} 0+4x10 + +eval_fail instant at 50m histogram_quantile(0.99, {__name__=~"request_duration.*"}) From 472456efdeabf007eda963ec53ff67cc6c75d690 Mon Sep 17 00:00:00 2001 From: David N Perkins Date: Fri, 7 Jan 2022 15:38:15 -0500 Subject: [PATCH 002/180] Added issue link to the comment Signed-off-by: David N Perkins --- promql/functions.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/promql/functions.go b/promql/functions.go index 3fad20d15c..20eafd0e61 100644 --- a/promql/functions.go +++ b/promql/functions.go @@ -811,7 +811,7 @@ func funcHistogramQuantile(vals []parser.Value, args parser.Expressions, enh *Ev } l := sigf(el.Metric) // Add the metric name (which is always removed) to the signature to prevent combining multiple histograms - // with the same label set + // with the same label set. See https://github.com/prometheus/prometheus/issues/9910 l = l + el.Metric.Get(model.MetricNameLabel) mb, ok := enh.signatureToMetricWithBuckets[l] From d588b14d9ceba6bc23b0697dc12c7a2468a0e8ae Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 29 Aug 2021 17:55:45 +0100 Subject: [PATCH 003/180] remote_write: detailed test for shard calculation Drive the input parameters to `calculateDesiredShards()` very precisely, to illustrate some questionable behaviour marked with `?!`. See https://github.com/prometheus/prometheus/issues/9178, https://github.com/prometheus/prometheus/issues/9207, Signed-off-by: Bryan Boreham --- storage/remote/queue_manager_test.go | 183 +++++++++++++++++++++++++++ 1 file changed, 183 insertions(+) diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go index e0b056e5b3..284b36b182 100644 --- a/storage/remote/queue_manager_test.go +++ b/storage/remote/queue_manager_test.go @@ -908,6 +908,189 @@ func TestCalculateDesiredShards(t *testing.T) { require.Equal(t, int64(0), pendingSamples, "Remote write never caught up, there are still %d pending samples.", pendingSamples) } +func TestCalculateDesiredShardsDetail(t *testing.T) { + c := NewTestWriteClient() + cfg := config.DefaultQueueConfig + mcfg := config.DefaultMetadataConfig + + dir, err := ioutil.TempDir("", "TestCalculateDesiredShards") + require.NoError(t, err) + defer func() { + require.NoError(t, os.RemoveAll(dir)) + }() + + metrics := newQueueManagerMetrics(nil, "", "") + samplesIn := newEWMARate(ewmaWeight, shardUpdateDuration) + m := NewQueueManager(metrics, nil, nil, nil, dir, samplesIn, cfg, mcfg, nil, nil, c, defaultFlushDeadline, newPool(), newHighestTimestampMetric(), nil, false) + + for _, tc := range []struct { + name string + prevShards int + dataIn int64 // Quantities normalised to seconds. + dataOut int64 + dataDropped int64 + dataOutDuration float64 + backlog float64 + expectedShards int + }{ + { + name: "nothing in or out 1", + prevShards: 1, + expectedShards: 1, // Shards stays the same. + }, + { + name: "nothing in or out 10", + prevShards: 10, + expectedShards: 10, // Shards stays the same. + }, + { + name: "steady throughput", + prevShards: 1, + dataIn: 10, + dataOut: 10, + dataOutDuration: 1, + expectedShards: 1, + }, + { + name: "scale down", + prevShards: 10, + dataIn: 10, + dataOut: 10, + dataOutDuration: 5, + expectedShards: 5, + }, + { + name: "scale down constrained", + prevShards: 7, + dataIn: 10, + dataOut: 10, + dataOutDuration: 5, + expectedShards: 7, + }, + { + name: "scale up", + prevShards: 1, + dataIn: 10, + dataOut: 10, + dataOutDuration: 10, + expectedShards: 10, + }, + { + name: "scale up constrained", + prevShards: 8, + dataIn: 10, + dataOut: 10, + dataOutDuration: 10, + expectedShards: 8, + }, + { + name: "backlogged 20s", + prevShards: 2, + dataIn: 10, + dataOut: 10, + dataOutDuration: 2, + backlog: 20, + expectedShards: 2, // ? should be trying to catch up + }, + { + name: "backlogged 90s", + prevShards: 2, + dataIn: 10, + dataOut: 10, + dataOutDuration: 2, + backlog: 90, + expectedShards: 4, // ?! should be trying to go much faster + }, + { + name: "backlog reduced", + prevShards: 4, + dataIn: 10, + dataOut: 20, + dataOutDuration: 4, + backlog: 10, + expectedShards: 3, // ?! shouldn't downshard from 4 to 3: less than 30% change + }, + { + name: "backlog eliminated", + prevShards: 3, + dataIn: 10, + dataOut: 10, + dataOutDuration: 2, + backlog: 0, + expectedShards: 2, // Shard back down. + }, + { + name: "slight slowdown", + prevShards: 1, + dataIn: 10, + dataOut: 10, + dataOutDuration: 1.2, + expectedShards: 1, // no reaction - less than 30% change + }, + { + name: "bigger slowdown", + prevShards: 1, + dataIn: 10, + dataOut: 10, + dataOutDuration: 1.4, + expectedShards: 2, // now it shards up + }, + { + name: "speed up", + prevShards: 2, + dataIn: 10, + dataOut: 10, + dataOutDuration: 1.2, + backlog: 0, + expectedShards: 2, // No reaction - 1.2 is rounded up to 2. + }, + { + name: "speed up more", + prevShards: 2, + dataIn: 10, + dataOut: 10, + dataOutDuration: 0.9, + backlog: 0, + expectedShards: 1, + }, + { + name: "marginal decision A", + prevShards: 3, + dataIn: 10, + dataOut: 10, + dataOutDuration: 2.01, + backlog: 0, + expectedShards: 3, // 2.01 rounds up to 3. + }, + { + name: "marginal decision B", + prevShards: 3, + dataIn: 10, + dataOut: 10, + dataOutDuration: 1.99, + backlog: 0, + expectedShards: 2, // 1.99 rounds up to 2. + }, + } { + t.Run(tc.name, func(t *testing.T) { + m.numShards = tc.prevShards + forceEMWA(samplesIn, tc.dataIn*int64(shardUpdateDuration/time.Second)) + samplesIn.tick() + forceEMWA(m.dataOut, tc.dataOut*int64(shardUpdateDuration/time.Second)) + forceEMWA(m.dataDropped, tc.dataDropped*int64(shardUpdateDuration/time.Second)) + forceEMWA(m.dataOutDuration, int64(tc.dataOutDuration*float64(shardUpdateDuration))) + m.highestRecvTimestamp.value = tc.backlog // Not Set() because it can only increase value. + + require.Equal(t, tc.expectedShards, m.calculateDesiredShards()) + }) + } +} + +func forceEMWA(r *ewmaRate, rate int64) { + r.init = false + r.newEvents.Store(rate) +} + func TestQueueManagerMetrics(t *testing.T) { reg := prometheus.NewPedanticRegistry() metrics := newQueueManagerMetrics(reg, "name", "http://localhost:1234") From 6d01ce8c4dbeffd1a2651caf9bb0cad7de949ef2 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Sun, 29 Aug 2021 18:11:13 +0100 Subject: [PATCH 004/180] remote_write: shard up more when backlogged Change the coefficient from 1% to 5%, so instead of targetting to clear the backlog in 100s we target 20s. Update unit test to reflect the new behaviour. Signed-off-by: Bryan Boreham --- storage/remote/queue_manager.go | 10 ++++------ storage/remote/queue_manager_test.go | 12 ++++++------ 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index e4a2b5399c..d7981a0936 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -831,14 +831,12 @@ func (t *QueueManager) calculateDesiredShards() int { return t.numShards } - // When behind we will try to catch up on a proporation of samples per tick. - // This works similarly to an integral accumulator in that pending samples - // is the result of the error integral. - const integralGain = 0.1 / float64(shardUpdateDuration/time.Second) - var ( + // When behind we will try to catch up on 5% of samples per second. + backlogCatchup = 0.05 * dataPending + // Calculate Time to send one sample, averaged across all sends done this tick. timePerSample = dataOutDuration / dataOutRate - desiredShards = timePerSample * (dataInRate*dataKeptRatio + integralGain*dataPending) + desiredShards = timePerSample * (dataInRate*dataKeptRatio + backlogCatchup) ) t.metrics.desiredNumShards.Set(desiredShards) level.Debug(t.logger).Log("msg", "QueueManager.calculateDesiredShards", diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go index 284b36b182..56498a0f08 100644 --- a/storage/remote/queue_manager_test.go +++ b/storage/remote/queue_manager_test.go @@ -990,25 +990,25 @@ func TestCalculateDesiredShardsDetail(t *testing.T) { dataOut: 10, dataOutDuration: 2, backlog: 20, - expectedShards: 2, // ? should be trying to catch up + expectedShards: 4, }, { name: "backlogged 90s", - prevShards: 2, + prevShards: 4, dataIn: 10, dataOut: 10, - dataOutDuration: 2, + dataOutDuration: 4, backlog: 90, - expectedShards: 4, // ?! should be trying to go much faster + expectedShards: 22, }, { name: "backlog reduced", - prevShards: 4, + prevShards: 22, dataIn: 10, dataOut: 20, dataOutDuration: 4, backlog: 10, - expectedShards: 3, // ?! shouldn't downshard from 4 to 3: less than 30% change + expectedShards: 3, }, { name: "backlog eliminated", From 954c0e8020c1c86c1b5898778aacb7fa5ecc9341 Mon Sep 17 00:00:00 2001 From: Bryan Boreham Date: Mon, 30 Aug 2021 13:39:16 +0100 Subject: [PATCH 005/180] remote_write: round desired shards up before check Previously we would reject an increase from 2 to 2.5 as being within 30%; by rounding up first we see this as an increase from 2 to 3. Signed-off-by: Bryan Boreham --- storage/remote/queue_manager.go | 4 +++- storage/remote/queue_manager_test.go | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index d7981a0936..32a4aba4ae 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -859,11 +859,13 @@ func (t *QueueManager) calculateDesiredShards() int { ) level.Debug(t.logger).Log("msg", "QueueManager.updateShardsLoop", "lowerBound", lowerBound, "desiredShards", desiredShards, "upperBound", upperBound) + + desiredShards = math.Ceil(desiredShards) // Round up to be on the safe side. if lowerBound <= desiredShards && desiredShards <= upperBound { return t.numShards } - numShards := int(math.Ceil(desiredShards)) + numShards := int(desiredShards) // Do not downshard if we are more than ten seconds back. if numShards < t.numShards && delay > 10.0 { level.Debug(t.logger).Log("msg", "Not downsharding due to being too far behind") diff --git a/storage/remote/queue_manager_test.go b/storage/remote/queue_manager_test.go index 56498a0f08..a2d63b0bcb 100644 --- a/storage/remote/queue_manager_test.go +++ b/storage/remote/queue_manager_test.go @@ -1025,7 +1025,7 @@ func TestCalculateDesiredShardsDetail(t *testing.T) { dataIn: 10, dataOut: 10, dataOutDuration: 1.2, - expectedShards: 1, // no reaction - less than 30% change + expectedShards: 2, // 1.2 is rounded up to 2. }, { name: "bigger slowdown", @@ -1033,7 +1033,7 @@ func TestCalculateDesiredShardsDetail(t *testing.T) { dataIn: 10, dataOut: 10, dataOutDuration: 1.4, - expectedShards: 2, // now it shards up + expectedShards: 2, }, { name: "speed up", From bff9d0687472e8669c3c37f051b3ea59cb62c45b Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Mon, 10 Jan 2022 17:39:51 +0100 Subject: [PATCH 006/180] Service Discovery Page rework (#10131) * rework the target page Signed-off-by: Augustin Husson * put back the URL of the endpoint Signed-off-by: Augustin Husson * replace old code by the new one and change function style Signed-off-by: Augustin Husson * align filter and search bar on the same row Signed-off-by: Augustin Husson * remove unnecessary return Signed-off-by: Augustin Husson * upgrade kvsearch to v0.3.0 Signed-off-by: Augustin Husson * fix unit test Signed-off-by: Augustin Husson * add missing style on column Signed-off-by: Augustin Husson * add placeholder and autofocus Signed-off-by: Augustin Husson * put back the previous table design Signed-off-by: Augustin Husson * fix issue relative to the position of the tooltip Signed-off-by: Augustin Husson * fix health filter Signed-off-by: Augustin Husson * fix test on label tooltip Signed-off-by: Augustin Husson * simplify filter condition Signed-off-by: Augustin Husson * rework service discovery page Signed-off-by: Augustin Husson * introduced generic custom infinite scroll component Signed-off-by: Augustin Husson * adjust the placeholder in discovery page Signed-off-by: Augustin Husson * ignore returning type missing Signed-off-by: Augustin Husson * apply fix required by the review Signed-off-by: Augustin Husson * index discoveredLabels Signed-off-by: Augustin Husson --- .../src/components/CustomInfiniteScroll.tsx | 50 ++++++++ .../pages/serviceDiscovery/LabelsTable.tsx | 53 ++++---- .../src/pages/serviceDiscovery/Services.tsx | 46 ++++++- .../src/pages/targets/ScrapePoolContent.tsx | 120 +++++++----------- 4 files changed, 168 insertions(+), 101 deletions(-) create mode 100644 web/ui/react-app/src/components/CustomInfiniteScroll.tsx diff --git a/web/ui/react-app/src/components/CustomInfiniteScroll.tsx b/web/ui/react-app/src/components/CustomInfiniteScroll.tsx new file mode 100644 index 0000000000..87f1d6067c --- /dev/null +++ b/web/ui/react-app/src/components/CustomInfiniteScroll.tsx @@ -0,0 +1,50 @@ +import { ComponentType, useEffect, useState } from 'react'; +import InfiniteScroll from 'react-infinite-scroll-component'; + +const initialNumberOfItemsDisplayed = 50; + +export interface InfiniteScrollItemsProps { + items: T[]; +} + +interface CustomInfiniteScrollProps { + allItems: T[]; + child: ComponentType>; +} + +// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types +const CustomInfiniteScroll = ({ allItems, child }: CustomInfiniteScrollProps) => { + const [items, setItems] = useState(allItems.slice(0, 50)); + const [index, setIndex] = useState(initialNumberOfItemsDisplayed); + const [hasMore, setHasMore] = useState(allItems.length > initialNumberOfItemsDisplayed); + const Child = child; + + useEffect(() => { + setItems(allItems.slice(0, initialNumberOfItemsDisplayed)); + setHasMore(allItems.length > initialNumberOfItemsDisplayed); + }, [allItems]); + + const fetchMoreData = () => { + if (items.length === allItems.length) { + setHasMore(false); + } else { + const newIndex = index + initialNumberOfItemsDisplayed; + setIndex(newIndex); + setItems(allItems.slice(0, newIndex)); + } + }; + + return ( + loading...} + dataLength={items.length} + height={items.length > 25 ? '75vh' : ''} + > + + + ); +}; + +export default CustomInfiniteScroll; diff --git a/web/ui/react-app/src/pages/serviceDiscovery/LabelsTable.tsx b/web/ui/react-app/src/pages/serviceDiscovery/LabelsTable.tsx index 01c657c3a0..6fb633db3d 100644 --- a/web/ui/react-app/src/pages/serviceDiscovery/LabelsTable.tsx +++ b/web/ui/react-app/src/pages/serviceDiscovery/LabelsTable.tsx @@ -2,6 +2,7 @@ import React, { FC, useState } from 'react'; import { Badge, Table } from 'reactstrap'; import { TargetLabels } from './Services'; import { ToggleMoreLess } from '../../components/ToggleMoreLess'; +import CustomInfiniteScroll, { InfiniteScrollItemsProps } from '../../components/CustomInfiniteScroll'; interface LabelProps { value: TargetLabels[]; @@ -20,6 +21,33 @@ const formatLabels = (labels: Record | string) => { }); }; +const LabelsTableContent: FC> = ({ items }) => { + return ( + + + + + + + + + {items.map((_, i) => { + return ( + + + {items[i].isDropped ? ( + + ) : ( + + )} + + ); + })} + +
Discovered LabelsTarget Labels
{formatLabels(items[i].discoveredLabels)}Dropped{formatLabels(items[i].labels)}
+ ); +}; + export const LabelsTable: FC = ({ value, name }) => { const [showMore, setShowMore] = useState(false); @@ -35,30 +63,7 @@ export const LabelsTable: FC = ({ value, name }) => { {name} - {showMore ? ( - - - - - - - - - {value.map((_, i) => { - return ( - - - {value[i].isDropped ? ( - - ) : ( - - )} - - ); - })} - -
Discovered LabelsTarget Labels
{formatLabels(value[i].discoveredLabels)}Dropped{formatLabels(value[i].labels)}
- ) : null} + {showMore ? : null} ); }; diff --git a/web/ui/react-app/src/pages/serviceDiscovery/Services.tsx b/web/ui/react-app/src/pages/serviceDiscovery/Services.tsx index 28d6db6dc0..ed90b7e3e1 100644 --- a/web/ui/react-app/src/pages/serviceDiscovery/Services.tsx +++ b/web/ui/react-app/src/pages/serviceDiscovery/Services.tsx @@ -1,4 +1,4 @@ -import React, { FC } from 'react'; +import React, { ChangeEvent, FC, useEffect, useState } from 'react'; import { useFetch } from '../../hooks/useFetch'; import { LabelsTable } from './LabelsTable'; import { DroppedTarget, Labels, Target } from '../targets/target'; @@ -7,6 +7,10 @@ import { withStatusIndicator } from '../../components/withStatusIndicator'; import { mapObjEntries } from '../../utils'; import { usePathPrefix } from '../../contexts/PathPrefixContext'; import { API_PATH } from '../../constants/constants'; +import { KVSearch } from '@nexucis/kvsearch'; +import { Container, Input, InputGroup, InputGroupAddon, InputGroupText } from 'reactstrap'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { faSearch } from '@fortawesome/free-solid-svg-icons'; interface ServiceMap { activeTargets: Target[]; @@ -19,6 +23,11 @@ export interface TargetLabels { isDropped: boolean; } +const kvSearch = new KVSearch({ + shouldSort: true, + indexedKeys: ['labels', 'discoveredLabels', ['discoveredLabels', /.*/], ['labels', /.*/]], +}); + export const processSummary = ( activeTargets: Target[], droppedTargets: DroppedTarget[] @@ -82,14 +91,41 @@ export const processTargets = (activeTargets: Target[], droppedTargets: DroppedT }; export const ServiceDiscoveryContent: FC = ({ activeTargets, droppedTargets }) => { - const targets = processSummary(activeTargets, droppedTargets); - const labels = processTargets(activeTargets, droppedTargets); + const [activeTargetList, setActiveTargetList] = useState(activeTargets); + const [targetList, setTargetList] = useState(processSummary(activeTargets, droppedTargets)); + const [labelList, setLabelList] = useState(processTargets(activeTargets, droppedTargets)); + + const handleSearchChange = (e: ChangeEvent) => { + if (e.target.value !== '') { + const result = kvSearch.filter(e.target.value.trim(), activeTargets); + setActiveTargetList( + result.map((value) => { + return value.original as unknown as Target; + }) + ); + } else { + setActiveTargetList(activeTargets); + } + }; + + useEffect(() => { + setTargetList(processSummary(activeTargetList, droppedTargets)); + setLabelList(processTargets(activeTargetList, droppedTargets)); + }, [activeTargetList, droppedTargets]); return ( <>

Service Discovery

+ + + + {} + + + +
- {mapObjEntries(labels, ([k, v]) => { + {mapObjEntries(labelList, ([k, v]) => { return ; })} diff --git a/web/ui/react-app/src/pages/targets/ScrapePoolContent.tsx b/web/ui/react-app/src/pages/targets/ScrapePoolContent.tsx index 8a55a09b16..e1cbce6fcb 100644 --- a/web/ui/react-app/src/pages/targets/ScrapePoolContent.tsx +++ b/web/ui/react-app/src/pages/targets/ScrapePoolContent.tsx @@ -1,6 +1,5 @@ -import React, { FC, useEffect, useState } from 'react'; +import React, { FC } from 'react'; import { getColor, Target } from './target'; -import InfiniteScroll from 'react-infinite-scroll-component'; import { Badge, Table } from 'reactstrap'; import TargetLabels from './TargetLabels'; import styles from './ScrapePoolPanel.module.css'; @@ -8,84 +7,61 @@ import { formatRelative } from '../../utils'; import { now } from 'moment'; import TargetScrapeDuration from './TargetScrapeDuration'; import EndpointLink from './EndpointLink'; +import CustomInfiniteScroll, { InfiniteScrollItemsProps } from '../../components/CustomInfiniteScroll'; const columns = ['Endpoint', 'State', 'Labels', 'Last Scrape', 'Scrape Duration', 'Error']; -const initialNumberOfTargetsDisplayed = 50; interface ScrapePoolContentProps { targets: Target[]; } -export const ScrapePoolContent: FC = ({ targets }) => { - const [items, setItems] = useState(targets.slice(0, 50)); - const [index, setIndex] = useState(initialNumberOfTargetsDisplayed); - const [hasMore, setHasMore] = useState(targets.length > initialNumberOfTargetsDisplayed); - - useEffect(() => { - setItems(targets.slice(0, initialNumberOfTargetsDisplayed)); - setHasMore(targets.length > initialNumberOfTargetsDisplayed); - }, [targets]); - - const fetchMoreData = () => { - if (items.length === targets.length) { - setHasMore(false); - } else { - const newIndex = index + initialNumberOfTargetsDisplayed; - setIndex(newIndex); - setItems(targets.slice(0, newIndex)); - } - }; - +const ScrapePoolContentTable: FC> = ({ items }) => { return ( - loading...} - dataLength={items.length} - height={items.length > 25 ? '75vh' : ''} - > - - - - {columns.map((column) => ( - - ))} - - - - {items.map((target, index) => ( - - - - - - - - +
{column}
- - - {target.health.toUpperCase()} - - - {formatRelative(target.lastScrape, now())} - - - {target.lastError ? {target.lastError} : null} -
+ + + {columns.map((column) => ( + ))} - -
{column}
- + + + + {items.map((target, index) => ( + + + + + + {target.health.toUpperCase()} + + + + + {formatRelative(target.lastScrape, now())} + + + + + {target.lastError ? {target.lastError} : null} + + + ))} + + ); }; + +export const ScrapePoolContent: FC = ({ targets }) => { + return ; +}; From dd02675467de8a6818d1b22770dbc859cdcf9641 Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Mon, 10 Jan 2022 23:26:24 +0100 Subject: [PATCH 007/180] move dependabot frequency running to monthly (#10144) Signed-off-by: Augustin Husson --- .github/dependabot.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 22791e7880..ce621be9c6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,17 +3,16 @@ updates: - package-ecosystem: "gomod" directory: "/" schedule: - interval: "weekly" + interval: "monthly" - package-ecosystem: "npm" directory: "/web/ui" - open-pull-requests-limit: 0 schedule: - interval: "weekly" + interval: "monthly" - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "monthly" - package-ecosystem: "docker" directory: "/" schedule: - interval: "weekly" + interval: "monthly" From 714bc3ff2973f97e9aa9652bb30e1903ccdbd171 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 10 Jan 2022 23:36:26 +0100 Subject: [PATCH 008/180] build(deps): bump google.golang.org/api from 0.63.0 to 0.64.0 (#10135) Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.63.0 to 0.64.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.63.0...v0.64.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 5b9095633e..3a8d250a88 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 golang.org/x/tools v0.1.8 - google.golang.org/api v0.63.0 + google.golang.org/api v0.64.0 google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb google.golang.org/protobuf v1.27.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 diff --git a/go.sum b/go.sum index fec2cfed30..cae70a0c07 100644 --- a/go.sum +++ b/go.sum @@ -1679,7 +1679,6 @@ golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1830,8 +1829,8 @@ google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqiv google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= -google.golang.org/api v0.63.0 h1:n2bqqK895ygnBpdPDYetfy23K7fJ22wsrZKCyfuRkkA= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.64.0 h1:l3pi8ncrQgB9+ncFw3A716L8lWujnXniBYbxWqqy6tE= +google.golang.org/api v0.64.0/go.mod h1:931CdxA8Rm4t6zqTFGSsgwbAEZ2+GMYurbndwSimebM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1907,7 +1906,6 @@ google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= -google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb h1:ZrsicilzPCS/Xr8qtBZZLpy4P9TYXAfl49ctG1/5tgw= google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= From 61509fc8400be7418869343d1338beae8db2e9a7 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Wed, 5 Jan 2022 15:50:16 +0100 Subject: [PATCH 009/180] PromQL: Promote negative offset and @ modifer to stable Following the argument that breaking the invariant that PromQL does not look ahead of the evaluation time implies a breaking change, we still need to keep the feature flag around, but at least we can communicate that the feature is considered stable, and that the feature flags will be ignored from v3 on. Signed-off-by: beorn7 --- cmd/prometheus/main.go | 4 ++-- docs/feature_flags.md | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index eaa97fe32e..9b662249b3 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -168,10 +168,10 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { switch o { case "promql-at-modifier": c.enablePromQLAtModifier = true - level.Info(logger).Log("msg", "Experimental promql-at-modifier enabled") + level.Info(logger).Log("msg", "promql-at-modifier enabled") case "promql-negative-offset": c.enablePromQLNegativeOffset = true - level.Info(logger).Log("msg", "Experimental promql-negative-offset enabled") + level.Info(logger).Log("msg", "promql-negative-offset enabled") case "remote-write-receiver": c.web.EnableRemoteWriteReceiver = true level.Warn(logger).Log("msg", "Remote write receiver enabled via feature flag remote-write-receiver. This is DEPRECATED. Use --web.enable-remote-write-receiver.") diff --git a/docs/feature_flags.md b/docs/feature_flags.md index fd09c40886..33d8221350 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -18,6 +18,11 @@ They may be enabled by default in future versions. The `@` modifier lets you specify the evaluation time for instant vector selectors, range vector selectors, and subqueries. More details can be found [here](querying/basics.md#modifier). +This feature is considered stable, but since it breaks the invariant that +PromQL does not look ahead of the evaluation time, it still needs to be enabled +via this feature flag. The feature will be permanently enabled (and the feature +flag ignored) upon major release v3. + ## Expand environment variables in external labels `--enable-feature=expand-external-labels` @@ -40,6 +45,11 @@ with more recent data. More details can be found [here](querying/basics.md#offset-modifier). +This feature is considered stable, but since it breaks the invariant that +PromQL does not look ahead of the evaluation time, it still needs to be enabled +via this feature flag. The feature will be permanently enabled (and the feature +flag ignored) upon major release v3. + ## Remote Write Receiver `--enable-feature=remote-write-receiver` From 129ed4ec8baf32f9bccd6a73508db45dbec9cbf5 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Tue, 11 Jan 2022 17:24:03 +0530 Subject: [PATCH 010/180] Fix Example() function in TSDB (#10153) * Fix Example() function in TSDB Signed-off-by: Ganesh Vernekar * Fix tests Signed-off-by: Ganesh Vernekar --- tsdb/chunks/chunk_write_queue.go | 19 ++++++++ tsdb/chunks/chunk_write_queue_test.go | 27 ++--------- tsdb/chunks/head_chunks.go | 4 ++ tsdb/db_test.go | 4 ++ tsdb/example_test.go | 66 ++++++++++++--------------- 5 files changed, 60 insertions(+), 60 deletions(-) diff --git a/tsdb/chunks/chunk_write_queue.go b/tsdb/chunks/chunk_write_queue.go index ce9b94d85b..5cdd2e81f0 100644 --- a/tsdb/chunks/chunk_write_queue.go +++ b/tsdb/chunks/chunk_write_queue.go @@ -163,3 +163,22 @@ func (c *chunkWriteQueue) stop() { c.workerWg.Wait() } + +func (c *chunkWriteQueue) queueIsEmpty() bool { + return c.queueSize() == 0 +} + +func (c *chunkWriteQueue) queueIsFull() bool { + // When the queue is full and blocked on the writer the chunkRefMap has one more job than the cap of the jobCh + // because one job is currently being processed and blocked in the writer. + return c.queueSize() == cap(c.jobs)+1 +} + +func (c *chunkWriteQueue) queueSize() int { + c.chunkRefMapMtx.Lock() + defer c.chunkRefMapMtx.Unlock() + + // Looking at chunkRefMap instead of jobCh because the job is popped from the chan before it has + // been fully processed, it remains in the chunkRefMap until the processing is complete. + return len(c.chunkRefMap) +} diff --git a/tsdb/chunks/chunk_write_queue_test.go b/tsdb/chunks/chunk_write_queue_test.go index 251c96395a..0e971a336b 100644 --- a/tsdb/chunks/chunk_write_queue_test.go +++ b/tsdb/chunks/chunk_write_queue_test.go @@ -146,7 +146,7 @@ func TestChunkWriteQueue_WrappingAroundSizeLimit(t *testing.T) { } // The queue should be full. - require.True(t, queueIsFull(q)) + require.True(t, q.queueIsFull()) // Adding another job should block as long as no job from the queue gets consumed. addedJob := atomic.NewBool(false) @@ -166,19 +166,19 @@ func TestChunkWriteQueue_WrappingAroundSizeLimit(t *testing.T) { require.Eventually(t, func() bool { return addedJob.Load() }, time.Second, time.Millisecond*10) // The queue should be full again. - require.True(t, queueIsFull(q)) + require.True(t, q.queueIsFull()) // Consume +1 jobs from the queue. // To drain the queue we need to consume +1 jobs because 1 job // is already in the state of being processed. for job := 0; job < sizeLimit+1; job++ { - require.False(t, queueIsEmpty(q)) + require.False(t, q.queueIsEmpty()) unblockChunkWriter() } // Wait until all jobs have been processed. callbackWg.Wait() - require.True(t, queueIsEmpty(q)) + require.True(t, q.queueIsEmpty()) } func TestChunkWriteQueue_HandlerErrorViaCallback(t *testing.T) { @@ -266,22 +266,3 @@ func BenchmarkChunkWriteQueue_addJob(b *testing.B) { }) } } - -func queueIsEmpty(q *chunkWriteQueue) bool { - return queueSize(q) == 0 -} - -func queueIsFull(q *chunkWriteQueue) bool { - // When the queue is full and blocked on the writer the chunkRefMap has one more job than the cap of the jobCh - // because one job is currently being processed and blocked in the writer. - return queueSize(q) == cap(q.jobs)+1 -} - -func queueSize(q *chunkWriteQueue) int { - q.chunkRefMapMtx.Lock() - defer q.chunkRefMapMtx.Unlock() - - // Looking at chunkRefMap instead of jobCh because the job is popped from the chan before it has - // been fully processed, it remains in the chunkRefMap until the processing is complete. - return len(q.chunkRefMap) -} diff --git a/tsdb/chunks/head_chunks.go b/tsdb/chunks/head_chunks.go index 11c175cbf7..1b5845808a 100644 --- a/tsdb/chunks/head_chunks.go +++ b/tsdb/chunks/head_chunks.go @@ -473,6 +473,10 @@ func (cdm *ChunkDiskMapper) CutNewFile() { cdm.evtlPos.cutFileOnNextChunk() } +func (cdm *ChunkDiskMapper) IsQueueEmpty() bool { + return cdm.writeQueue.queueIsEmpty() +} + // cutAndExpectRef creates a new m-mapped file. // The write lock should be held before calling this. // It ensures that the position in the new file matches the given chunk reference, if not then it errors. diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 45d2325bae..d94c387b34 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -1461,6 +1461,10 @@ func TestSizeRetention(t *testing.T) { } require.NoError(t, headApp.Commit()) + require.Eventually(t, func() bool { + return db.Head().chunkDiskMapper.IsQueueEmpty() + }, 2*time.Second, 100*time.Millisecond) + // Test that registered size matches the actual disk size. require.NoError(t, db.reloadBlocks()) // Reload the db to register the new db size. require.Equal(t, len(blocks), len(db.Blocks())) // Ensure all blocks are registered. diff --git a/tsdb/example_test.go b/tsdb/example_test.go index 9af341e1ec..1947a48a55 100644 --- a/tsdb/example_test.go +++ b/tsdb/example_test.go @@ -16,97 +16,89 @@ package tsdb import ( "context" "fmt" + "io/ioutil" "math" - "testing" + "os" "time" - "github.com/stretchr/testify/require" - "github.com/prometheus/prometheus/model/labels" ) -func TestExample(t *testing.T) { +func Example() { // Create a random dir to work in. Open() doesn't require a pre-existing dir, but // we want to make sure not to make a mess where we shouldn't. - dir := t.TempDir() + dir, err := ioutil.TempDir("", "tsdb-test") + noErr(err) // Open a TSDB for reading and/or writing. db, err := Open(dir, nil, nil, DefaultOptions(), nil) - require.NoError(t, err) + noErr(err) // Open an appender for writing. app := db.Appender(context.Background()) - lbls := labels.FromStrings("foo", "bar") - var appendedSamples []sample + series := labels.FromStrings("foo", "bar") // Ref is 0 for the first append since we don't know the reference for the series. - ts, v := time.Now().Unix(), 123.0 - ref, err := app.Append(0, lbls, ts, v) - require.NoError(t, err) - appendedSamples = append(appendedSamples, sample{ts, v}) + ref, err := app.Append(0, series, time.Now().Unix(), 123) + noErr(err) // Another append for a second later. // Re-using the ref from above since it's the same series, makes append faster. time.Sleep(time.Second) - ts, v = time.Now().Unix(), 124 - _, err = app.Append(ref, lbls, ts, v) - require.NoError(t, err) - appendedSamples = append(appendedSamples, sample{ts, v}) + _, err = app.Append(ref, series, time.Now().Unix(), 124) + noErr(err) // Commit to storage. err = app.Commit() - require.NoError(t, err) + noErr(err) // In case you want to do more appends after app.Commit(), // you need a new appender. - // app = db.Appender(context.Background()) - // + app = db.Appender(context.Background()) // ... adding more samples. - // - // Commit to storage. - // err = app.Commit() - // require.NoError(t, err) // Open a querier for reading. querier, err := db.Querier(context.Background(), math.MinInt64, math.MaxInt64) - require.NoError(t, err) - + noErr(err) ss := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - var queriedSamples []sample + for ss.Next() { series := ss.At() fmt.Println("series:", series.Labels().String()) it := series.Iterator() for it.Next() { - ts, v := it.At() - fmt.Println("sample", ts, v) - queriedSamples = append(queriedSamples, sample{ts, v}) + _, v := it.At() // We ignore the timestamp here, only to have a predictable output we can test against (below) + fmt.Println("sample", v) } - require.NoError(t, it.Err()) fmt.Println("it.Err():", it.Err()) } - require.NoError(t, ss.Err()) fmt.Println("ss.Err():", ss.Err()) ws := ss.Warnings() if len(ws) > 0 { fmt.Println("warnings:", ws) } err = querier.Close() - require.NoError(t, err) + noErr(err) // Clean up any last resources when done. err = db.Close() - require.NoError(t, err) - - require.Equal(t, appendedSamples, queriedSamples) + noErr(err) + err = os.RemoveAll(dir) + noErr(err) // Output: // series: {foo="bar"} - // sample 123 - // sample 124 + // sample 123 + // sample 124 // it.Err(): // ss.Err(): } + +func noErr(err error) { + if err != nil { + panic(err) + } +} From 076109fa1910ad2198bf2c447a174fee31114982 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 11 Jan 2022 15:56:25 +0100 Subject: [PATCH 011/180] build(deps): bump github.com/aws/aws-sdk-go from 1.42.28 to 1.42.31 (#10146) Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.42.28 to 1.42.31. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.42.28...v1.42.31) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3a8d250a88..8f19a4edbe 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 - github.com/aws/aws-sdk-go v1.42.28 + github.com/aws/aws-sdk-go v1.42.31 github.com/cespare/xxhash/v2 v2.1.2 github.com/containerd/containerd v1.5.7 // indirect github.com/dennwc/varint v1.0.0 diff --git a/go.sum b/go.sum index cae70a0c07..fa026e4806 100644 --- a/go.sum +++ b/go.sum @@ -188,8 +188,8 @@ github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZve github.com/aws/aws-sdk-go v1.34.28/go.mod h1:H7NKnBqNVzoTJpGfLrQkkD+ytBA93eiDYi/+8rV9s48= github.com/aws/aws-sdk-go v1.38.35/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go v1.40.11/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= -github.com/aws/aws-sdk-go v1.42.28 h1:YJtgL7IGSN41saY4JLW08jya5nU0vGcuAeAa1OL2M6c= -github.com/aws/aws-sdk-go v1.42.28/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= +github.com/aws/aws-sdk-go v1.42.31 h1:tSv/YzjrFlbSqWmov9quBxrSNXLPUjJI7nPEB57S1+M= +github.com/aws/aws-sdk-go v1.42.31/go.mod h1:OGr6lGMAKGlG9CVrYnWYDKIyb829c6EVBRjxqjmPepc= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= From b39f2739e5b01560ad8299d2579f1041a0e9ae5f Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 11 Jan 2022 17:01:02 +0100 Subject: [PATCH 012/180] PromQL: Always enable negative offset and @ modifier This follows the line of argument that the invariant of not looking ahead of the query time was merely emerging behavior and not a documented stable feature. Any query that looks ahead of the query time was simply invalid before the introduction of the negative offset and the @ modifier. Signed-off-by: beorn7 --- cmd/prometheus/main.go | 16 ++++++---------- cmd/promtool/main.go | 15 +++++++++------ docs/feature_flags.md | 31 ------------------------------- docs/querying/basics.md | 10 +++------- go.mod | 2 +- go.sum | 4 ++-- promql/engine.go | 11 +++++++++-- promql/test.go | 4 +++- web/api/v1/api.go | 10 ---------- 9 files changed, 33 insertions(+), 70 deletions(-) diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 9b662249b3..9f4c500715 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -149,8 +149,6 @@ type flagConfig struct { featureList []string // These options are extracted from featureList // for ease of use. - enablePromQLAtModifier bool - enablePromQLNegativeOffset bool enableExpandExternalLabels bool enableNewSDManager bool @@ -166,12 +164,6 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { opts := strings.Split(f, ",") for _, o := range opts { switch o { - case "promql-at-modifier": - c.enablePromQLAtModifier = true - level.Info(logger).Log("msg", "promql-at-modifier enabled") - case "promql-negative-offset": - c.enablePromQLNegativeOffset = true - level.Info(logger).Log("msg", "promql-negative-offset enabled") case "remote-write-receiver": c.web.EnableRemoteWriteReceiver = true level.Warn(logger).Log("msg", "Remote write receiver enabled via feature flag remote-write-receiver. This is DEPRECATED. Use --web.enable-remote-write-receiver.") @@ -195,6 +187,8 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error { level.Info(logger).Log("msg", "Experimental agent mode enabled.") case "": continue + case "promql-at-modifier", "promql-negative-offset": + level.Warn(logger).Log("msg", "This option for --enable-feature is now permanently enabled and therefore a no-op.", "option", o) default: level.Warn(logger).Log("msg", "Unknown option for --enable-feature", "option", o) } @@ -570,8 +564,10 @@ func main() { ActiveQueryTracker: promql.NewActiveQueryTracker(localStoragePath, cfg.queryConcurrency, log.With(logger, "component", "activeQueryTracker")), LookbackDelta: time.Duration(cfg.lookbackDelta), NoStepSubqueryIntervalFn: noStepSubqueryInterval.Get, - EnableAtModifier: cfg.enablePromQLAtModifier, - EnableNegativeOffset: cfg.enablePromQLNegativeOffset, + // EnableAtModifier and EnableNegativeOffset have to be + // always on for regular PromQL as of Prometheus v2.33. + EnableAtModifier: true, + EnableNegativeOffset: true, } queryEngine = promql.NewEngine(opts) diff --git a/cmd/promtool/main.go b/cmd/promtool/main.go index a8098bea1d..bc1b2928a8 100644 --- a/cmd/promtool/main.go +++ b/cmd/promtool/main.go @@ -203,17 +203,14 @@ func main() { p = &promqlPrinter{} } - var queryOpts promql.LazyLoaderOpts for _, f := range *featureList { opts := strings.Split(f, ",") for _, o := range opts { switch o { - case "promql-at-modifier": - queryOpts.EnableAtModifier = true - case "promql-negative-offset": - queryOpts.EnableNegativeOffset = true case "": continue + case "promql-at-modifier", "promql-negative-offset": + fmt.Printf(" WARNING: Option for --enable-feature is a no-op after promotion to a stable feature: %q\n", o) default: fmt.Printf(" WARNING: Unknown option for --enable-feature: %q\n", o) } @@ -258,7 +255,13 @@ func main() { os.Exit(QueryLabels(*queryLabelsServer, *queryLabelsName, *queryLabelsBegin, *queryLabelsEnd, p)) case testRulesCmd.FullCommand(): - os.Exit(RulesUnitTest(queryOpts, *testRulesFiles...)) + os.Exit(RulesUnitTest( + promql.LazyLoaderOpts{ + EnableAtModifier: true, + EnableNegativeOffset: true, + }, + *testRulesFiles...), + ) case tsdbBenchWriteCmd.FullCommand(): os.Exit(checkErr(benchmarkWrite(*benchWriteOutPath, *benchSamplesFile, *benchWriteNumMetrics, *benchWriteNumScrapes))) diff --git a/docs/feature_flags.md b/docs/feature_flags.md index 33d8221350..cb5b6fa2f8 100644 --- a/docs/feature_flags.md +++ b/docs/feature_flags.md @@ -11,18 +11,6 @@ Their behaviour can change in future releases which will be communicated via the You can enable them using the `--enable-feature` flag with a comma separated list of features. They may be enabled by default in future versions. -## `@` Modifier in PromQL - -`--enable-feature=promql-at-modifier` - -The `@` modifier lets you specify the evaluation time for instant vector selectors, -range vector selectors, and subqueries. More details can be found [here](querying/basics.md#modifier). - -This feature is considered stable, but since it breaks the invariant that -PromQL does not look ahead of the evaluation time, it still needs to be enabled -via this feature flag. The feature will be permanently enabled (and the feature -flag ignored) upon major release v3. - ## Expand environment variables in external labels `--enable-feature=expand-external-labels` @@ -31,25 +19,6 @@ Replace `${var}` or `$var` in the [`external_labels`](configuration/configuratio values according to the values of the current environment variables. References to undefined variables are replaced by the empty string. -## Negative offset in PromQL - -This negative offset is disabled by default since it breaks the invariant -that PromQL does not look ahead of the evaluation time for samples. - -`--enable-feature=promql-negative-offset` - -In contrast to the positive offset modifier, the negative offset modifier lets -one shift a vector selector into the future. An example in which one may want -to use a negative offset is reviewing past data and making temporal comparisons -with more recent data. - -More details can be found [here](querying/basics.md#offset-modifier). - -This feature is considered stable, but since it breaks the invariant that -PromQL does not look ahead of the evaluation time, it still needs to be enabled -via this feature flag. The feature will be permanently enabled (and the feature -flag ignored) upon major release v3. - ## Remote Write Receiver `--enable-feature=remote-write-receiver` diff --git a/docs/querying/basics.md b/docs/querying/basics.md index f4c176ed7d..8a74f0df56 100644 --- a/docs/querying/basics.md +++ b/docs/querying/basics.md @@ -209,9 +209,7 @@ can be specified: rate(http_requests_total[5m] offset -1w) -This feature is enabled by setting `--enable-feature=promql-negative-offset` -flag. See [feature flags](../feature_flags.md) for more details about -this flag. +Note that this allows a query to look ahead of its evaluation time. ### @ modifier @@ -249,10 +247,6 @@ These 2 queries will produce the same result. # offset before @ http_requests_total offset 5m @ 1609746000 -This modifier is disabled by default since it breaks the invariant that PromQL -does not look ahead of the evaluation time for samples. It can be enabled by setting -`--enable-feature=promql-at-modifier` flag. See [feature flags](../feature_flags.md) for more details about this flag. - Additionally, `start()` and `end()` can also be used as values for the `@` modifier as special values. For a range query, they resolve to the start and end of the range query respectively and remain the same for all steps. @@ -262,6 +256,8 @@ For an instant query, `start()` and `end()` both resolve to the evaluation time. http_requests_total @ start() rate(http_requests_total[5m] @ end()) +Note that the `@` modifier allows a query to look ahead of its evaluation time. + ## Subquery Subquery allows you to run an instant query for a given range and resolution. The result of a subquery is a range vector. diff --git a/go.mod b/go.mod index 3a8d250a88..cdc9eae836 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 - golang.org/x/tools v0.1.8 + golang.org/x/tools v0.1.9-0.20211209172050-90a85b2969be google.golang.org/api v0.64.0 google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb google.golang.org/protobuf v1.27.1 diff --git a/go.sum b/go.sum index cae70a0c07..5d8a7b229a 100644 --- a/go.sum +++ b/go.sum @@ -1785,8 +1785,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.8 h1:P1HhGGuLW4aAclzjtmJdf0mJOjVUZUzOTqkAkWL+l6w= -golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/tools v0.1.9-0.20211209172050-90a85b2969be h1:JRBiPXZpZ1FsceyPRRme0vX394zXC3xlhqu705k9nzM= +golang.org/x/tools v0.1.9-0.20211209172050-90a85b2969be/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/promql/engine.go b/promql/engine.go index 9b0a328797..8fe2fe2e4b 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -222,10 +222,17 @@ type EngineOpts struct { // a subquery in milliseconds if no step in range vector was specified `[30m:]`. NoStepSubqueryIntervalFn func(rangeMillis int64) int64 - // EnableAtModifier if true enables @ modifier. Disabled otherwise. + // EnableAtModifier if true enables @ modifier. Disabled otherwise. This + // is supposed to be enabled for regular PromQL (as of Prometheus v2.33) + // but the option to disable it is still provided here for those using + // the Engine outside of Prometheus. EnableAtModifier bool - // EnableNegativeOffset if true enables negative (-) offset values. Disabled otherwise. + // EnableNegativeOffset if true enables negative (-) offset + // values. Disabled otherwise. This is supposed to be enabled for + // regular PromQL (as of Prometheus v2.33) but the option to disable it + // is still provided here for those using the Engine outside of + // Prometheus. EnableNegativeOffset bool } diff --git a/promql/test.go b/promql/test.go index 19b2dd8300..5f3707b28e 100644 --- a/promql/test.go +++ b/promql/test.go @@ -680,7 +680,9 @@ type LazyLoader struct { // LazyLoaderOpts are options for the lazy loader. type LazyLoaderOpts struct { - // Disabled PromQL engine features. + // Both of these must be set to true for regular PromQL (as of + // Prometheus v2.33). They can still be disabled here for legacy and + // other uses. EnableAtModifier, EnableNegativeOffset bool } diff --git a/web/api/v1/api.go b/web/api/v1/api.go index e6a6daffca..79d357cdd7 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -378,11 +378,6 @@ func (api *API) query(r *http.Request) (result apiFuncResult) { } qry, err := api.QueryEngine.NewInstantQuery(api.Queryable, r.FormValue("query"), ts) - if err == promql.ErrValidationAtModifierDisabled { - err = errors.New("@ modifier is disabled, use --enable-feature=promql-at-modifier to enable it") - } else if err == promql.ErrValidationNegativeOffsetDisabled { - err = errors.New("negative offset is disabled, use --enable-feature=promql-negative-offset to enable it") - } if err != nil { return invalidParamError(err, "query") } @@ -458,11 +453,6 @@ func (api *API) queryRange(r *http.Request) (result apiFuncResult) { } qry, err := api.QueryEngine.NewRangeQuery(api.Queryable, r.FormValue("query"), start, end, step) - if err == promql.ErrValidationAtModifierDisabled { - err = errors.New("@ modifier is disabled, use --enable-feature=promql-at-modifier to enable it") - } else if err == promql.ErrValidationNegativeOffsetDisabled { - err = errors.New("negative offset is disabled, use --enable-feature=promql-negative-offset to enable it") - } if err != nil { return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil} } From c2b80d86437fc7766d65fd4f6b7503f0dd618be7 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Tue, 11 Jan 2022 18:23:40 +0100 Subject: [PATCH 013/180] PromQL: Test negative offset in PromQL tests Signed-off-by: beorn7 --- promql/test.go | 1 + promql/testdata/at_modifier.test | 8 ++++++++ promql/testdata/selectors.test | 6 ++++++ 3 files changed, 15 insertions(+) diff --git a/promql/test.go b/promql/test.go index 5f3707b28e..1da225a3df 100644 --- a/promql/test.go +++ b/promql/test.go @@ -613,6 +613,7 @@ func (t *Test) clear() { Timeout: 100 * time.Second, NoStepSubqueryIntervalFn: func(int64) int64 { return durationMilliseconds(1 * time.Minute) }, EnableAtModifier: true, + EnableNegativeOffset: true, } t.queryEngine = NewEngine(opts) diff --git a/promql/testdata/at_modifier.test b/promql/testdata/at_modifier.test index 18cd93a4ba..3ba6afc493 100644 --- a/promql/testdata/at_modifier.test +++ b/promql/testdata/at_modifier.test @@ -18,6 +18,14 @@ eval instant at 10s metric offset 50s @ 100 metric{job="1"} 5 metric{job="2"} 10 +eval instant at 10s metric @ 0 offset -50s + metric{job="1"} 5 + metric{job="2"} 10 + +eval instant at 10s metric offset -50s @ 0 + metric{job="1"} 5 + metric{job="2"} 10 + eval instant at 10s -metric @ 100 {job="1"} -10 {job="2"} -20 diff --git a/promql/testdata/selectors.test b/promql/testdata/selectors.test index c4309957b0..6742d83e99 100644 --- a/promql/testdata/selectors.test +++ b/promql/testdata/selectors.test @@ -29,6 +29,12 @@ eval instant at 18000s rate(http_requests{instance!="3"}[1m] offset 10000s) {job="api-server", instance="0", group="canary"} 3 {job="api-server", instance="1", group="canary"} 4 +eval instant at 4000s rate(http_requests{instance!="3"}[1m] offset -4000s) + {job="api-server", instance="0", group="production"} 1 + {job="api-server", instance="1", group="production"} 2 + {job="api-server", instance="0", group="canary"} 3 + {job="api-server", instance="1", group="canary"} 4 + eval instant at 18000s rate(http_requests[40s]) - rate(http_requests[1m] offset 10000s) {job="api-server", instance="0", group="production"} 2 {job="api-server", instance="1", group="production"} 1 From 2f4289a3bfaca725a14f41e30f1c91de92936242 Mon Sep 17 00:00:00 2001 From: Abhishek Date: Wed, 12 Jan 2022 09:14:22 +0530 Subject: [PATCH 014/180] add limit to the rules api (#10152) --- docs/querying/api.md | 1 + web/api/v1/api.go | 2 ++ web/api/v1/api_test.go | 3 +++ 3 files changed, 6 insertions(+) diff --git a/docs/querying/api.md b/docs/querying/api.md index 84d8c8f962..e3a8af7872 100644 --- a/docs/querying/api.md +++ b/docs/querying/api.md @@ -620,6 +620,7 @@ $ curl http://localhost:9090/api/v1/rules ], "file": "/rules.yaml", "interval": 60, + "limit": 0, "name": "example" } ] diff --git a/web/api/v1/api.go b/web/api/v1/api.go index e6a6daffca..25918f791a 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -1164,6 +1164,7 @@ type RuleGroup struct { // same array. Rules []Rule `json:"rules"` Interval float64 `json:"interval"` + Limit int `json:"limit"` EvaluationTime float64 `json:"evaluationTime"` LastEvaluation time.Time `json:"lastEvaluation"` } @@ -1216,6 +1217,7 @@ func (api *API) rules(r *http.Request) apiFuncResult { Name: grp.Name(), File: grp.File(), Interval: grp.Interval().Seconds(), + Limit: grp.Limit(), Rules: []Rule{}, EvaluationTime: grp.GetEvaluationTime().Seconds(), LastEvaluation: grp.GetLastEvaluation(), diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index 74857ae93d..5de0d60f89 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -1470,6 +1470,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E Name: "grp", File: "/path/to/file", Interval: 1, + Limit: 0, Rules: []Rule{ AlertingRule{ State: "inactive", @@ -1516,6 +1517,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E Name: "grp", File: "/path/to/file", Interval: 1, + Limit: 0, Rules: []Rule{ AlertingRule{ State: "inactive", @@ -1555,6 +1557,7 @@ func testEndpoints(t *testing.T, api *API, tr *testTargetRetriever, es storage.E Name: "grp", File: "/path/to/file", Interval: 1, + Limit: 0, Rules: []Rule{ RecordingRule{ Name: "recording-rule-1", From b9bc8e7c330afc09c7926c83ddf0ee4f21764371 Mon Sep 17 00:00:00 2001 From: Augustin Husson Date: Wed, 12 Jan 2022 16:20:22 +0100 Subject: [PATCH 015/180] create a component to handle the search bar with debounce Signed-off-by: Augustin Husson --- web/ui/react-app/src/components/SearchBar.tsx | 31 +++++++++++++++++++ .../src/pages/serviceDiscovery/Services.tsx | 12 ++----- .../src/pages/targets/ScrapePoolList.tsx | 12 ++----- 3 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 web/ui/react-app/src/components/SearchBar.tsx diff --git a/web/ui/react-app/src/components/SearchBar.tsx b/web/ui/react-app/src/components/SearchBar.tsx new file mode 100644 index 0000000000..761b290684 --- /dev/null +++ b/web/ui/react-app/src/components/SearchBar.tsx @@ -0,0 +1,31 @@ +import React, { ChangeEvent, FC } from 'react'; +import { Input, InputGroup, InputGroupAddon, InputGroupText } from 'reactstrap'; +import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; +import { faSearch } from '@fortawesome/free-solid-svg-icons'; + +export interface SearchBarProps { + handleChange: (e: ChangeEvent) => void; + placeholder: string; +} + +const SearchBar: FC = ({ handleChange, placeholder }) => { + let filterTimeout: NodeJS.Timeout; + + const handleSearchChange = (e: ChangeEvent) => { + clearTimeout(filterTimeout); + filterTimeout = setTimeout(() => { + handleChange(e); + }, 300); + }; + + return ( + + + {} + + + + ); +}; + +export default SearchBar; diff --git a/web/ui/react-app/src/pages/serviceDiscovery/Services.tsx b/web/ui/react-app/src/pages/serviceDiscovery/Services.tsx index ed90b7e3e1..9f407ac322 100644 --- a/web/ui/react-app/src/pages/serviceDiscovery/Services.tsx +++ b/web/ui/react-app/src/pages/serviceDiscovery/Services.tsx @@ -8,9 +8,8 @@ import { mapObjEntries } from '../../utils'; import { usePathPrefix } from '../../contexts/PathPrefixContext'; import { API_PATH } from '../../constants/constants'; import { KVSearch } from '@nexucis/kvsearch'; -import { Container, Input, InputGroup, InputGroupAddon, InputGroupText } from 'reactstrap'; -import { FontAwesomeIcon } from '@fortawesome/react-fontawesome'; -import { faSearch } from '@fortawesome/free-solid-svg-icons'; +import { Container } from 'reactstrap'; +import SearchBar from '../../components/SearchBar'; interface ServiceMap { activeTargets: Target[]; @@ -117,12 +116,7 @@ export const ServiceDiscoveryContent: FC = ({ activeTargets, dropped <>

Service Discovery

- - - {} - - - +