From f9d21f10ecd2a343a381044f131ea4e46381ce09 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Thu, 4 Jun 2020 16:00:37 +0100 Subject: [PATCH 01/31] Only relabelling should apply for scrape_samples_scraped_post_relabelling. (#7342) More consistent variable names. Fixes #7298 Signed-off-by: Brian Brazil --- scrape/scrape.go | 14 +++++++------- scrape/scrape_test.go | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scrape/scrape.go b/scrape/scrape.go index d1987e97c6..4c6d3bab00 100644 --- a/scrape/scrape.go +++ b/scrape/scrape.go @@ -1140,7 +1140,7 @@ loop: if ok { err = app.AddFast(ce.ref, t, v) - sampleAdded, err = sl.checkAddError(ce, met, tp, err, &sampleLimitErr, &appErrs) + _, err = sl.checkAddError(ce, met, tp, err, &sampleLimitErr, &appErrs) // In theory this should never happen. if err == storage.ErrNotFound { ok = false @@ -1187,10 +1187,10 @@ loop: } } - // Increment added even if there's a sampleLimitErr so we correctly report the number of samples scraped. - if sampleAdded || sampleLimitErr != nil { - added++ - } + // Increment added even if there's an error so we correctly report the + // number of samples remaining after relabelling. + added++ + } if sampleLimitErr != nil { if err == nil { @@ -1275,7 +1275,7 @@ const ( scrapeSeriesAddedMetricName = "scrape_series_added" + "\xff" ) -func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, appended, seriesAdded int, scrapeErr error) (err error) { +func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, added, seriesAdded int, scrapeErr error) (err error) { sl.scraper.Report(start, duration, scrapeErr) ts := timestamp.FromTime(start) @@ -1302,7 +1302,7 @@ func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, a if err = sl.addReportSample(app, scrapeSamplesMetricName, ts, float64(scraped)); err != nil { return } - if err = sl.addReportSample(app, samplesPostRelabelMetricName, ts, float64(appended)); err != nil { + if err = sl.addReportSample(app, samplesPostRelabelMetricName, ts, float64(added)); err != nil { return } if err = sl.addReportSample(app, scrapeSeriesAddedMetricName, ts, float64(seriesAdded)); err != nil { diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index 3225876441..34cd026802 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -1340,7 +1340,7 @@ func TestScrapeLoopAppendGracefullyIfAmendOrOutOfOrderOrOutOfBounds(t *testing.T } testutil.Equals(t, want, app.result, "Appended samples not as expected") testutil.Equals(t, 4, total) - testutil.Equals(t, 1, added) + testutil.Equals(t, 4, added) testutil.Equals(t, 1, seriesAdded) } @@ -1365,7 +1365,7 @@ func TestScrapeLoopOutOfBoundsTimeError(t *testing.T) { now := time.Now().Add(20 * time.Minute) total, added, seriesAdded, err := sl.append([]byte("normal 1\n"), "", now) testutil.Equals(t, 1, total) - testutil.Equals(t, 0, added) + testutil.Equals(t, 1, added) testutil.Equals(t, 0, seriesAdded) testutil.Ok(t, err) From 82c7cd320a8a2933c353d661a26a5ab61b9e84c1 Mon Sep 17 00:00:00 2001 From: Bert Hartmann Date: Thu, 4 Jun 2020 15:54:47 -0400 Subject: [PATCH 02/31] increase the remote write bucket range (#7323) * increase the remote write bucket range Increase the range of remote write buckets to capture times above 10s for laggy scenarios Buckets had been: {.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} Buckets are now: {0.03125, 0.0625, 0.125, 0.25, 0.5, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512} Signed-off-by: Bert Hartmann * revert back to DefBuckets with addons to be backwards compatible Signed-off-by: Bert Hartmann * shuffle the buckets to maintain 2-2.5x increases Signed-off-by: Bert Hartmann --- storage/remote/queue_manager.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/remote/queue_manager.go b/storage/remote/queue_manager.go index 6a8b4c4325..befa9a7b7b 100644 --- a/storage/remote/queue_manager.go +++ b/storage/remote/queue_manager.go @@ -115,7 +115,7 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager Subsystem: subsystem, Name: "sent_batch_duration_seconds", Help: "Duration of sample batch send calls to the remote storage.", - Buckets: prometheus.DefBuckets, + Buckets: append(prometheus.DefBuckets, 25, 60, 120, 300), ConstLabels: constLabels, }) m.highestSentTimestamp = &maxGauge{ From 6ff4814a492ae6f9e3f5cf7b022a823b631b7a95 Mon Sep 17 00:00:00 2001 From: Hu Shuai Date: Fri, 5 Jun 2020 16:48:33 +0800 Subject: [PATCH 03/31] Add some unit tests for pkg/labels/labels.go (#7347) This PR is about adding some unit tests for pkg/labels/labels.go. Signed-off-by: Hu Shuai --- pkg/labels/labels_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/labels/labels_test.go b/pkg/labels/labels_test.go index 7bffff8afd..51faea985b 100644 --- a/pkg/labels/labels_test.go +++ b/pkg/labels/labels_test.go @@ -531,3 +531,16 @@ func TestLabels_Copy(t *testing.T) { func TestLabels_Map(t *testing.T) { testutil.Equals(t, map[string]string{"aaa": "111", "bbb": "222"}, Labels{{"aaa", "111"}, {"bbb", "222"}}.Map()) } + +func TestLabels_WithLabels(t *testing.T) { + testutil.Equals(t, Labels{{"aaa", "111"}, {"bbb", "222"}}, Labels{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}}.WithLabels("aaa", "bbb")) +} + +func TestLabels_WithoutLabels(t *testing.T) { + testutil.Equals(t, Labels{{"aaa", "111"}}, Labels{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}}.WithoutLabels("bbb", "ccc")) + testutil.Equals(t, Labels{{"aaa", "111"}}, Labels{{"aaa", "111"}, {"bbb", "222"}, {MetricName, "333"}}.WithoutLabels("bbb")) +} + +func TestLabels_FromStrings(t *testing.T) { + testutil.Equals(t, Labels{{"aaa", "111"}, {"bbb", "222"}}, FromStrings("aaa", "111", "bbb", "222")) +} From 4284dd1f1b8ec701673d65e4d12db1df59851061 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Fri, 5 Jun 2020 19:51:25 +0200 Subject: [PATCH 04/31] promql: cleanup: use errors.As (#7351) This was TODO because circleci was not in go1.13 yet. Signed-off-by: Julien Pivotto --- promql/engine_test.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/promql/engine_test.go b/promql/engine_test.go index a202a0bf92..e0a83aa1d8 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -19,7 +19,6 @@ import ( "io/ioutil" "os" "sort" - "strings" "testing" "time" @@ -117,9 +116,7 @@ func TestQueryTimeout(t *testing.T) { testutil.NotOk(t, res.Err, "expected timeout error but got none") var e ErrQueryTimeout - // TODO: when circleci-windows moves to go 1.13: - // testutil.Assert(t, errors.As(res.Err, &e), "expected timeout error but got: %s", res.Err) - testutil.Assert(t, strings.HasPrefix(res.Err.Error(), e.Error()), "expected timeout error but got: %s", res.Err) + testutil.Assert(t, errors.As(res.Err, &e), "expected timeout error but got: %s", res.Err) } const errQueryCanceled = ErrQueryCanceled("test statement execution") @@ -499,9 +496,7 @@ func TestEngineShutdown(t *testing.T) { testutil.NotOk(t, res2.Err, "expected error on querying with canceled context but got none") var e ErrQueryCanceled - // TODO: when circleci-windows moves to go 1.13: - // testutil.Assert(t, errors.As(res2.Err, &e), "expected cancellation error but got: %s", res2.Err) - testutil.Assert(t, strings.HasPrefix(res2.Err.Error(), e.Error()), "expected cancellation error but got: %s", res2.Err) + testutil.Assert(t, errors.As(res2.Err, &e), "expected cancellation error but got: %s", res2.Err) } func TestEngineEvalStmtTimestamps(t *testing.T) { From e963d953eac7d5181024eba2215740835b65c0a9 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Sun, 7 Jun 2020 17:56:31 +0200 Subject: [PATCH 05/31] Update yarn.lock (#7356) Fixes websocket-extensions security warning. Signed-off-by: Ben Kochie --- web/ui/react-app/yarn.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/ui/react-app/yarn.lock b/web/ui/react-app/yarn.lock index e77c9c7360..41b4aba543 100644 --- a/web/ui/react-app/yarn.lock +++ b/web/ui/react-app/yarn.lock @@ -11222,9 +11222,9 @@ websocket-driver@>=0.5.1: websocket-extensions ">=0.1.1" websocket-extensions@>=0.1.1: - version "0.1.3" - resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.3.tgz#5d2ff22977003ec687a4b87073dfbbac146ccf29" - integrity sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg== + version "0.1.4" + resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.4.tgz#7f8473bc839dfd87608adb95d7eb075211578a42" + integrity sha512-OqedPIGOfsDlo31UNwYbCFMSaO9m9G/0faIHj5/dZFDMFqPTcx6UwqyOy3COEaEOg/9VsGIpdqn62W5KhoKSpg== whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3, whatwg-encoding@^1.0.5: version "1.0.5" From e5ffa8c9a08a5ee4185271c8c26051ddc1388b7a Mon Sep 17 00:00:00 2001 From: Hu Shuai Date: Mon, 8 Jun 2020 15:46:21 +0800 Subject: [PATCH 06/31] Add some unit tests for pkg/labels/labels.go (#7358) This PR is about adding some unit tests for pkg/labels/labels.go. Signed-off-by: Hu Shuai --- pkg/labels/labels_test.go | 68 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) diff --git a/pkg/labels/labels_test.go b/pkg/labels/labels_test.go index 51faea985b..00d3f6a0a0 100644 --- a/pkg/labels/labels_test.go +++ b/pkg/labels/labels_test.go @@ -544,3 +544,71 @@ func TestLabels_WithoutLabels(t *testing.T) { func TestLabels_FromStrings(t *testing.T) { testutil.Equals(t, Labels{{"aaa", "111"}, {"bbb", "222"}}, FromStrings("aaa", "111", "bbb", "222")) } + +func TestBulider_NewBulider(t *testing.T) { + testutil.Equals( + t, + &Builder{ + base: Labels{{"aaa", "111"}}, + del: []string{}, + add: []Label{}, + }, + NewBuilder(Labels{{"aaa", "111"}}), + ) +} + +func TestBuilder_Del(t *testing.T) { + testutil.Equals( + t, + &Builder{ + del: []string{"bbb"}, + add: []Label{{"aaa", "111"}, {"ccc", "333"}}, + }, + (&Builder{ + del: []string{}, + add: []Label{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}}, + }).Del("bbb"), + ) +} + +func TestBuilder_Set(t *testing.T) { + testutil.Equals( + t, + &Builder{ + base: Labels{{"aaa", "111"}}, + del: []string{}, + add: []Label{{"bbb", "222"}}, + }, + (&Builder{ + base: Labels{{"aaa", "111"}}, + del: []string{}, + add: []Label{}, + }).Set("bbb", "222"), + ) + + testutil.Equals( + t, + &Builder{ + base: Labels{{"aaa", "111"}}, + del: []string{}, + add: []Label{{"bbb", "333"}}, + }, + (&Builder{ + base: Labels{{"aaa", "111"}}, + del: []string{}, + add: []Label{{"bbb", "222"}}, + }).Set("bbb", "333"), + ) +} + +func TestBuilder_Labels(t *testing.T) { + testutil.Equals( + t, + Labels{{"aaa", "111"}, {"ccc", "333"}, {"ddd", "444"}}, + (&Builder{ + base: Labels{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}}, + del: []string{"bbb"}, + add: []Label{{"ddd", "444"}}, + }).Labels(), + ) +} From bc1d7d7f270bf0c4553d3a972bcba72a3acf712a Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Mon, 8 Jun 2020 20:55:53 +0200 Subject: [PATCH 07/31] Fixed incorrect query results caused by buffer reuse in merge adapter (#7361) Signed-off-by: Marco Pracucci --- storage/fanout_test.go | 15 +++++++++++---- storage/generic.go | 14 ++++++-------- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/storage/fanout_test.go b/storage/fanout_test.go index cc839b6ed0..a5d04ea8f6 100644 --- a/storage/fanout_test.go +++ b/storage/fanout_test.go @@ -195,10 +195,18 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { } qs = append(qs, tc.extraQueriers...) - merged, _, _ := NewMergeQuerier(qs[0], qs, ChainedSeriesMerge).Select(false, nil) - for merged.Next() { + mergedQuerier, _, _ := NewMergeQuerier(qs[0], qs, ChainedSeriesMerge).Select(false, nil) + + // Get all merged series upfront to make sure there are no incorrectly retained shared + // buffers causing bugs. + var mergedSeries []Series + for mergedQuerier.Next() { + mergedSeries = append(mergedSeries, mergedQuerier.At()) + } + testutil.Ok(t, mergedQuerier.Err()) + + for _, actualSeries := range mergedSeries { testutil.Assert(t, tc.expected.Next(), "Expected Next() to be true") - actualSeries := merged.At() expectedSeries := tc.expected.At() testutil.Equals(t, expectedSeries.Labels(), actualSeries.Labels()) @@ -207,7 +215,6 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { testutil.Equals(t, expErr, actErr) testutil.Equals(t, expSmpl, actSmpl) } - testutil.Ok(t, merged.Err()) testutil.Assert(t, !tc.expected.Next(), "Expected Next() to be false") }) } diff --git a/storage/generic.go b/storage/generic.go index a5a694cb20..d23e51a6c6 100644 --- a/storage/generic.go +++ b/storage/generic.go @@ -108,26 +108,24 @@ func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matche type seriesMergerAdapter struct { VerticalSeriesMergeFunc - buf []Series } func (a *seriesMergerAdapter) Merge(s ...Labels) Labels { - a.buf = a.buf[:0] + buf := make([]Series, 0, len(s)) for _, ser := range s { - a.buf = append(a.buf, ser.(Series)) + buf = append(buf, ser.(Series)) } - return a.VerticalSeriesMergeFunc(a.buf...) + return a.VerticalSeriesMergeFunc(buf...) } type chunkSeriesMergerAdapter struct { VerticalChunkSeriesMergerFunc - buf []ChunkSeries } func (a *chunkSeriesMergerAdapter) Merge(s ...Labels) Labels { - a.buf = a.buf[:0] + buf := make([]ChunkSeries, 0, len(s)) for _, ser := range s { - a.buf = append(a.buf, ser.(ChunkSeries)) + buf = append(buf, ser.(ChunkSeries)) } - return a.VerticalChunkSeriesMergerFunc(a.buf...) + return a.VerticalChunkSeriesMergerFunc(buf...) } From 0f34535009dfae68ed68d5ee8a27e0f9b666c9ac Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 9 Jun 2020 05:47:53 +0100 Subject: [PATCH 08/31] Cut 2.18.2 + cherry pick of query bugfix. Signed-off-by: Bartlomiej Plotka --- CHANGELOG.md | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aec464e7fc..623fa8f73f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.18.2 / 2020-06-09 + +* [BUGFIX] TSDB: Fix incorrect query results when using Prometheus with remote reads configured #7361 + ## 2.18.1 / 2020-05-07 * [BUGFIX] TSDB: Fixed snapshot API. #7217 diff --git a/VERSION b/VERSION index a36e9b0906..61dc3342f0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.18.1 +2.18.2 From 76261150b7249860333559ec95c6fb16fbcdc379 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 9 Jun 2020 09:52:36 +0100 Subject: [PATCH 09/31] Cut 2.18.2 release with query bug fix. (#7364) * Fixed incorrect query results caused by buffer reuse in merge adapter (#7361) Signed-off-by: Marco Pracucci * Cut 2.18.2 + cherry pick of query bugfix. Signed-off-by: Bartlomiej Plotka Co-authored-by: Marco Pracucci --- CHANGELOG.md | 4 ++++ VERSION | 2 +- storage/fanout_test.go | 15 +++++++++++---- storage/generic.go | 14 ++++++-------- 4 files changed, 22 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aec464e7fc..623fa8f73f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## 2.18.2 / 2020-06-09 + +* [BUGFIX] TSDB: Fix incorrect query results when using Prometheus with remote reads configured #7361 + ## 2.18.1 / 2020-05-07 * [BUGFIX] TSDB: Fixed snapshot API. #7217 diff --git a/VERSION b/VERSION index a36e9b0906..61dc3342f0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.18.1 +2.18.2 diff --git a/storage/fanout_test.go b/storage/fanout_test.go index cc839b6ed0..a5d04ea8f6 100644 --- a/storage/fanout_test.go +++ b/storage/fanout_test.go @@ -195,10 +195,18 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { } qs = append(qs, tc.extraQueriers...) - merged, _, _ := NewMergeQuerier(qs[0], qs, ChainedSeriesMerge).Select(false, nil) - for merged.Next() { + mergedQuerier, _, _ := NewMergeQuerier(qs[0], qs, ChainedSeriesMerge).Select(false, nil) + + // Get all merged series upfront to make sure there are no incorrectly retained shared + // buffers causing bugs. + var mergedSeries []Series + for mergedQuerier.Next() { + mergedSeries = append(mergedSeries, mergedQuerier.At()) + } + testutil.Ok(t, mergedQuerier.Err()) + + for _, actualSeries := range mergedSeries { testutil.Assert(t, tc.expected.Next(), "Expected Next() to be true") - actualSeries := merged.At() expectedSeries := tc.expected.At() testutil.Equals(t, expectedSeries.Labels(), actualSeries.Labels()) @@ -207,7 +215,6 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { testutil.Equals(t, expErr, actErr) testutil.Equals(t, expSmpl, actSmpl) } - testutil.Ok(t, merged.Err()) testutil.Assert(t, !tc.expected.Next(), "Expected Next() to be false") }) } diff --git a/storage/generic.go b/storage/generic.go index a5a694cb20..d23e51a6c6 100644 --- a/storage/generic.go +++ b/storage/generic.go @@ -108,26 +108,24 @@ func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matche type seriesMergerAdapter struct { VerticalSeriesMergeFunc - buf []Series } func (a *seriesMergerAdapter) Merge(s ...Labels) Labels { - a.buf = a.buf[:0] + buf := make([]Series, 0, len(s)) for _, ser := range s { - a.buf = append(a.buf, ser.(Series)) + buf = append(buf, ser.(Series)) } - return a.VerticalSeriesMergeFunc(a.buf...) + return a.VerticalSeriesMergeFunc(buf...) } type chunkSeriesMergerAdapter struct { VerticalChunkSeriesMergerFunc - buf []ChunkSeries } func (a *chunkSeriesMergerAdapter) Merge(s ...Labels) Labels { - a.buf = a.buf[:0] + buf := make([]ChunkSeries, 0, len(s)) for _, ser := range s { - a.buf = append(a.buf, ser.(ChunkSeries)) + buf = append(buf, ser.(ChunkSeries)) } - return a.VerticalChunkSeriesMergerFunc(a.buf...) + return a.VerticalChunkSeriesMergerFunc(buf...) } From 1a4c97ec51a3b8add09c80765663856acc876057 Mon Sep 17 00:00:00 2001 From: Bartlomiej Plotka Date: Tue, 9 Jun 2020 09:56:28 +0100 Subject: [PATCH 10/31] Revert "Cut 2.18.2 release with query bug fix. (#7364)" (#7367) This reverts commit 76261150b7249860333559ec95c6fb16fbcdc379. --- CHANGELOG.md | 4 ---- VERSION | 2 +- storage/fanout_test.go | 15 ++++----------- storage/generic.go | 14 ++++++++------ 4 files changed, 13 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 623fa8f73f..aec464e7fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,3 @@ -## 2.18.2 / 2020-06-09 - -* [BUGFIX] TSDB: Fix incorrect query results when using Prometheus with remote reads configured #7361 - ## 2.18.1 / 2020-05-07 * [BUGFIX] TSDB: Fixed snapshot API. #7217 diff --git a/VERSION b/VERSION index 61dc3342f0..a36e9b0906 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.18.2 +2.18.1 diff --git a/storage/fanout_test.go b/storage/fanout_test.go index a5d04ea8f6..cc839b6ed0 100644 --- a/storage/fanout_test.go +++ b/storage/fanout_test.go @@ -195,18 +195,10 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { } qs = append(qs, tc.extraQueriers...) - mergedQuerier, _, _ := NewMergeQuerier(qs[0], qs, ChainedSeriesMerge).Select(false, nil) - - // Get all merged series upfront to make sure there are no incorrectly retained shared - // buffers causing bugs. - var mergedSeries []Series - for mergedQuerier.Next() { - mergedSeries = append(mergedSeries, mergedQuerier.At()) - } - testutil.Ok(t, mergedQuerier.Err()) - - for _, actualSeries := range mergedSeries { + merged, _, _ := NewMergeQuerier(qs[0], qs, ChainedSeriesMerge).Select(false, nil) + for merged.Next() { testutil.Assert(t, tc.expected.Next(), "Expected Next() to be true") + actualSeries := merged.At() expectedSeries := tc.expected.At() testutil.Equals(t, expectedSeries.Labels(), actualSeries.Labels()) @@ -215,6 +207,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { testutil.Equals(t, expErr, actErr) testutil.Equals(t, expSmpl, actSmpl) } + testutil.Ok(t, merged.Err()) testutil.Assert(t, !tc.expected.Next(), "Expected Next() to be false") }) } diff --git a/storage/generic.go b/storage/generic.go index d23e51a6c6..a5a694cb20 100644 --- a/storage/generic.go +++ b/storage/generic.go @@ -108,24 +108,26 @@ func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matche type seriesMergerAdapter struct { VerticalSeriesMergeFunc + buf []Series } func (a *seriesMergerAdapter) Merge(s ...Labels) Labels { - buf := make([]Series, 0, len(s)) + a.buf = a.buf[:0] for _, ser := range s { - buf = append(buf, ser.(Series)) + a.buf = append(a.buf, ser.(Series)) } - return a.VerticalSeriesMergeFunc(buf...) + return a.VerticalSeriesMergeFunc(a.buf...) } type chunkSeriesMergerAdapter struct { VerticalChunkSeriesMergerFunc + buf []ChunkSeries } func (a *chunkSeriesMergerAdapter) Merge(s ...Labels) Labels { - buf := make([]ChunkSeries, 0, len(s)) + a.buf = a.buf[:0] for _, ser := range s { - buf = append(buf, ser.(ChunkSeries)) + a.buf = append(a.buf, ser.(ChunkSeries)) } - return a.VerticalChunkSeriesMergerFunc(buf...) + return a.VerticalChunkSeriesMergerFunc(a.buf...) } From 66dfb951c4ca2c1dd3f266172a48a925403b13a5 Mon Sep 17 00:00:00 2001 From: Kemal Akkoyun Date: Tue, 9 Jun 2020 18:57:31 +0200 Subject: [PATCH 11/31] *: Consistent Error/Warning handling for SeriesSet iterator: Allowing Async Select (#7251) * Add errors and Warnings to SeriesSet Signed-off-by: Kemal Akkoyun * Change Querier interface and refactor accordingly Signed-off-by: Kemal Akkoyun * Refactor promql/engine to propagate warnings at eval stage Signed-off-by: Kemal Akkoyun * Address review issues Signed-off-by: Kemal Akkoyun * Make sure all the series from all Selects are pre-advanced Signed-off-by: Kemal Akkoyun * Address review issues Signed-off-by: Kemal Akkoyun * Separate merge series sets Signed-off-by: Kemal Akkoyun * Clean Signed-off-by: Kemal Akkoyun * Refactor merge querier failure handling Signed-off-by: Kemal Akkoyun * Refactored and simplified fanout with improvements from incoming chunk iterator PRs. * Secondary logic is hidden, instead of weird failed series set logic we had. * Fanout is well commented * Fanout closing record all errors * MergeQuerier improved API (clearer) * deferredGenericMergeSeriesSet is not needed as we return no samples anyway for failed series sets (next = false). Signed-off-by: Bartlomiej Plotka * Fix formatting Signed-off-by: Kemal Akkoyun * Fix CI issues Signed-off-by: Kemal Akkoyun * Added final tests for error handling. Signed-off-by: Bartlomiej Plotka * Addressed Brian's comments. * Moved hints in populate to be allocated only when needed. * Used sync.Once in secondary Querier to achieve all-or-nothing partial response logic. * Select after first Next is done will panic. NOTE: in lazySeriesSet in theory we could just panic, I think however we can totally just return error, it will panic in expand anyway. Signed-off-by: Bartlomiej Plotka * Utilize errWithWarnings Signed-off-by: Kemal Akkoyun * Fix recently introduced expansion issue Signed-off-by: Kemal Akkoyun * Add tests for secondary querier error handling Signed-off-by: Kemal Akkoyun * Implement lazy merge Signed-off-by: Kemal Akkoyun * Add name to test cases Signed-off-by: Kemal Akkoyun * Reorganize Signed-off-by: Kemal Akkoyun * Address review comments Signed-off-by: Kemal Akkoyun * Address review comments Signed-off-by: Kemal Akkoyun * Remove redundant warnings Signed-off-by: Kemal Akkoyun * Fix rebase mistake Signed-off-by: Kemal Akkoyun Co-authored-by: Bartlomiej Plotka --- promql/engine.go | 274 +++++++++++++------------- promql/engine_test.go | 52 +++-- promql/test_test.go | 3 +- rules/manager.go | 18 +- rules/manager_test.go | 12 +- scrape/scrape_test.go | 12 +- storage/fanout.go | 310 ++++++++++++++---------------- storage/fanout/fanout_test.go | 25 +-- storage/fanout_test.go | 348 +++++++++++++++++++++++++++++++--- storage/generic.go | 31 +-- storage/interface.go | 27 ++- storage/lazy.go | 67 +++++++ storage/noop.go | 12 +- storage/remote/codec.go | 28 +-- storage/remote/read.go | 20 +- storage/remote/read_test.go | 30 ++- storage/remote/storage.go | 3 + storage/secondary.go | 112 +++++++++++ tsdb/block_test.go | 4 +- tsdb/cmd/tsdb/main.go | 21 +- tsdb/db_test.go | 90 ++++----- tsdb/head_test.go | 54 +++--- tsdb/querier.go | 67 ++++--- tsdb/querier_bench_test.go | 3 +- tsdb/querier_test.go | 38 ++-- web/api/v1/api.go | 45 ++--- web/api/v1/api_test.go | 8 +- web/federate.go | 15 +- 28 files changed, 1113 insertions(+), 616 deletions(-) create mode 100644 storage/lazy.go create mode 100644 storage/secondary.go diff --git a/promql/engine.go b/promql/engine.go index c7f02b7ae7..bc2cfc1da5 100644 --- a/promql/engine.go +++ b/promql/engine.go @@ -29,7 +29,7 @@ import ( "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" - opentracing "github.com/opentracing/opentracing-go" + "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -418,7 +418,7 @@ func (ng *Engine) newTestQuery(f func(context.Context) error) Query { // // At this point per query only one EvalStmt is evaluated. Alert and record // statements are not handled by the Engine. -func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, w storage.Warnings, err error) { +func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws storage.Warnings, err error) { ng.metrics.currentQueries.Inc() defer ng.metrics.currentQueries.Dec() @@ -517,13 +517,9 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval } defer querier.Close() - warnings, err := ng.populateSeries(ctxPrepare, querier, s) + ng.populateSeries(querier, s) prepareSpanTimer.Finish() - if err != nil { - return nil, warnings, err - } - evalSpanTimer, ctxInnerEval := query.stats.GetSpanTimer(ctx, stats.InnerEvalTime, ng.metrics.queryInnerEval) // Instant evaluation. This is executed as a range evaluation with one step. if s.Start == s.End && s.Interval == 0 { @@ -539,7 +535,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval lookbackDelta: ng.lookbackDelta, } - val, err := evaluator.Eval(s.Expr) + val, warnings, err := evaluator.Eval(s.Expr) if err != nil { return nil, warnings, err } @@ -588,7 +584,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval logger: ng.logger, lookbackDelta: ng.lookbackDelta, } - val, err := evaluator.Eval(s.Expr) + val, warnings, err := evaluator.Eval(s.Expr) if err != nil { return nil, warnings, err } @@ -649,35 +645,29 @@ func (ng *Engine) findMinTime(s *parser.EvalStmt) time.Time { return s.Start.Add(-maxOffset) } -func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s *parser.EvalStmt) (storage.Warnings, error) { - var ( - // Whenever a MatrixSelector is evaluated, evalRange is set to the corresponding range. - // The evaluation of the VectorSelector inside then evaluates the given range and unsets - // the variable. - evalRange time.Duration - warnings storage.Warnings - err error - ) +func (ng *Engine) populateSeries(querier storage.Querier, s *parser.EvalStmt) { + // Whenever a MatrixSelector is evaluated, evalRange is set to the corresponding range. + // The evaluation of the VectorSelector inside then evaluates the given range and unsets + // the variable. + var evalRange time.Duration parser.Inspect(s.Expr, func(node parser.Node, path []parser.Node) error { - var set storage.SeriesSet - var wrn storage.Warnings - hints := &storage.SelectHints{ - Start: timestamp.FromTime(s.Start), - End: timestamp.FromTime(s.End), - Step: durationToInt64Millis(s.Interval), - } - - // We need to make sure we select the timerange selected by the subquery. - // TODO(gouthamve): cumulativeSubqueryOffset gives the sum of range and the offset - // we can optimise it by separating out the range and offsets, and subtracting the offsets - // from end also. - subqOffset := ng.cumulativeSubqueryOffset(path) - offsetMilliseconds := durationMilliseconds(subqOffset) - hints.Start = hints.Start - offsetMilliseconds - switch n := node.(type) { case *parser.VectorSelector: + hints := &storage.SelectHints{ + Start: timestamp.FromTime(s.Start), + End: timestamp.FromTime(s.End), + Step: durationToInt64Millis(s.Interval), + } + + // We need to make sure we select the timerange selected by the subquery. + // TODO(gouthamve): cumulativeSubqueryOffset gives the sum of range and the offset + // we can optimise it by separating out the range and offsets, and subtracting the offsets + // from end also. + subqOffset := ng.cumulativeSubqueryOffset(path) + offsetMilliseconds := durationMilliseconds(subqOffset) + hints.Start = hints.Start - offsetMilliseconds + if evalRange == 0 { hints.Start = hints.Start - durationMilliseconds(ng.lookbackDelta) } else { @@ -696,20 +686,12 @@ func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s hints.End = hints.End - offsetMilliseconds } - set, wrn, err = querier.Select(false, hints, n.LabelMatchers...) - warnings = append(warnings, wrn...) - if err != nil { - level.Error(ng.logger).Log("msg", "error selecting series set", "err", err) - return err - } - n.UnexpandedSeriesSet = set - + n.UnexpandedSeriesSet = querier.Select(false, hints, n.LabelMatchers...) case *parser.MatrixSelector: evalRange = n.Range } return nil }) - return warnings, err } // extractFuncFromPath walks up the path and searches for the first instance of @@ -743,34 +725,40 @@ func extractGroupsFromPath(p []parser.Node) (bool, []string) { return false, nil } -func checkForSeriesSetExpansion(ctx context.Context, expr parser.Expr) { +func checkAndExpandSeriesSet(ctx context.Context, expr parser.Expr) (storage.Warnings, error) { switch e := expr.(type) { case *parser.MatrixSelector: - checkForSeriesSetExpansion(ctx, e.VectorSelector) + return checkAndExpandSeriesSet(ctx, e.VectorSelector) case *parser.VectorSelector: - if e.Series == nil { - series, err := expandSeriesSet(ctx, e.UnexpandedSeriesSet) - if err != nil { - panic(err) - } else { - e.Series = series - } + if e.Series != nil { + return nil, nil } + series, ws, err := expandSeriesSet(ctx, e.UnexpandedSeriesSet) + e.Series = series + return ws, err } + return nil, nil } -func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.Series, err error) { +func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.Series, ws storage.Warnings, err error) { for it.Next() { select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, nil, ctx.Err() default: } res = append(res, it.At()) } - return res, it.Err() + return res, it.Warnings(), it.Err() } +type errWithWarnings struct { + err error + warnings storage.Warnings +} + +func (e errWithWarnings) Error() string { return e.err.Error() } + // An evaluator evaluates given expressions over given fixed timestamps. It // is attached to an engine through which it connects to a querier and reports // errors. On timeout or cancellation of its context it terminates. @@ -799,26 +787,33 @@ func (ev *evaluator) error(err error) { } // recover is the handler that turns panics into returns from the top level of evaluation. -func (ev *evaluator) recover(errp *error) { +func (ev *evaluator) recover(ws *storage.Warnings, errp *error) { e := recover() if e == nil { return } - if err, ok := e.(runtime.Error); ok { + + switch err := e.(type) { + case runtime.Error: // Print the stack trace but do not inhibit the running application. buf := make([]byte, 64<<10) buf = buf[:runtime.Stack(buf, false)] level.Error(ev.logger).Log("msg", "runtime panic in parser", "err", e, "stacktrace", string(buf)) *errp = errors.Wrap(err, "unexpected error") - } else { + case errWithWarnings: + *errp = err.err + *ws = append(*ws, err.warnings...) + default: *errp = e.(error) } } -func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, err error) { - defer ev.recover(&err) - return ev.eval(expr), nil +func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, ws storage.Warnings, err error) { + defer ev.recover(&ws, &err) + + v, ws = ev.eval(expr) + return v, ws, nil } // EvalNodeHelper stores extra information and caches for evaluating a single node across steps. @@ -884,17 +879,20 @@ func (enh *EvalNodeHelper) signatureFunc(on bool, names ...string) func(labels.L // the given function with the values computed for each expression at that // step. The return value is the combination into time series of all the // function call results. -func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, exprs ...parser.Expr) Matrix { +func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) (Vector, storage.Warnings), exprs ...parser.Expr) (Matrix, storage.Warnings) { numSteps := int((ev.endTimestamp-ev.startTimestamp)/ev.interval) + 1 matrixes := make([]Matrix, len(exprs)) origMatrixes := make([]Matrix, len(exprs)) originalNumSamples := ev.currentSamples + var warnings storage.Warnings for i, e := range exprs { // Functions will take string arguments from the expressions, not the values. if e != nil && e.Type() != parser.ValueTypeString { // ev.currentSamples will be updated to the correct value within the ev.eval call. - matrixes[i] = ev.eval(e).(Matrix) + val, ws := ev.eval(e) + warnings = append(warnings, ws...) + matrixes[i] = val.(Matrix) // Keep a copy of the original point slices so that they // can be returned to the pool. @@ -946,11 +944,12 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e } // Make the function call. enh.ts = ts - result := f(args, enh) + result, ws := f(args, enh) if result.ContainsSameLabelset() { ev.errorf("vector cannot contain metrics with the same labelset") } enh.out = result[:0] // Reuse result vector. + warnings = append(warnings, ws...) ev.currentSamples += len(result) // When we reset currentSamples to tempNumSamples during the next iteration of the loop it also @@ -969,7 +968,7 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e mat[i] = Series{Metric: s.Metric, Points: []Point{s.Point}} } ev.currentSamples = originalNumSamples + mat.TotalSamples() - return mat + return mat, warnings } // Add samples in output vector to output series. @@ -1001,29 +1000,30 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e mat = append(mat, ss) } ev.currentSamples = originalNumSamples + mat.TotalSamples() - return mat + return mat, warnings } // evalSubquery evaluates given SubqueryExpr and returns an equivalent // evaluated MatrixSelector in its place. Note that the Name and LabelMatchers are not set. -func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) *parser.MatrixSelector { - val := ev.eval(subq).(Matrix) +func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) (*parser.MatrixSelector, storage.Warnings) { + val, ws := ev.eval(subq) + mat := val.(Matrix) vs := &parser.VectorSelector{ Offset: subq.Offset, - Series: make([]storage.Series, 0, len(val)), + Series: make([]storage.Series, 0, len(mat)), } ms := &parser.MatrixSelector{ Range: subq.Range, VectorSelector: vs, } - for _, s := range val { + for _, s := range mat { vs.Series = append(vs.Series, NewStorageSeries(s)) } - return ms + return ms, ws } // eval evaluates the given expression as the given AST expression node requires. -func (ev *evaluator) eval(expr parser.Expr) parser.Value { +func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) { // This is the top-level evaluation method. // Thus, we check for timeout/cancellation here. if err := contextDone(ev.ctx, "expression evaluation"); err != nil { @@ -1035,16 +1035,16 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { case *parser.AggregateExpr: unwrapParenExpr(&e.Param) if s, ok := e.Param.(*parser.StringLiteral); ok { - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.aggregation(e.Op, e.Grouping, e.Without, s.Val, v[0].(Vector), enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.aggregation(e.Op, e.Grouping, e.Without, s.Val, v[0].(Vector), enh), nil }, e.Expr) } - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { var param float64 if e.Param != nil { param = v[0].(Vector)[0].V } - return ev.aggregation(e.Op, e.Grouping, e.Without, param, v[1].(Vector), enh) + return ev.aggregation(e.Op, e.Grouping, e.Without, param, v[1].(Vector), enh), nil }, e.Param, e.Expr) case *parser.Call: @@ -1056,15 +1056,19 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { // a vector selector. vs, ok := e.Args[0].(*parser.VectorSelector) if ok { - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return call([]parser.Value{ev.vectorSelector(vs, enh.ts)}, e.Args, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + val, ws := ev.vectorSelector(vs, enh.ts) + return call([]parser.Value{val}, e.Args, enh), ws }) } } // Check if the function has a matrix argument. - var matrixArgIndex int - var matrixArg bool + var ( + matrixArgIndex int + matrixArg bool + warnings storage.Warnings + ) for i := range e.Args { unwrapParenExpr(&e.Args[i]) a := e.Args[i] @@ -1078,14 +1082,16 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { matrixArgIndex = i matrixArg = true // Replacing parser.SubqueryExpr with parser.MatrixSelector. - e.Args[i] = ev.evalSubquery(subq) + val, ws := ev.evalSubquery(subq) + e.Args[i] = val + warnings = append(warnings, ws...) break } } if !matrixArg { // Does not have a matrix argument. - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return call(v, e.Args, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return call(v, e.Args, enh), warnings }, e.Args...) } @@ -1095,16 +1101,22 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { otherInArgs := make([]Vector, len(e.Args)) for i, e := range e.Args { if i != matrixArgIndex { - otherArgs[i] = ev.eval(e).(Matrix) + val, ws := ev.eval(e) + otherArgs[i] = val.(Matrix) otherInArgs[i] = Vector{Sample{}} inArgs[i] = otherInArgs[i] + warnings = append(warnings, ws...) } } sel := e.Args[matrixArgIndex].(*parser.MatrixSelector) selVS := sel.VectorSelector.(*parser.VectorSelector) - checkForSeriesSetExpansion(ev.ctx, sel) + ws, err := checkAndExpandSeriesSet(ev.ctx, sel) + warnings = append(warnings, ws...) + if err != nil { + ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), warnings}) + } mat := make(Matrix, 0, len(selVS.Series)) // Output matrix. offset := durationMilliseconds(selVS.Offset) selRange := durationMilliseconds(sel.Range) @@ -1182,7 +1194,7 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { // Iterate once to look for a complete series. for _, s := range mat { if len(s.Points) == steps { - return Matrix{} + return Matrix{}, warnings } } @@ -1193,7 +1205,7 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { found[p.T] = struct{}{} } if i > 0 && len(found) == steps { - return Matrix{} + return Matrix{}, warnings } } @@ -1209,20 +1221,21 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { Metric: createLabelsForAbsentFunction(e.Args[0]), Points: newp, }, - } + }, warnings } if mat.ContainsSameLabelset() { ev.errorf("vector cannot contain metrics with the same labelset") } - return mat + return mat, warnings case *parser.ParenExpr: return ev.eval(e.Expr) case *parser.UnaryExpr: - mat := ev.eval(e.Expr).(Matrix) + val, ws := ev.eval(e.Expr) + mat := val.(Matrix) if e.Op == parser.SUB { for i := range mat { mat[i].Metric = dropMetricName(mat[i].Metric) @@ -1234,53 +1247,56 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { ev.errorf("vector cannot contain metrics with the same labelset") } } - return mat + return mat, ws case *parser.BinaryExpr: switch lt, rt := e.LHS.Type(), e.RHS.Type(); { case lt == parser.ValueTypeScalar && rt == parser.ValueTypeScalar: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { val := scalarBinop(e.Op, v[0].(Vector)[0].Point.V, v[1].(Vector)[0].Point.V) - return append(enh.out, Sample{Point: Point{V: val}}) + return append(enh.out, Sample{Point: Point{V: val}}), nil }, e.LHS, e.RHS) case lt == parser.ValueTypeVector && rt == parser.ValueTypeVector: switch e.Op { case parser.LAND: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorAnd(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorAnd(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil }, e.LHS, e.RHS) case parser.LOR: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorOr(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorOr(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil }, e.LHS, e.RHS) case parser.LUNLESS: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorUnless(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorUnless(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil }, e.LHS, e.RHS) default: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, enh), nil }, e.LHS, e.RHS) } case lt == parser.ValueTypeVector && rt == parser.ValueTypeScalar: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].Point.V}, false, e.ReturnBool, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].Point.V}, false, e.ReturnBool, enh), nil }, e.LHS, e.RHS) case lt == parser.ValueTypeScalar && rt == parser.ValueTypeVector: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].Point.V}, true, e.ReturnBool, enh) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].Point.V}, true, e.ReturnBool, enh), nil }, e.LHS, e.RHS) } case *parser.NumberLiteral: - return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector { - return append(enh.out, Sample{Point: Point{V: e.Val}}) + return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) { + return append(enh.out, Sample{Point: Point{V: e.Val}}), nil }) case *parser.VectorSelector: - checkForSeriesSetExpansion(ev.ctx, e) + ws, err := checkAndExpandSeriesSet(ev.ctx, e) + if err != nil { + ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws}) + } mat := make(Matrix, 0, len(e.Series)) it := storage.NewBuffer(durationMilliseconds(ev.lookbackDelta)) for i, s := range e.Series { @@ -1307,9 +1323,8 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { } else { putPointSlice(ss.Points) } - } - return mat + return mat, ws case *parser.MatrixSelector: if ev.startTimestamp != ev.endTimestamp { @@ -1342,11 +1357,11 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value { newEv.startTimestamp += newEv.interval } - res := newEv.eval(e.Expr) + res, ws := newEv.eval(e.Expr) ev.currentSamples = newEv.currentSamples - return res + return res, ws case *parser.StringLiteral: - return String{V: e.Val, T: ev.startTimestamp} + return String{V: e.Val, T: ev.startTimestamp}, nil } panic(errors.Errorf("unhandled expression of type: %T", expr)) @@ -1357,13 +1372,12 @@ func durationToInt64Millis(d time.Duration) int64 { } // vectorSelector evaluates a *parser.VectorSelector expression. -func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) Vector { - checkForSeriesSetExpansion(ev.ctx, node) - - var ( - vec = make(Vector, 0, len(node.Series)) - ) - +func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) (Vector, storage.Warnings) { + ws, err := checkAndExpandSeriesSet(ev.ctx, node) + if err != nil { + ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws}) + } + vec := make(Vector, 0, len(node.Series)) it := storage.NewBuffer(durationMilliseconds(ev.lookbackDelta)) for i, s := range node.Series { it.Reset(s.Iterator()) @@ -1381,7 +1395,7 @@ func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) Vecto ev.error(ErrTooManySamples(env)) } } - return vec + return vec, ws } // vectorSelectorSingle evaluates a instant vector for the iterator of one time series. @@ -1429,21 +1443,23 @@ func putPointSlice(p []Point) { } // matrixSelector evaluates a *parser.MatrixSelector expression. -func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) Matrix { - checkForSeriesSetExpansion(ev.ctx, node) - - vs := node.VectorSelector.(*parser.VectorSelector) - +func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) (Matrix, storage.Warnings) { var ( + vs = node.VectorSelector.(*parser.VectorSelector) + offset = durationMilliseconds(vs.Offset) maxt = ev.startTimestamp - offset mint = maxt - durationMilliseconds(node.Range) matrix = make(Matrix, 0, len(vs.Series)) + + it = storage.NewBuffer(durationMilliseconds(node.Range)) ) + ws, err := checkAndExpandSeriesSet(ev.ctx, node) + if err != nil { + ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws}) + } - it := storage.NewBuffer(durationMilliseconds(node.Range)) series := vs.Series - for i, s := range series { if err := contextDone(ev.ctx, "expression evaluation"); err != nil { ev.error(err) @@ -1461,7 +1477,7 @@ func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) Matrix { putPointSlice(ss.Points) } } - return matrix + return matrix, ws } // matrixIterSlice populates a matrix vector covering the requested range for a diff --git a/promql/engine_test.go b/promql/engine_test.go index e0a83aa1d8..c9c046318c 100644 --- a/promql/engine_test.go +++ b/promql/engine_test.go @@ -172,8 +172,8 @@ type errQuerier struct { err error } -func (q *errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { - return errSeriesSet{err: q.err}, nil, q.err +func (q *errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet { + return errSeriesSet{err: q.err} } func (*errQuerier) LabelValues(string) ([]string, storage.Warnings, error) { return nil, nil, nil } func (*errQuerier) LabelNames() ([]string, storage.Warnings, error) { return nil, nil, nil } @@ -184,9 +184,10 @@ type errSeriesSet struct { err error } -func (errSeriesSet) Next() bool { return false } -func (errSeriesSet) At() storage.Series { return nil } -func (e errSeriesSet) Err() error { return e.err } +func (errSeriesSet) Next() bool { return false } +func (errSeriesSet) At() storage.Series { return nil } +func (e errSeriesSet) Err() error { return e.err } +func (e errSeriesSet) Warnings() storage.Warnings { return nil } func TestQueryError(t *testing.T) { opts := EngineOpts{ @@ -208,14 +209,14 @@ func TestQueryError(t *testing.T) { res := vectorQuery.Exec(ctx) testutil.NotOk(t, res.Err, "expected error on failed select but got none") - testutil.Equals(t, errStorage, res.Err) + testutil.Assert(t, errors.Is(res.Err, errStorage), "expected error doesn't match") matrixQuery, err := engine.NewInstantQuery(queryable, "foo[1m]", time.Unix(1, 0)) testutil.Ok(t, err) res = matrixQuery.Exec(ctx) testutil.NotOk(t, res.Err, "expected error on failed select but got none") - testutil.Equals(t, errStorage, res.Err) + testutil.Assert(t, errors.Is(res.Err, errStorage), "expected error doesn't match") } // hintCheckerQuerier implements storage.Querier which checks the start and end times @@ -231,7 +232,7 @@ type hintCheckerQuerier struct { t *testing.T } -func (q *hintCheckerQuerier) Select(_ bool, sp *storage.SelectHints, _ ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *hintCheckerQuerier) Select(_ bool, sp *storage.SelectHints, _ ...*labels.Matcher) storage.SeriesSet { testutil.Equals(q.t, q.start, sp.Start) testutil.Equals(q.t, q.end, sp.End) testutil.Equals(q.t, q.grouping, sp.Grouping) @@ -239,7 +240,7 @@ func (q *hintCheckerQuerier) Select(_ bool, sp *storage.SelectHints, _ ...*label testutil.Equals(q.t, q.selRange, sp.Range) testutil.Equals(q.t, q.function, sp.Func) - return errSeriesSet{err: nil}, nil, nil + return errSeriesSet{err: nil} } func (*hintCheckerQuerier) LabelValues(string) ([]string, storage.Warnings, error) { return nil, nil, nil @@ -594,9 +595,8 @@ load 10s } testutil.Ok(t, res.Err) - testutil.Equals(t, c.Result, res.Value) + testutil.Equals(t, c.Result, res.Value, "query %q failed", c.Query) } - } func TestMaxQuerySamples(t *testing.T) { @@ -826,7 +826,7 @@ load 10s res := qry.Exec(test.Context()) testutil.Equals(t, c.Result.Err, res.Err) - testutil.Equals(t, c.Result.Value, res.Value) + testutil.Equals(t, c.Result.Value, res.Value, "query %q failed", c.Query) } } @@ -834,7 +834,7 @@ func TestRecoverEvaluatorRuntime(t *testing.T) { ev := &evaluator{logger: log.NewNopLogger()} var err error - defer ev.recover(&err) + defer ev.recover(nil, &err) // Cause a runtime panic. var a []int @@ -857,7 +857,31 @@ func TestRecoverEvaluatorError(t *testing.T) { t.Fatalf("wrong error message: %q, expected %q", err, e) } }() - defer ev.recover(&err) + defer ev.recover(nil, &err) + + panic(e) +} + +func TestRecoverEvaluatorErrorWithWarnings(t *testing.T) { + ev := &evaluator{logger: log.NewNopLogger()} + var err error + var ws storage.Warnings + + warnings := storage.Warnings{errors.New("custom warning")} + e := errWithWarnings{ + err: errors.New("custom error"), + warnings: warnings, + } + + defer func() { + if err.Error() != e.Error() { + t.Fatalf("wrong error message: %q, expected %q", err, e) + } + if len(ws) != len(warnings) && ws[0] != warnings[0] { + t.Fatalf("wrong warning message: %q, expected %q", ws[0], warnings[0]) + } + }() + defer ev.recover(&ws, &err) panic(e) } diff --git a/promql/test_test.go b/promql/test_test.go index 7de4387f5e..f18271ad14 100644 --- a/promql/test_test.go +++ b/promql/test_test.go @@ -133,8 +133,7 @@ func TestLazyLoader_WithSamplesTill(t *testing.T) { } // Get the series for the matcher. - ss, _, err := querier.Select(false, nil, matchers...) - testutil.Ok(t, err) + ss := querier.Select(false, nil, matchers...) testutil.Assert(t, ss.Next(), "") storageSeries := ss.At() testutil.Assert(t, !ss.Next(), "Expecting only 1 series") diff --git a/rules/manager.go b/rules/manager.go index 1e549fef67..f999aa19bc 100644 --- a/rules/manager.go +++ b/rules/manager.go @@ -707,12 +707,7 @@ func (g *Group) RestoreForState(ts time.Time) { matchers = append(matchers, mt) } - sset, err, _ := q.Select(false, nil, matchers...) - if err != nil { - level.Error(g.logger).Log("msg", "Failed to restore 'for' state", - labels.AlertName, alertRule.Name(), "stage", "Select", "err", err) - return - } + sset := q.Select(false, nil, matchers...) seriesFound := false var s storage.Series @@ -727,6 +722,17 @@ func (g *Group) RestoreForState(ts time.Time) { } } + if err := sset.Err(); err != nil { + // Querier Warnings are ignored. We do not care unless we have an error. + level.Error(g.logger).Log( + "msg", "Failed to restore 'for' state", + labels.AlertName, alertRule.Name(), + "stage", "Select", + "err", err, + ) + return + } + if !seriesFound { return } diff --git a/rules/manager_test.go b/rules/manager_test.go index 17cc1870df..2941039835 100644 --- a/rules/manager_test.go +++ b/rules/manager_test.go @@ -563,9 +563,7 @@ func TestStaleness(t *testing.T) { matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "a_plus_one") testutil.Ok(t, err) - set, _, err := querier.Select(false, nil, matcher) - testutil.Ok(t, err) - + set := querier.Select(false, nil, matcher) samples, err := readSeriesSet(set) testutil.Ok(t, err) @@ -686,9 +684,7 @@ func TestDeletedRuleMarkedStale(t *testing.T) { matcher, err := labels.NewMatcher(labels.MatchEqual, "l1", "v1") testutil.Ok(t, err) - set, _, err := querier.Select(false, nil, matcher) - testutil.Ok(t, err) - + set := querier.Select(false, nil, matcher) samples, err := readSeriesSet(set) testutil.Ok(t, err) @@ -1107,9 +1103,7 @@ func countStaleNaN(t *testing.T, st storage.Storage) int { matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_2") testutil.Ok(t, err) - set, _, err := querier.Select(false, nil, matcher) - testutil.Ok(t, err) - + set := querier.Select(false, nil, matcher) samples, err := readSeriesSet(set) testutil.Ok(t, err) diff --git a/scrape/scrape_test.go b/scrape/scrape_test.go index 34cd026802..1651b437d6 100644 --- a/scrape/scrape_test.go +++ b/scrape/scrape_test.go @@ -1617,9 +1617,9 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) { q, err := s.Querier(ctx, time.Time{}.UnixNano(), 0) testutil.Ok(t, err) - series, _, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) - testutil.Ok(t, err) + series := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) testutil.Equals(t, false, series.Next(), "series found in tsdb") + testutil.Ok(t, series.Err()) // We add a good metric to check that it is recorded. _, _, _, err = sl.append([]byte("test_metric{le=\"500\"} 1\n"), "", time.Time{}) @@ -1627,9 +1627,9 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) { q, err = s.Querier(ctx, time.Time{}.UnixNano(), 0) testutil.Ok(t, err) - series, _, err = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "le", "500")) - testutil.Ok(t, err) + series = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "le", "500")) testutil.Equals(t, true, series.Next(), "series not found in tsdb") + testutil.Ok(t, series.Err()) testutil.Equals(t, false, series.Next(), "more than one series found in tsdb") } @@ -1663,9 +1663,9 @@ func TestScrapeLoopDiscardUnnamedMetrics(t *testing.T) { q, err := s.Querier(ctx, time.Time{}.UnixNano(), 0) testutil.Ok(t, err) - series, _, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) - testutil.Ok(t, err) + series := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) testutil.Equals(t, false, series.Next(), "series found in tsdb") + testutil.Ok(t, series.Err()) } func TestReusableConfig(t *testing.T) { diff --git a/storage/fanout.go b/storage/fanout.go index 8692078a34..73df5c8c18 100644 --- a/storage/fanout.go +++ b/storage/fanout.go @@ -16,14 +16,15 @@ package storage import ( "container/heap" "context" - "reflect" "sort" "strings" + "sync" "github.com/go-kit/kit/log" "github.com/go-kit/kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/chunks" @@ -37,8 +38,15 @@ type fanout struct { secondaries []Storage } -// NewFanout returns a new fan-out Storage, which proxies reads and writes +// NewFanout returns a new fanout Storage, which proxies reads and writes // through to multiple underlying storages. +// +// The difference between primary and secondary Storage is only for read (Querier) path and it goes as follows: +// * If the primary querier returns an error, then any of the Querier operations will fail. +// * If any secondary querier returns an error the result from that queries is discarded. The overall operation will succeed, +// and the error from the secondary querier will be returned as a warning. +// +// NOTE: In the case of Prometheus, it treats all remote storages as secondary / best effort. func NewFanout(logger log.Logger, primary Storage, secondaries ...Storage) Storage { return &fanout{ logger: logger, @@ -56,8 +64,8 @@ func (f *fanout) StartTime() (int64, error) { return int64(model.Latest), err } - for _, storage := range f.secondaries { - t, err := storage.StartTime() + for _, s := range f.secondaries { + t, err := s.StartTime() if err != nil { return int64(model.Latest), err } @@ -69,29 +77,27 @@ func (f *fanout) StartTime() (int64, error) { } func (f *fanout) Querier(ctx context.Context, mint, maxt int64) (Querier, error) { - queriers := make([]Querier, 0, 1+len(f.secondaries)) - - // Add primary querier. - primaryQuerier, err := f.primary.Querier(ctx, mint, maxt) + primary, err := f.primary.Querier(ctx, mint, maxt) if err != nil { return nil, err } - queriers = append(queriers, primaryQuerier) - // Add secondary queriers. + secondaries := make([]Querier, 0, len(f.secondaries)) for _, storage := range f.secondaries { querier, err := storage.Querier(ctx, mint, maxt) if err != nil { - for _, q := range queriers { - // TODO(bwplotka): Log error. - _ = q.Close() + // Close already open Queriers, append potential errors to returned error. + errs := tsdb_errors.MultiError{err} + errs.Add(primary.Close()) + for _, q := range secondaries { + errs.Add(q.Close()) } - return nil, err + return nil, errs.Err() } - queriers = append(queriers, querier) - } - return NewMergeQuerier(primaryQuerier, queriers, ChainedSeriesMerge), nil + secondaries = append(secondaries, querier) + } + return NewMergeQuerier(primary, secondaries, ChainedSeriesMerge), nil } func (f *fanout) Appender() Appender { @@ -109,18 +115,12 @@ func (f *fanout) Appender() Appender { // Close closes the storage and all its underlying resources. func (f *fanout) Close() error { - if err := f.primary.Close(); err != nil { - return err + errs := tsdb_errors.MultiError{} + errs.Add(f.primary.Close()) + for _, s := range f.secondaries { + errs.Add(s.Close()) } - - // TODO return multiple errors? - var lastErr error - for _, storage := range f.secondaries { - if err := storage.Close(); err != nil { - lastErr = err - } - } - return lastErr + return errs.Err() } // fanoutAppender implements Appender. @@ -188,153 +188,138 @@ func (f *fanoutAppender) Rollback() (err error) { } type mergeGenericQuerier struct { - mergeFunc genericSeriesMergeFunc + queriers []genericQuerier - primaryQuerier genericQuerier - queriers []genericQuerier - failedQueriers map[genericQuerier]struct{} - setQuerierMap map[genericSeriesSet]genericQuerier + // mergeFn is used when we see series from different queriers Selects with the same labels. + mergeFn genericSeriesMergeFunc } -// NewMergeQuerier returns a new Querier that merges results of chkQuerierSeries queriers. -// NewMergeQuerier will return NoopQuerier if no queriers are passed to it -// and will filter NoopQueriers from its arguments, in order to reduce overhead -// when only one querier is passed. -// The difference between primary and secondary is as follows: f the primaryQuerier returns an error, query fails. -// For secondaries it just return warnings. -func NewMergeQuerier(primaryQuerier Querier, queriers []Querier, mergeFunc VerticalSeriesMergeFunc) Querier { - filtered := make([]genericQuerier, 0, len(queriers)) - for _, querier := range queriers { +// NewMergeQuerier returns a new Querier that merges results of given primary and slice of secondary queriers. +// See NewFanout commentary to learn more about primary vs secondary differences. +// +// In case of overlaps between the data given by primary + secondaries Selects, merge function will be used. +func NewMergeQuerier(primary Querier, secondaries []Querier, mergeFn VerticalSeriesMergeFunc) Querier { + queriers := make([]genericQuerier, 0, len(secondaries)+1) + if primary != nil { + queriers = append(queriers, newGenericQuerierFrom(primary)) + } + for _, querier := range secondaries { if _, ok := querier.(noopQuerier); !ok && querier != nil { - filtered = append(filtered, newGenericQuerierFrom(querier)) + queriers = append(queriers, newSecondaryQuerierFrom(querier)) } } - if len(filtered) == 0 { - return primaryQuerier - } - - if primaryQuerier == nil && len(filtered) == 1 { - return &querierAdapter{filtered[0]} - } - return &querierAdapter{&mergeGenericQuerier{ - mergeFunc: (&seriesMergerAdapter{VerticalSeriesMergeFunc: mergeFunc}).Merge, - primaryQuerier: newGenericQuerierFrom(primaryQuerier), - queriers: filtered, - failedQueriers: make(map[genericQuerier]struct{}), - setQuerierMap: make(map[genericSeriesSet]genericQuerier), + mergeFn: (&seriesMergerAdapter{VerticalSeriesMergeFunc: mergeFn}).Merge, + queriers: queriers, }} } -// NewMergeChunkQuerier returns a new ChunkQuerier that merges results of chkQuerierSeries chunk queriers. -// NewMergeChunkQuerier will return NoopChunkQuerier if no chunk queriers are passed to it, -// and will filter NoopQuerieNoopChunkQuerierrs from its arguments, in order to reduce overhead -// when only one chunk querier is passed. -func NewMergeChunkQuerier(primaryQuerier ChunkQuerier, queriers []ChunkQuerier, merger VerticalChunkSeriesMergerFunc) ChunkQuerier { - filtered := make([]genericQuerier, 0, len(queriers)) - for _, querier := range queriers { +// NewMergeChunkQuerier returns a new ChunkQuerier that merges results of given primary and slice of secondary chunk queriers. +// See NewFanout commentary to learn more about primary vs secondary differences. +// +// In case of overlaps between the data given by primary + secondaries Selects, merge function will be used. +// TODO(bwplotka): Currently merge will compact overlapping chunks with bigger chunk, without limit. Split it: https://github.com/prometheus/tsdb/issues/670 +func NewMergeChunkQuerier(primary ChunkQuerier, secondaries []ChunkQuerier, mergeFn VerticalChunkSeriesMergerFunc) ChunkQuerier { + queriers := make([]genericQuerier, 0, len(secondaries)+1) + if primary != nil { + queriers = append(queriers, newGenericQuerierFromChunk(primary)) + } + for _, querier := range secondaries { if _, ok := querier.(noopChunkQuerier); !ok && querier != nil { - filtered = append(filtered, newGenericQuerierFromChunk(querier)) + queriers = append(queriers, newSecondaryQuerierFromChunk(querier)) } } - if len(filtered) == 0 { - return primaryQuerier - } - - if primaryQuerier == nil && len(filtered) == 1 { - return &chunkQuerierAdapter{filtered[0]} - } - return &chunkQuerierAdapter{&mergeGenericQuerier{ - mergeFunc: (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge, - primaryQuerier: newGenericQuerierFromChunk(primaryQuerier), - queriers: filtered, - failedQueriers: make(map[genericQuerier]struct{}), - setQuerierMap: make(map[genericSeriesSet]genericQuerier), + mergeFn: (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: mergeFn}).Merge, + queriers: queriers, }} } // Select returns a set of series that matches the given label matchers. -func (q *mergeGenericQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (genericSeriesSet, Warnings, error) { +func (q *mergeGenericQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet { if len(q.queriers) == 1 { return q.queriers[0].Select(sortSeries, hints, matchers...) } var ( - seriesSets = make([]genericSeriesSet, 0, len(q.queriers)) - warnings Warnings - priErr error + seriesSets = make([]genericSeriesSet, 0, len(q.queriers)) + wg sync.WaitGroup + seriesSetChan = make(chan genericSeriesSet) ) - type queryResult struct { - qr genericQuerier - set genericSeriesSet - wrn Warnings - selectError error - } - queryResultChan := make(chan *queryResult) + // Schedule all Selects for all queriers we know about. for _, querier := range q.queriers { + wg.Add(1) go func(qr genericQuerier) { + defer wg.Done() + // We need to sort for NewMergeSeriesSet to work. - set, wrn, err := qr.Select(true, hints, matchers...) - queryResultChan <- &queryResult{qr: qr, set: set, wrn: wrn, selectError: err} + seriesSetChan <- qr.Select(true, hints, matchers...) }(querier) } - for i := 0; i < len(q.queriers); i++ { - qryResult := <-queryResultChan - q.setQuerierMap[qryResult.set] = qryResult.qr - if qryResult.wrn != nil { - warnings = append(warnings, qryResult.wrn...) + go func() { + wg.Wait() + close(seriesSetChan) + }() + + for r := range seriesSetChan { + seriesSets = append(seriesSets, r) + } + return &lazySeriesSet{create: create(seriesSets, q.mergeFn)} +} + +func create(seriesSets []genericSeriesSet, mergeFn genericSeriesMergeFunc) func() (genericSeriesSet, bool) { + // Returned function gets called with the first call to Next(). + return func() (genericSeriesSet, bool) { + if len(seriesSets) == 1 { + return seriesSets[0], seriesSets[0].Next() } - if qryResult.selectError != nil { - q.failedQueriers[qryResult.qr] = struct{}{} - // If the error source isn't the primary querier, return the error as a warning and continue. - if !reflect.DeepEqual(qryResult.qr, q.primaryQuerier) { - warnings = append(warnings, qryResult.selectError) - } else { - priErr = qryResult.selectError + var h genericSeriesSetHeap + for _, set := range seriesSets { + if set == nil { + continue + } + if set.Next() { + heap.Push(&h, set) + continue + } + // When primary fails ignore results from secondaries. + // Only the primary querier returns error. + if err := set.Err(); err != nil { + return errorOnlySeriesSet{err}, false } - continue } - seriesSets = append(seriesSets, qryResult.set) + set := &genericMergeSeriesSet{ + mergeFn: mergeFn, + sets: seriesSets, + heap: h, + } + return set, set.Next() } - if priErr != nil { - return nil, nil, priErr - } - return newGenericMergeSeriesSet(seriesSets, q, q.mergeFunc), warnings, nil } // LabelValues returns all potential values for a label name. func (q *mergeGenericQuerier) LabelValues(name string) ([]string, Warnings, error) { - var results [][]string - var warnings Warnings + var ( + results [][]string + warnings Warnings + ) for _, querier := range q.queriers { values, wrn, err := querier.LabelValues(name) if wrn != nil { + // TODO(bwplotka): We could potentially wrap warnings. warnings = append(warnings, wrn...) } if err != nil { - q.failedQueriers[querier] = struct{}{} - // If the error source isn't the primary querier, return the error as a warning and continue. - if querier != q.primaryQuerier { - warnings = append(warnings, err) - continue - } else { - return nil, nil, err - } + return nil, nil, errors.Wrapf(err, "LabelValues() from Querier for label %s", name) } results = append(results, values) } return mergeStringSlices(results), warnings, nil } -func (q *mergeGenericQuerier) IsFailedSet(set genericSeriesSet) bool { - _, isFailedQuerier := q.failedQueriers[q.setQuerierMap[set]] - return isFailedQuerier -} - func mergeStringSlices(ss [][]string) []string { switch len(ss) { case 0: @@ -381,37 +366,31 @@ func (q *mergeGenericQuerier) LabelNames() ([]string, Warnings, error) { for _, querier := range q.queriers { names, wrn, err := querier.LabelNames() if wrn != nil { + // TODO(bwplotka): We could potentially wrap warnings. warnings = append(warnings, wrn...) } - if err != nil { - q.failedQueriers[querier] = struct{}{} - // If the error source isn't the primaryQuerier querier, return the error as a warning and continue. - if querier != q.primaryQuerier { - warnings = append(warnings, err) - continue - } else { - return nil, nil, errors.Wrap(err, "LabelNames() from Querier") - } + return nil, nil, errors.Wrap(err, "LabelNames() from Querier") } - for _, name := range names { labelNamesMap[name] = struct{}{} } } + if len(labelNamesMap) == 0 { + return nil, warnings, nil + } labelNames := make([]string, 0, len(labelNamesMap)) for name := range labelNamesMap { labelNames = append(labelNames, name) } sort.Strings(labelNames) - return labelNames, warnings, nil } // Close releases the resources of the Querier. func (q *mergeGenericQuerier) Close() error { - var errs tsdb_errors.MultiError + errs := tsdb_errors.MultiError{} for _, querier := range q.queriers { if err := querier.Close(); err != nil { errs.Add(err) @@ -420,18 +399,6 @@ func (q *mergeGenericQuerier) Close() error { return errs.Err() } -// genericMergeSeriesSet implements genericSeriesSet -type genericMergeSeriesSet struct { - currentLabels labels.Labels - mergeFunc genericSeriesMergeFunc - - heap genericSeriesSetHeap - sets []genericSeriesSet - - currentSets []genericSeriesSet - querier *mergeGenericQuerier -} - // VerticalSeriesMergeFunc returns merged series implementation that merges series with same labels together. // It has to handle time-overlapped series as well. type VerticalSeriesMergeFunc func(...Series) Series @@ -447,7 +414,7 @@ func NewMergeSeriesSet(sets []SeriesSet, merger VerticalSeriesMergeFunc) SeriesS genericSets = append(genericSets, &genericSeriesSetAdapter{s}) } - return &seriesSetAdapter{newGenericMergeSeriesSet(genericSets, nil, (&seriesMergerAdapter{VerticalSeriesMergeFunc: merger}).Merge)} + return &seriesSetAdapter{newGenericMergeSeriesSet(genericSets, (&seriesMergerAdapter{VerticalSeriesMergeFunc: merger}).Merge)} } // NewMergeChunkSeriesSet returns a new ChunkSeriesSet that merges results of chkQuerierSeries ChunkSeriesSets. @@ -457,21 +424,30 @@ func NewMergeChunkSeriesSet(sets []ChunkSeriesSet, merger VerticalChunkSeriesMer genericSets = append(genericSets, &genericChunkSeriesSetAdapter{s}) } - return &chunkSeriesSetAdapter{newGenericMergeSeriesSet(genericSets, nil, (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge)} + return &chunkSeriesSetAdapter{newGenericMergeSeriesSet(genericSets, (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge)} +} + +// genericMergeSeriesSet implements genericSeriesSet. +type genericMergeSeriesSet struct { + currentLabels labels.Labels + mergeFn genericSeriesMergeFunc + + heap genericSeriesSetHeap + sets []genericSeriesSet + currentSets []genericSeriesSet } // newGenericMergeSeriesSet returns a new genericSeriesSet that merges (and deduplicates) -// series returned by the chkQuerierSeries series sets when iterating. -// Each chkQuerierSeries series set must return its series in labels order, otherwise +// series returned by the series sets when iterating. +// Each series set must return its series in labels order, otherwise // merged series set will be incorrect. -// Argument 'querier' is optional and can be nil. Pass Querier if you want to retry query in case of failing series set. -// Overlapped situations are merged using provided mergeFunc. -func newGenericMergeSeriesSet(sets []genericSeriesSet, querier *mergeGenericQuerier, mergeFunc genericSeriesMergeFunc) genericSeriesSet { +// Overlapping cases are merged using provided mergeFn. +func newGenericMergeSeriesSet(sets []genericSeriesSet, mergeFn genericSeriesMergeFunc) genericSeriesSet { if len(sets) == 1 { return sets[0] } - // Sets need to be pre-advanced, so we can introspect the label of the + // We are pre-advancing sets, so we can introspect the label of the // series under the cursor. var h genericSeriesSetHeap for _, set := range sets { @@ -483,26 +459,25 @@ func newGenericMergeSeriesSet(sets []genericSeriesSet, querier *mergeGenericQuer } } return &genericMergeSeriesSet{ - mergeFunc: mergeFunc, - heap: h, - sets: sets, - querier: querier, + mergeFn: mergeFn, + sets: sets, + heap: h, } } func (c *genericMergeSeriesSet) Next() bool { // Run in a loop because the "next" series sets may not be valid anymore. - // If a remote querier fails, we discard all series sets from that querier. // If, for the current label set, all the next series sets come from // failed remote storage sources, we want to keep trying with the next label set. for { - // Firstly advance all the current series sets. If any of them have run out + // Firstly advance all the current series sets. If any of them have run out // we can drop them, otherwise they should be inserted back into the heap. for _, set := range c.currentSets { if set.Next() { heap.Push(&c.heap, set) } } + if len(c.heap) == 0 { return false } @@ -512,9 +487,6 @@ func (c *genericMergeSeriesSet) Next() bool { c.currentLabels = c.heap[0].At().Labels() for len(c.heap) > 0 && labels.Equal(c.currentLabels, c.heap[0].At().Labels()) { set := heap.Pop(&c.heap).(genericSeriesSet) - if c.querier != nil && c.querier.IsFailedSet(set) { - continue - } c.currentSets = append(c.currentSets, set) } @@ -535,7 +507,7 @@ func (c *genericMergeSeriesSet) At() Labels { for _, seriesSet := range c.currentSets { series = append(series, seriesSet.At()) } - return c.mergeFunc(series...) + return c.mergeFn(series...) } func (c *genericMergeSeriesSet) Err() error { @@ -547,6 +519,14 @@ func (c *genericMergeSeriesSet) Err() error { return nil } +func (c *genericMergeSeriesSet) Warnings() Warnings { + var ws Warnings + for _, set := range c.sets { + ws = append(ws, set.Warnings()...) + } + return ws +} + type genericSeriesSetHeap []genericSeriesSet func (h genericSeriesSetHeap) Len() int { return len(h) } diff --git a/storage/fanout/fanout_test.go b/storage/fanout/fanout_test.go index d62b6f5e19..e648296e50 100644 --- a/storage/fanout/fanout_test.go +++ b/storage/fanout/fanout_test.go @@ -16,14 +16,14 @@ package storage import ( "context" "errors" + "testing" "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/storage" "github.com/prometheus/prometheus/util/teststorage" "github.com/prometheus/prometheus/util/testutil" - - "testing" ) func TestSelectSorted(t *testing.T) { @@ -79,8 +79,7 @@ func TestSelectSorted(t *testing.T) { matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "a") testutil.Ok(t, err) - seriesSet, _, err := querier.Select(true, nil, matcher) - testutil.Ok(t, err) + seriesSet := querier.Select(true, nil, matcher) result := make(map[int64]float64) var labelsResult labels.Labels @@ -95,6 +94,7 @@ func TestSelectSorted(t *testing.T) { } } + testutil.Ok(t, seriesSet.Err()) testutil.Equals(t, labelsResult, outputLabel) testutil.Equals(t, inputTotalSize, len(result)) } @@ -131,19 +131,12 @@ func TestFanoutErrors(t *testing.T) { defer querier.Close() matcher := labels.MustNewMatcher(labels.MatchEqual, "a", "b") - ss, warnings, err := querier.Select(true, nil, matcher) - testutil.Equals(t, tc.err, err) - testutil.Equals(t, tc.warnings, warnings) - - // Only test series iteration if there are no errors. - if err != nil { - continue - } - + ss := querier.Select(true, nil, matcher) for ss.Next() { ss.At() } - testutil.Ok(t, ss.Err()) + testutil.Equals(t, tc.err, ss.Err()) + testutil.Equals(t, tc.warnings, ss.Warnings()) } } @@ -169,8 +162,8 @@ func (errStorage) Close() error { type errQuerier struct{} -func (errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { - return nil, nil, errSelect +func (errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet { + return storage.ErrSeriesSet(errSelect) } func (errQuerier) LabelValues(name string) ([]string, storage.Warnings, error) { diff --git a/storage/fanout_test.go b/storage/fanout_test.go index a5d04ea8f6..307e11c613 100644 --- a/storage/fanout_test.go +++ b/storage/fanout_test.go @@ -17,8 +17,11 @@ import ( "fmt" "math" "sort" + "sync" "testing" + "github.com/pkg/errors" + "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/tsdb/chunkenc" "github.com/prometheus/prometheus/tsdb/tsdbutil" @@ -56,24 +59,37 @@ func TestMergeTwoStringSlices(t *testing.T) { func TestMergeQuerierWithChainMerger(t *testing.T) { for _, tc := range []struct { - name string - querierSeries [][]Series - extraQueriers []Querier + name string + primaryQuerierSeries []Series + querierSeries [][]Series + extraQueriers []Querier expected SeriesSet }{ { - name: "1 querier with no series", + name: "one primary querier with no series", + primaryQuerierSeries: []Series{}, + expected: NewMockSeriesSet(), + }, + { + name: "one secondary querier with no series", querierSeries: [][]Series{{}}, expected: NewMockSeriesSet(), }, { - name: "many queriers with no series", + name: "many secondary queriers with no series", querierSeries: [][]Series{{}, {}, {}, {}, {}, {}, {}}, expected: NewMockSeriesSet(), }, { - name: "1 querier, two series", + name: "mix of queriers with no series", + primaryQuerierSeries: []Series{}, + querierSeries: [][]Series{{}, {}, {}, {}, {}, {}, {}}, + expected: NewMockSeriesSet(), + }, + // Test rest of cases on secondary queriers as the different between primary vs secondary is just error handling. + { + name: "one querier, two series", querierSeries: [][]Series{{ NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{1, 1}, sample{2, 2}, sample{3, 3}}), NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}), @@ -84,7 +100,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { ), }, { - name: "2 queriers, 1 different series each", + name: "two queriers, one different series each", querierSeries: [][]Series{{ NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{1, 1}, sample{2, 2}, sample{3, 3}}), }, { @@ -96,7 +112,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { ), }, { - name: "2 time unsorted queriers, 2 series each", + name: "two time unsorted queriers, two series each", querierSeries: [][]Series{{ NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{5, 5}, sample{6, 6}}), NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}), @@ -116,7 +132,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { ), }, { - name: "5 queriers, only 2 queriers have 2 time unsorted series each", + name: "five queriers, only two queriers have two time unsorted series each", querierSeries: [][]Series{{}, {}, { NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{5, 5}, sample{6, 6}}), NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}), @@ -136,7 +152,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { ), }, { - name: "2 queriers, only 2 queriers have 2 time unsorted series each, with 3 noop and one nil querier together", + name: "two queriers, only two queriers have two time unsorted series each, with 3 noop and one nil querier together", querierSeries: [][]Series{{}, {}, { NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{5, 5}, sample{6, 6}}), NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}), @@ -157,7 +173,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { ), }, { - name: "2 queriers, with 2 series, one is overlapping", + name: "two queriers, with two series, one is overlapping", querierSeries: [][]Series{{}, {}, { NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{2, 21}, sample{3, 31}, sample{5, 5}, sample{6, 6}}), NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}), @@ -177,7 +193,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { ), }, { - name: "2 queries, one with NaN samples series", + name: "two queries, one with NaN samples series", querierSeries: [][]Series{{ NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, math.NaN()}}), }, { @@ -189,13 +205,17 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + var p Querier + if tc.primaryQuerierSeries != nil { + p = &mockQuerier{toReturn: tc.primaryQuerierSeries} + } var qs []Querier for _, in := range tc.querierSeries { qs = append(qs, &mockQuerier{toReturn: in}) } qs = append(qs, tc.extraQueriers...) - mergedQuerier, _, _ := NewMergeQuerier(qs[0], qs, ChainedSeriesMerge).Select(false, nil) + mergedQuerier := NewMergeQuerier(p, qs, ChainedSeriesMerge).Select(false, nil) // Get all merged series upfront to make sure there are no incorrectly retained shared // buffers causing bugs. @@ -222,22 +242,35 @@ func TestMergeQuerierWithChainMerger(t *testing.T) { func TestMergeChunkQuerierWithNoVerticalChunkSeriesMerger(t *testing.T) { for _, tc := range []struct { - name string - chkQuerierSeries [][]ChunkSeries - extraQueriers []ChunkQuerier + name string + primaryChkQuerierSeries []ChunkSeries + chkQuerierSeries [][]ChunkSeries + extraQueriers []ChunkQuerier expected ChunkSeriesSet }{ { - name: "one querier with no series", + name: "one primary querier with no series", + primaryChkQuerierSeries: []ChunkSeries{}, + expected: NewMockChunkSeriesSet(), + }, + { + name: "one secondary querier with no series", chkQuerierSeries: [][]ChunkSeries{{}}, expected: NewMockChunkSeriesSet(), }, { - name: "many queriers with no series", + name: "many secondary queriers with no series", chkQuerierSeries: [][]ChunkSeries{{}, {}, {}, {}, {}, {}, {}}, expected: NewMockChunkSeriesSet(), }, + { + name: "mix of queriers with no series", + primaryChkQuerierSeries: []ChunkSeries{}, + chkQuerierSeries: [][]ChunkSeries{{}, {}, {}, {}, {}, {}, {}}, + expected: NewMockChunkSeriesSet(), + }, + // Test rest of cases on secondary queriers as the different between primary vs secondary is just error handling. { name: "one querier, two series", chkQuerierSeries: [][]ChunkSeries{{ @@ -347,13 +380,19 @@ func TestMergeChunkQuerierWithNoVerticalChunkSeriesMerger(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { + var p ChunkQuerier + if tc.primaryChkQuerierSeries != nil { + p = &mockChunkQurier{toReturn: tc.primaryChkQuerierSeries} + } + var qs []ChunkQuerier for _, in := range tc.chkQuerierSeries { qs = append(qs, &mockChunkQurier{toReturn: in}) } qs = append(qs, tc.extraQueriers...) - merged, _, _ := NewMergeChunkQuerier(qs[0], qs, NewVerticalChunkSeriesMerger(nil)).Select(false, nil) + // TODO(bwplotka): Add case of overlap to check if those are handled well. + merged := NewMergeChunkQuerier(p, qs, NewVerticalChunkSeriesMerger(nil)).Select(false, nil) for merged.Next() { testutil.Assert(t, tc.expected.Next(), "Expected Next() to be true") actualSeries := merged.At() @@ -384,14 +423,14 @@ func (a seriesByLabel) Len() int { return len(a) } func (a seriesByLabel) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a seriesByLabel) Less(i, j int) bool { return labels.Compare(a[i].Labels(), a[j].Labels()) < 0 } -func (m *mockQuerier) Select(sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) (SeriesSet, Warnings, error) { +func (m *mockQuerier) Select(sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) SeriesSet { cpy := make([]Series, len(m.toReturn)) copy(cpy, m.toReturn) if sortSeries { sort.Sort(seriesByLabel(cpy)) } - return NewMockSeriesSet(cpy...), nil, nil + return NewMockSeriesSet(cpy...) } type mockChunkQurier struct { @@ -408,14 +447,14 @@ func (a chunkSeriesByLabel) Less(i, j int) bool { return labels.Compare(a[i].Labels(), a[j].Labels()) < 0 } -func (m *mockChunkQurier) Select(sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) { +func (m *mockChunkQurier) Select(sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) ChunkSeriesSet { cpy := make([]ChunkSeries, len(m.toReturn)) copy(cpy, m.toReturn) if sortSeries { sort.Sort(chunkSeriesByLabel(cpy)) } - return NewMockChunkSeriesSet(cpy...), nil, nil + return NewMockChunkSeriesSet(cpy...) } type mockSeriesSet struct { @@ -439,6 +478,8 @@ func (m *mockSeriesSet) At() Series { return m.series[m.idx] } func (m *mockSeriesSet) Err() error { return nil } +func (m *mockSeriesSet) Warnings() Warnings { return nil } + type mockChunkSeriesSet struct { idx int series []ChunkSeries @@ -460,6 +501,8 @@ func (m *mockChunkSeriesSet) At() ChunkSeries { return m.series[m.idx] } func (m *mockChunkSeriesSet) Err() error { return nil } +func (m *mockChunkSeriesSet) Warnings() Warnings { return nil } + func TestChainSampleIterator(t *testing.T) { for _, tc := range []struct { input []chunkenc.Iterator @@ -571,7 +614,7 @@ func makeMergeSeriesSet(numSeriesSets, numSeries, numSamples int) SeriesSet { for i := 0; i < numSeriesSets; i++ { seriesSets = append(seriesSets, &genericSeriesSetAdapter{makeSeriesSet(numSeries, numSamples)}) } - return &seriesSetAdapter{newGenericMergeSeriesSet(seriesSets, nil, (&seriesMergerAdapter{VerticalSeriesMergeFunc: ChainedSeriesMerge}).Merge)} + return &seriesSetAdapter{newGenericMergeSeriesSet(seriesSets, (&seriesMergerAdapter{VerticalSeriesMergeFunc: ChainedSeriesMerge}).Merge)} } func benchmarkDrain(seriesSet SeriesSet, b *testing.B) { @@ -603,3 +646,260 @@ func BenchmarkMergeSeriesSet(b *testing.B) { }) } } + +type mockGenericQuerier struct { + mtx sync.Mutex + + closed bool + labelNamesCalls int + labelNamesRequested []string + sortedSeriesRequested []bool + + resp []string + warnings Warnings + err error +} + +func (m *mockGenericQuerier) Select(b bool, _ *SelectHints, _ ...*labels.Matcher) genericSeriesSet { + m.mtx.Lock() + m.sortedSeriesRequested = append(m.sortedSeriesRequested, b) + m.mtx.Unlock() + return &mockGenericSeriesSet{resp: m.resp, warnings: m.warnings, err: m.err} +} + +func (m *mockGenericQuerier) LabelValues(name string) ([]string, Warnings, error) { + m.mtx.Lock() + m.labelNamesRequested = append(m.labelNamesRequested, name) + m.mtx.Unlock() + return m.resp, m.warnings, m.err +} + +func (m *mockGenericQuerier) LabelNames() ([]string, Warnings, error) { + m.mtx.Lock() + m.labelNamesCalls++ + m.mtx.Unlock() + return m.resp, m.warnings, m.err +} + +func (m *mockGenericQuerier) Close() error { + m.closed = true + return nil +} + +type mockGenericSeriesSet struct { + resp []string + warnings Warnings + err error + + curr int +} + +func (m *mockGenericSeriesSet) Next() bool { + if m.err != nil { + return false + } + if m.curr >= len(m.resp) { + return false + } + m.curr++ + return true +} + +func (m *mockGenericSeriesSet) Err() error { return m.err } +func (m *mockGenericSeriesSet) Warnings() Warnings { return m.warnings } + +func (m *mockGenericSeriesSet) At() Labels { + return mockLabels(m.resp[m.curr-1]) +} + +type mockLabels string + +func (l mockLabels) Labels() labels.Labels { + return labels.FromStrings("test", string(l)) +} + +func unwrapMockGenericQuerier(t *testing.T, qr genericQuerier) *mockGenericQuerier { + m, ok := qr.(*mockGenericQuerier) + if !ok { + s, ok := qr.(*secondaryQuerier) + testutil.Assert(t, ok, "expected secondaryQuerier got something else") + m, ok = s.genericQuerier.(*mockGenericQuerier) + testutil.Assert(t, ok, "expected mockGenericQuerier got something else") + } + return m +} + +func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) { + var ( + errStorage = errors.New("storage error") + warnStorage = errors.New("storage warning") + ) + for _, tcase := range []struct { + name string + queriers []genericQuerier + + expectedSelectsSeries []labels.Labels + expectedLabels []string + + expectedWarnings [3]Warnings + expectedErrs [3]error + }{ + {}, + { + name: "one successful primary querier", + queriers: []genericQuerier{&mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil}}, + expectedSelectsSeries: []labels.Labels{ + labels.FromStrings("test", "a"), + labels.FromStrings("test", "b"), + }, + expectedLabels: []string{"a", "b"}, + }, + { + name: "multiple successful primary queriers", + queriers: []genericQuerier{ + &mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil}, + &mockGenericQuerier{resp: []string{"b", "c"}, warnings: nil, err: nil}, + }, + expectedSelectsSeries: []labels.Labels{ + labels.FromStrings("test", "a"), + labels.FromStrings("test", "b"), + labels.FromStrings("test", "c"), + }, + expectedLabels: []string{"a", "b", "c"}, + }, + { + name: "one failed primary querier", + queriers: []genericQuerier{&mockGenericQuerier{warnings: nil, err: errStorage}}, + expectedErrs: [3]error{errStorage, errStorage, errStorage}, + }, + { + name: "one successful primary querier with successful secondaries", + queriers: []genericQuerier{ + &mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}}, + }, + expectedSelectsSeries: []labels.Labels{ + labels.FromStrings("test", "a"), + labels.FromStrings("test", "b"), + labels.FromStrings("test", "c"), + }, + expectedLabels: []string{"a", "b", "c"}, + }, + { + name: "one successful primary querier with empty response and successful secondaries", + queriers: []genericQuerier{ + &mockGenericQuerier{resp: []string{}, warnings: nil, err: nil}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}}, + }, + expectedSelectsSeries: []labels.Labels{ + labels.FromStrings("test", "b"), + labels.FromStrings("test", "c"), + }, + expectedLabels: []string{"b", "c"}, + }, + { + name: "one failed primary querier with successful secondaries", + queriers: []genericQuerier{ + &mockGenericQuerier{warnings: nil, err: errStorage}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}}, + }, + expectedErrs: [3]error{errStorage, errStorage, errStorage}, + }, + { + name: "one successful primary querier with failed secondaries", + queriers: []genericQuerier{ + &mockGenericQuerier{resp: []string{"a"}, warnings: nil, err: nil}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: errStorage}}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: errStorage}}, + }, + expectedSelectsSeries: []labels.Labels{ + labels.FromStrings("test", "a"), + }, + expectedLabels: []string{"a"}, + expectedWarnings: [3]Warnings{ + []error{errStorage, errStorage}, + []error{errStorage, errStorage}, + []error{errStorage, errStorage}, + }, + }, + { + name: "successful queriers with warnings", + queriers: []genericQuerier{ + &mockGenericQuerier{resp: []string{"a"}, warnings: []error{warnStorage}, err: nil}, + &secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: []error{warnStorage}, err: nil}}, + }, + expectedSelectsSeries: []labels.Labels{ + labels.FromStrings("test", "a"), + labels.FromStrings("test", "b"), + }, + expectedLabels: []string{"a", "b"}, + expectedWarnings: [3]Warnings{ + []error{warnStorage, warnStorage}, + []error{warnStorage, warnStorage}, + []error{warnStorage, warnStorage}, + }, + }, + } { + t.Run(tcase.name, func(t *testing.T) { + q := &mergeGenericQuerier{ + queriers: tcase.queriers, + mergeFn: func(l ...Labels) Labels { return l[0] }, + } + + t.Run("Select", func(t *testing.T) { + res := q.Select(false, nil) + var lbls []labels.Labels + for res.Next() { + lbls = append(lbls, res.At().Labels()) + } + testutil.Equals(t, tcase.expectedWarnings[0], res.Warnings()) + testutil.Equals(t, tcase.expectedErrs[0], res.Err()) + testutil.Assert(t, errors.Is(res.Err(), tcase.expectedErrs[0]), "expected error doesn't match") + testutil.Equals(t, tcase.expectedSelectsSeries, lbls) + + for _, qr := range q.queriers { + m := unwrapMockGenericQuerier(t, qr) + + exp := []bool{true} + if len(q.queriers) == 1 { + exp[0] = false + } + testutil.Equals(t, exp, m.sortedSeriesRequested) + } + }) + t.Run("LabelNames", func(t *testing.T) { + res, w, err := q.LabelNames() + testutil.Equals(t, tcase.expectedWarnings[1], w) + testutil.Assert(t, errors.Is(err, tcase.expectedErrs[1]), "expected error doesn't match") + testutil.Equals(t, tcase.expectedLabels, res) + + if err != nil { + return + } + for _, qr := range q.queriers { + m := unwrapMockGenericQuerier(t, qr) + + testutil.Equals(t, 1, m.labelNamesCalls) + } + }) + t.Run("LabelValues", func(t *testing.T) { + res, w, err := q.LabelValues("test") + testutil.Equals(t, tcase.expectedWarnings[2], w) + testutil.Assert(t, errors.Is(err, tcase.expectedErrs[2]), "expected error doesn't match") + testutil.Equals(t, tcase.expectedLabels, res) + + if err != nil { + return + } + for _, qr := range q.queriers { + m := unwrapMockGenericQuerier(t, qr) + + testutil.Equals(t, []string{"test"}, m.labelNamesRequested) + } + }) + }) + } +} diff --git a/storage/generic.go b/storage/generic.go index d23e51a6c6..36283a40a8 100644 --- a/storage/generic.go +++ b/storage/generic.go @@ -20,13 +20,14 @@ import "github.com/prometheus/prometheus/pkg/labels" type genericQuerier interface { baseQuerier - Select(bool, *SelectHints, ...*labels.Matcher) (genericSeriesSet, Warnings, error) + Select(bool, *SelectHints, ...*labels.Matcher) genericSeriesSet } type genericSeriesSet interface { Next() bool At() Labels Err() error + Warnings() Warnings } type genericSeriesMergeFunc func(...Labels) Labels @@ -55,13 +56,11 @@ type genericQuerierAdapter struct { cq ChunkQuerier } -func (q *genericQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (genericSeriesSet, Warnings, error) { +func (q *genericQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet { if q.q != nil { - s, w, err := q.q.Select(sortSeries, hints, matchers...) - return &genericSeriesSetAdapter{s}, w, err + return &genericSeriesSetAdapter{q.q.Select(sortSeries, hints, matchers...)} } - s, w, err := q.cq.Select(sortSeries, hints, matchers...) - return &genericChunkSeriesSetAdapter{s}, w, err + return &genericChunkSeriesSetAdapter{q.cq.Select(sortSeries, hints, matchers...)} } func newGenericQuerierFrom(q Querier) genericQuerier { @@ -84,9 +83,8 @@ func (a *seriesSetAdapter) At() Series { return a.genericSeriesSet.At().(Series) } -func (q *querierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (SeriesSet, Warnings, error) { - s, w, err := q.genericQuerier.Select(sortSeries, hints, matchers...) - return &seriesSetAdapter{s}, w, err +func (q *querierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) SeriesSet { + return &seriesSetAdapter{q.genericQuerier.Select(sortSeries, hints, matchers...)} } type chunkQuerierAdapter struct { @@ -101,9 +99,8 @@ func (a *chunkSeriesSetAdapter) At() ChunkSeries { return a.genericSeriesSet.At().(ChunkSeries) } -func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) { - s, w, err := q.genericQuerier.Select(sortSeries, hints, matchers...) - return &chunkSeriesSetAdapter{s}, w, err +func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) ChunkSeriesSet { + return &chunkSeriesSetAdapter{q.genericQuerier.Select(sortSeries, hints, matchers...)} } type seriesMergerAdapter struct { @@ -129,3 +126,13 @@ func (a *chunkSeriesMergerAdapter) Merge(s ...Labels) Labels { } return a.VerticalChunkSeriesMergerFunc(buf...) } + +type noopGenericSeriesSet struct{} + +func (noopGenericSeriesSet) Next() bool { return false } + +func (noopGenericSeriesSet) At() Labels { return nil } + +func (noopGenericSeriesSet) Err() error { return nil } + +func (noopGenericSeriesSet) Warnings() Warnings { return nil } diff --git a/storage/interface.go b/storage/interface.go index 0dc3b9efb2..dd558bb538 100644 --- a/storage/interface.go +++ b/storage/interface.go @@ -65,7 +65,7 @@ type Querier interface { // Select returns a set of series that matches the given label matchers. // Caller can specify if it requires returned series to be sorted. Prefer not requiring sorting for better performance. // It allows passing hints that can help in optimising select, but it's up to implementation how this is used if used at all. - Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (SeriesSet, Warnings, error) + Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) SeriesSet } // A ChunkQueryable handles queries against a storage. @@ -82,7 +82,7 @@ type ChunkQuerier interface { // Select returns a set of series that matches the given label matchers. // Caller can specify if it requires returned series to be sorted. Prefer not requiring sorting for better performance. // It allows passing hints that can help in optimising select, but it's up to implementation how this is used if used at all. - Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) + Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) ChunkSeriesSet } type baseQuerier interface { @@ -153,7 +153,12 @@ type Appender interface { type SeriesSet interface { Next() bool At() Series + // The error that iteration as failed with. + // When an error occurs, set cannot continue to iterate. Err() error + // A collection of warnings for the whole set. + // Warnings could be return even iteration has not failed with error. + Warnings() Warnings } var emptySeriesSet = errSeriesSet{} @@ -164,12 +169,19 @@ func EmptySeriesSet() SeriesSet { } type errSeriesSet struct { + ws Warnings err error } -func (s errSeriesSet) Next() bool { return false } -func (s errSeriesSet) At() Series { return nil } -func (s errSeriesSet) Err() error { return s.err } +func (s errSeriesSet) Next() bool { return false } +func (s errSeriesSet) At() Series { return nil } +func (s errSeriesSet) Err() error { return s.err } +func (s errSeriesSet) Warnings() Warnings { return s.ws } + +// ErrSeriesSet returns a series set that wraps an error. +func ErrSeriesSet(err error) SeriesSet { + return errSeriesSet{err: err} +} // Series exposes a single time series and allows iterating over samples. type Series interface { @@ -181,7 +193,12 @@ type Series interface { type ChunkSeriesSet interface { Next() bool At() ChunkSeries + // The error that iteration has failed with. + // When an error occurs, set cannot continue to iterate. Err() error + // A collection of warnings for the whole set. + // Warnings could be return even iteration has not failed with error. + Warnings() Warnings } // ChunkSeries exposes a single time series and allows iterating over chunks. diff --git a/storage/lazy.go b/storage/lazy.go new file mode 100644 index 0000000000..8eaa8772ab --- /dev/null +++ b/storage/lazy.go @@ -0,0 +1,67 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +type lazySeriesSet struct { + create func() (s genericSeriesSet, ok bool) + + set genericSeriesSet +} + +func (c *lazySeriesSet) Next() bool { + if c.set != nil { + return c.set.Next() + } + + var ok bool + c.set, ok = c.create() + return ok +} + +func (c *lazySeriesSet) Err() error { + if c.set != nil { + return c.set.Err() + } + return nil +} + +func (c *lazySeriesSet) At() Labels { + if c.set != nil { + return c.set.At() + } + return nil +} + +func (c *lazySeriesSet) Warnings() Warnings { + if c.set != nil { + return c.set.Warnings() + } + return nil +} + +type warningsOnlySeriesSet Warnings + +func (warningsOnlySeriesSet) Next() bool { return false } +func (warningsOnlySeriesSet) Err() error { return nil } +func (warningsOnlySeriesSet) At() Labels { return nil } +func (c warningsOnlySeriesSet) Warnings() Warnings { return Warnings(c) } + +type errorOnlySeriesSet struct { + err error +} + +func (errorOnlySeriesSet) Next() bool { return false } +func (errorOnlySeriesSet) At() Labels { return nil } +func (s errorOnlySeriesSet) Err() error { return s.err } +func (errorOnlySeriesSet) Warnings() Warnings { return nil } diff --git a/storage/noop.go b/storage/noop.go index c4ab1ba340..00599aba74 100644 --- a/storage/noop.go +++ b/storage/noop.go @@ -24,8 +24,8 @@ func NoopQuerier() Querier { return noopQuerier{} } -func (noopQuerier) Select(bool, *SelectHints, ...*labels.Matcher) (SeriesSet, Warnings, error) { - return NoopSeriesSet(), nil, nil +func (noopQuerier) Select(bool, *SelectHints, ...*labels.Matcher) SeriesSet { + return NoopSeriesSet() } func (noopQuerier) LabelValues(string) ([]string, Warnings, error) { @@ -47,8 +47,8 @@ func NoopChunkedQuerier() ChunkQuerier { return noopChunkQuerier{} } -func (noopChunkQuerier) Select(bool, *SelectHints, ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) { - return NoopChunkedSeriesSet(), nil, nil +func (noopChunkQuerier) Select(bool, *SelectHints, ...*labels.Matcher) ChunkSeriesSet { + return NoopChunkedSeriesSet() } func (noopChunkQuerier) LabelValues(string) ([]string, Warnings, error) { @@ -76,6 +76,8 @@ func (noopSeriesSet) At() Series { return nil } func (noopSeriesSet) Err() error { return nil } +func (noopSeriesSet) Warnings() Warnings { return nil } + type noopChunkedSeriesSet struct{} // NoopChunkedSeriesSet is a ChunkSeriesSet that does nothing. @@ -88,3 +90,5 @@ func (noopChunkedSeriesSet) Next() bool { return false } func (noopChunkedSeriesSet) At() ChunkSeries { return nil } func (noopChunkedSeriesSet) Err() error { return nil } + +func (noopChunkedSeriesSet) Warnings() Warnings { return nil } diff --git a/storage/remote/codec.go b/storage/remote/codec.go index 3b6556a124..6985a5f6a6 100644 --- a/storage/remote/codec.go +++ b/storage/remote/codec.go @@ -107,7 +107,7 @@ func ToQuery(from, to int64, matchers []*labels.Matcher, hints *storage.SelectHi } // ToQueryResult builds a QueryResult proto. -func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, error) { +func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, storage.Warnings, error) { numSamples := 0 resp := &prompb.QueryResult{} for ss.Next() { @@ -118,7 +118,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, for iter.Next() { numSamples++ if sampleLimit > 0 && numSamples > sampleLimit { - return nil, HTTPError{ + return nil, ss.Warnings(), HTTPError{ msg: fmt.Sprintf("exceeded sample limit (%d)", sampleLimit), status: http.StatusBadRequest, } @@ -130,7 +130,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, }) } if err := iter.Err(); err != nil { - return nil, err + return nil, ss.Warnings(), err } resp.Timeseries = append(resp.Timeseries, &prompb.TimeSeries{ @@ -139,9 +139,9 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, }) } if err := ss.Err(); err != nil { - return nil, err + return nil, ss.Warnings(), err } - return resp, nil + return resp, ss.Warnings(), nil } // FromQueryResult unpacks and sorts a QueryResult proto. @@ -195,7 +195,7 @@ func StreamChunkedReadResponses( ss storage.SeriesSet, sortedExternalLabels []prompb.Label, maxBytesInFrame int, -) error { +) (storage.Warnings, error) { var ( chks []prompb.Chunk lbls []prompb.Label @@ -218,7 +218,7 @@ func StreamChunkedReadResponses( // TODO(bwplotka): Use ChunkIterator once available in TSDB instead of re-encoding: https://github.com/prometheus/prometheus/pull/5882 chks, err = encodeChunks(iter, chks, maxBytesInFrame-lblsSize) if err != nil { - return err + return ss.Warnings(), err } if len(chks) == 0 { @@ -234,25 +234,25 @@ func StreamChunkedReadResponses( QueryIndex: queryIndex, }) if err != nil { - return errors.Wrap(err, "marshal ChunkedReadResponse") + return ss.Warnings(), errors.Wrap(err, "marshal ChunkedReadResponse") } if _, err := stream.Write(b); err != nil { - return errors.Wrap(err, "write to stream") + return ss.Warnings(), errors.Wrap(err, "write to stream") } chks = chks[:0] } if err := iter.Err(); err != nil { - return err + return ss.Warnings(), err } } if err := ss.Err(); err != nil { - return err + return ss.Warnings(), err } - return nil + return ss.Warnings(), nil } // encodeChunks expects iterator to be ready to use (aka iter.Next() called before invoking). @@ -365,6 +365,8 @@ func (e errSeriesSet) Err() error { return e.err } +func (e errSeriesSet) Warnings() storage.Warnings { return nil } + // concreteSeriesSet implements storage.SeriesSet. type concreteSeriesSet struct { cur int @@ -384,6 +386,8 @@ func (c *concreteSeriesSet) Err() error { return nil } +func (c *concreteSeriesSet) Warnings() storage.Warnings { return nil } + // concreteSeries implements storage.Series. type concreteSeries struct { labels labels.Labels diff --git a/storage/remote/read.go b/storage/remote/read.go index 219ba3ab38..f2fd212458 100644 --- a/storage/remote/read.go +++ b/storage/remote/read.go @@ -71,10 +71,10 @@ type querier struct { } // Select implements storage.Querier and uses the given matchers to read series sets from the Client. -func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { query, err := ToQuery(q.mint, q.maxt, matchers, hints) if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } remoteReadGauge := remoteReadQueries.WithLabelValues(q.client.remoteName, q.client.url.String()) @@ -86,10 +86,10 @@ func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers . res, err := q.client.Read(q.ctx, query) if err != nil { - return nil, nil, fmt.Errorf("remote_read: %v", err) + return storage.ErrSeriesSet(fmt.Errorf("remote_read: %v", err)) } - return FromQueryResult(sortSeries, res), nil, nil + return FromQueryResult(sortSeries, res) } // LabelValues implements storage.Querier and is a noop. @@ -132,13 +132,9 @@ type externalLabelsQuerier struct { // Select adds equality matchers for all external labels to the list of matchers // before calling the wrapped storage.Queryable. The added external labels are // removed from the returned series sets. -func (q externalLabelsQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q externalLabelsQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { m, added := q.addExternalLabels(matchers) - s, warnings, err := q.Querier.Select(sortSeries, hints, m...) - if err != nil { - return nil, warnings, err - } - return newSeriesSetFilter(s, added), warnings, nil + return newSeriesSetFilter(q.Querier.Select(sortSeries, hints, m...), added) } // PreferLocalStorageFilter returns a QueryableFunc which creates a NoopQuerier @@ -185,7 +181,7 @@ type requiredMatchersQuerier struct { // Select returns a NoopSeriesSet if the given matchers don't match the label // set of the requiredMatchersQuerier. Otherwise it'll call the wrapped querier. -func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet { ms := q.requiredMatchers for _, m := range matchers { for i, r := range ms { @@ -199,7 +195,7 @@ func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHi } } if len(ms) > 0 { - return storage.NoopSeriesSet(), nil, nil + return storage.NoopSeriesSet() } return q.Querier.Select(sortSeries, hints, matchers...) } diff --git a/storage/remote/read_test.go b/storage/remote/read_test.go index 0706d7bff6..6a9cdd712b 100644 --- a/storage/remote/read_test.go +++ b/storage/remote/read_test.go @@ -117,10 +117,8 @@ func TestExternalLabelsQuerierSelect(t *testing.T) { }, } want := newSeriesSetFilter(mockSeriesSet{}, q.externalLabels) - have, _, err := q.Select(false, nil, matchers...) - if err != nil { - t.Error(err) - } + have := q.Select(false, nil, matchers...) + if !reflect.DeepEqual(want, have) { t.Errorf("expected series set %+v, got %+v", want, have) } @@ -218,16 +216,12 @@ func TestSeriesSetFilter(t *testing.T) { }, } - for i, tc := range tests { + for _, tc := range tests { filtered := newSeriesSetFilter(FromQueryResult(true, tc.in), tc.toRemove) - have, err := ToQueryResult(filtered, 1e6) - if err != nil { - t.Fatal(err) - } - - if !reflect.DeepEqual(have, tc.expected) { - t.Fatalf("%d. unexpected labels; want %v, got %v", i, tc.expected, have) - } + act, ws, err := ToQueryResult(filtered, 1e6) + testutil.Ok(t, err) + testutil.Equals(t, 0, len(ws)) + testutil.Equals(t, tc.expected, act) } } @@ -242,8 +236,8 @@ type mockSeriesSet struct { storage.SeriesSet } -func (mockQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { - return mockSeriesSet{}, nil, nil +func (mockQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet { + return mockSeriesSet{} } func TestPreferLocalStorageFilter(t *testing.T) { @@ -398,10 +392,8 @@ func TestRequiredLabelsQuerierSelect(t *testing.T) { requiredMatchers: test.requiredMatchers, } - have, _, err := q.Select(false, nil, test.matchers...) - if err != nil { - t.Error(err) - } + have := q.Select(false, nil, test.matchers...) + if want := test.seriesSet; want != have { t.Errorf("%d. expected series set %+v, got %+v", i, want, have) } diff --git a/storage/remote/storage.go b/storage/remote/storage.go index 1b756af112..9f2d8a58d8 100644 --- a/storage/remote/storage.go +++ b/storage/remote/storage.go @@ -132,6 +132,9 @@ func (s *Storage) StartTime() (int64, error) { // Querier returns a storage.MergeQuerier combining the remote client queriers // of each configured remote read endpoint. +// Returned querier will never return error as all queryables are assumed best effort. +// Additionally all returned queriers ensure that its Select's SeriesSets have ready data after first `Next` invoke. +// This is because Prometheus (fanout and secondary queries) can't handle the stream failing half way through by design. func (s *Storage) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) { s.mtx.Lock() queryables := s.queryables diff --git a/storage/secondary.go b/storage/secondary.go new file mode 100644 index 0000000000..7f70930697 --- /dev/null +++ b/storage/secondary.go @@ -0,0 +1,112 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package storage + +import ( + "sync" + + "github.com/prometheus/prometheus/pkg/labels" +) + +// secondaryQuerier is a wrapper that allows a querier to be treated in a best effort manner. +// This means that an error on any method returned by Querier except Close will be returned as a warning, +// and the result will be empty. +// +// Additionally, Querier ensures that if ANY SeriesSet returned by this querier's Select failed on an initial Next, +// All other SeriesSet will be return no response as well. This ensures consistent partial response strategy, where you +// have either full results or none from each secondary Querier. +// NOTE: This works well only for implementations that only fail during first Next() (e.g fetch from network). If implementation fails +// during further iterations, set will panic. If Select is invoked after first Next of any returned SeriesSet, querier will panic. +// +// Not go-routine safe. +// NOTE: Prometheus treats all remote storages as secondary / best effort. +type secondaryQuerier struct { + genericQuerier + + once sync.Once + done bool + asyncSets []genericSeriesSet +} + +func newSecondaryQuerierFrom(q Querier) genericQuerier { + return &secondaryQuerier{genericQuerier: newGenericQuerierFrom(q)} +} + +func newSecondaryQuerierFromChunk(cq ChunkQuerier) genericQuerier { + return &secondaryQuerier{genericQuerier: newGenericQuerierFromChunk(cq)} +} + +func (s *secondaryQuerier) LabelValues(name string) ([]string, Warnings, error) { + vals, w, err := s.genericQuerier.LabelValues(name) + if err != nil { + return nil, append([]error{err}, w...), nil + } + return vals, w, nil +} + +func (s *secondaryQuerier) LabelNames() ([]string, Warnings, error) { + names, w, err := s.genericQuerier.LabelNames() + if err != nil { + return nil, append([]error{err}, w...), nil + } + return names, w, nil +} + +func (s *secondaryQuerier) createFn(asyncSet genericSeriesSet) func() (genericSeriesSet, bool) { + s.asyncSets = append(s.asyncSets, asyncSet) + curr := len(s.asyncSets) - 1 + return func() (genericSeriesSet, bool) { + s.once.Do(func() { + // At first create invocation we iterate over all sets and ensure its Next() returns some value without + // errors. This is to ensure we support consistent partial failures. + for i, set := range s.asyncSets { + if set.Next() { + continue + } + ws := set.Warnings() + // Failed set. + if err := set.Err(); err != nil { + ws = append([]error{err}, ws...) + // Promote the warnings to the current one. + s.asyncSets[curr] = warningsOnlySeriesSet(ws) + // One of the sets failed, ensure rest of the sets returns nothing. (All or nothing logic). + for i := range s.asyncSets { + if curr != i { + s.asyncSets[i] = noopGenericSeriesSet{} + } + } + break + } + // Exhausted set. + s.asyncSets[i] = warningsOnlySeriesSet(ws) + } + + s.done = true + }) + + switch s.asyncSets[curr].(type) { + case warningsOnlySeriesSet, noopGenericSeriesSet: + return s.asyncSets[curr], false + default: + return s.asyncSets[curr], true + } + } +} + +func (s *secondaryQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet { + if s.done { + panic("secondaryQuerier: Select invoked after first Next of any returned SeriesSet was done") + } + return &lazySeriesSet{create: s.createFn(s.genericQuerier.Select(sortSeries, hints, matchers...))} +} diff --git a/tsdb/block_test.go b/tsdb/block_test.go index d7711a6cdb..1e95353ee4 100644 --- a/tsdb/block_test.go +++ b/tsdb/block_test.go @@ -203,9 +203,7 @@ func TestCorruptedChunk(t *testing.T) { querier, err := NewBlockQuerier(b, 0, 1) testutil.Ok(t, err) defer func() { testutil.Ok(t, querier.Close()) }() - set, ws, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + set := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) // Check query err. testutil.Equals(t, false, set.Next()) diff --git a/tsdb/cmd/tsdb/main.go b/tsdb/cmd/tsdb/main.go index ca4c481f63..e41f2f71f8 100644 --- a/tsdb/cmd/tsdb/main.go +++ b/tsdb/cmd/tsdb/main.go @@ -617,18 +617,7 @@ func dumpSamples(db *tsdb.DBReadOnly, mint, maxt int64) (err error) { err = merr.Err() }() - ss, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "", ".*")) - if err != nil { - return err - } - - if len(ws) > 0 { - var merr tsdb_errors.MultiError - for _, w := range ws { - merr.Add(w) - } - return merr.Err() - } + ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "", ".*")) for ss.Next() { series := ss.At() @@ -643,6 +632,14 @@ func dumpSamples(db *tsdb.DBReadOnly, mint, maxt int64) (err error) { } } + if ws := ss.Warnings(); len(ws) > 0 { + var merr tsdb_errors.MultiError + for _, w := range ws { + merr.Add(w) + } + return merr.Err() + } + if ss.Err() != nil { return ss.Err() } diff --git a/tsdb/db_test.go b/tsdb/db_test.go index 6c12147a94..e4fa3e7d05 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -67,15 +67,12 @@ func openTestDB(t testing.TB, opts *Options, rngs []int64) (db *DB, close func() // query runs a matcher query against the querier and fully expands its data. func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[string][]tsdbutil.Sample { - ss, ws, err := q.Select(false, nil, matchers...) + ss := q.Select(false, nil, matchers...) defer func() { testutil.Ok(t, q.Close()) }() - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) result := map[string][]tsdbutil.Sample{} - for ss.Next() { series := ss.At() @@ -95,6 +92,7 @@ func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[str result[name] = samples } testutil.Ok(t, ss.Err()) + testutil.Equals(t, 0, len(ss.Warnings())) return result } @@ -315,9 +313,7 @@ Outer: q, err := db.Querier(context.TODO(), 0, numSamples) testutil.Ok(t, err) - res, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) expSamples := make([]tsdbutil.Sample, 0, len(c.remaint)) for _, ts := range c.remaint { @@ -333,6 +329,7 @@ Outer: testutil.Equals(t, eok, rok) if !eok { + testutil.Equals(t, 0, len(res.Warnings())) continue Outer } sexp := expss.At() @@ -491,10 +488,7 @@ func TestDB_Snapshot(t *testing.T) { defer func() { testutil.Ok(t, querier.Close()) }() // sum values - seriesSet, ws, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) - + seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) sum := 0.0 for seriesSet.Next() { series := seriesSet.At().Iterator() @@ -505,6 +499,7 @@ func TestDB_Snapshot(t *testing.T) { testutil.Ok(t, series.Err()) } testutil.Ok(t, seriesSet.Err()) + testutil.Equals(t, 0, len(seriesSet.Warnings())) testutil.Equals(t, 1000.0, sum) } @@ -546,10 +541,7 @@ func TestDB_Snapshot_ChunksOutsideOfCompactedRange(t *testing.T) { defer func() { testutil.Ok(t, querier.Close()) }() // Sum values. - seriesSet, ws, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) - + seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) sum := 0.0 for seriesSet.Next() { series := seriesSet.At().Iterator() @@ -560,6 +552,7 @@ func TestDB_Snapshot_ChunksOutsideOfCompactedRange(t *testing.T) { testutil.Ok(t, series.Err()) } testutil.Ok(t, seriesSet.Err()) + testutil.Equals(t, 0, len(seriesSet.Warnings())) // Since we snapshotted with MaxTime - 10, so expect 10 less samples. testutil.Equals(t, 1000.0-10, sum) @@ -618,9 +611,7 @@ Outer: testutil.Ok(t, err) defer func() { testutil.Ok(t, q.Close()) }() - res, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) expSamples := make([]tsdbutil.Sample, 0, len(c.remaint)) for _, ts := range c.remaint { @@ -641,6 +632,7 @@ Outer: testutil.Equals(t, eok, rok) if !eok { + testutil.Equals(t, 0, len(res.Warnings())) continue Outer } sexp := expss.At() @@ -792,10 +784,7 @@ func TestDB_e2e(t *testing.T) { q, err := db.Querier(context.TODO(), mint, maxt) testutil.Ok(t, err) - ss, ws, err := q.Select(false, nil, qry.ms...) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) - + ss := q.Select(false, nil, qry.ms...) result := map[string][]tsdbutil.Sample{} for ss.Next() { @@ -810,6 +799,7 @@ func TestDB_e2e(t *testing.T) { } testutil.Ok(t, ss.Err()) + testutil.Equals(t, 0, len(ss.Warnings())) testutil.Equals(t, expected, result) q.Close() @@ -968,9 +958,7 @@ func TestTombstoneClean(t *testing.T) { testutil.Ok(t, err) defer q.Close() - res, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) expSamples := make([]tsdbutil.Sample, 0, len(c.remaint)) for _, ts := range c.remaint { @@ -1004,6 +992,7 @@ func TestTombstoneClean(t *testing.T) { testutil.Equals(t, errExp, errRes) testutil.Equals(t, smplExp, smplRes) } + testutil.Equals(t, 0, len(res.Warnings())) for _, b := range db.Blocks() { testutil.Equals(t, tombstones.NewMemTombstones(), b.tombstones) @@ -1306,19 +1295,17 @@ func TestNotMatcherSelectsLabelsUnsetSeries(t *testing.T) { defer func() { testutil.Ok(t, q.Close()) }() for _, c := range cases { - ss, ws, err := q.Select(false, nil, c.selector...) + ss := q.Select(false, nil, c.selector...) + lres, _, ws, err := expandSeriesSet(ss) testutil.Ok(t, err) testutil.Equals(t, 0, len(ws)) - - lres, _, err := expandSeriesSet(ss) - testutil.Ok(t, err) testutil.Equals(t, c.series, lres) } } // expandSeriesSet returns the raw labels in the order they are retrieved from // the series set and the samples keyed by Labels().String(). -func expandSeriesSet(ss storage.SeriesSet) ([]labels.Labels, map[string][]sample, error) { +func expandSeriesSet(ss storage.SeriesSet) ([]labels.Labels, map[string][]sample, storage.Warnings, error) { resultLabels := []labels.Labels{} resultSamples := map[string][]sample{} for ss.Next() { @@ -1332,7 +1319,7 @@ func expandSeriesSet(ss storage.SeriesSet) ([]labels.Labels, map[string][]sample resultLabels = append(resultLabels, series.Labels()) resultSamples[series.Labels().String()] = samples } - return resultLabels, resultSamples, ss.Err() + return resultLabels, resultSamples, ss.Warnings(), ss.Err() } func TestOverlappingBlocksDetectsAllOverlaps(t *testing.T) { @@ -2503,9 +2490,7 @@ func TestDBReadOnly_FlushWAL(t *testing.T) { defer func() { testutil.Ok(t, querier.Close()) }() // Sum the values. - seriesSet, ws, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, defaultLabelName, "flush")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, defaultLabelName, "flush")) sum := 0.0 for seriesSet.Next() { @@ -2517,6 +2502,7 @@ func TestDBReadOnly_FlushWAL(t *testing.T) { testutil.Ok(t, series.Err()) } testutil.Ok(t, seriesSet.Err()) + testutil.Equals(t, 0, len(seriesSet.Warnings())) testutil.Equals(t, 1000.0, sum) } @@ -2568,11 +2554,11 @@ func TestDBCannotSeePartialCommits(t *testing.T) { testutil.Ok(t, err) defer querier.Close() - ss, _, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + ss := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + _, seriesSet, ws, err := expandSeriesSet(ss) testutil.Ok(t, err) + testutil.Equals(t, 0, len(ws)) - _, seriesSet, err := expandSeriesSet(ss) - testutil.Ok(t, err) values := map[float64]struct{}{} for _, series := range seriesSet { values[series[len(series)-1].v] = struct{}{} @@ -2608,16 +2594,16 @@ func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) { defer querierAfterAddButBeforeCommit.Close() // None of the queriers should return anything after the Add but before the commit. - ss, _, err := querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - testutil.Ok(t, err) - _, seriesSet, err := expandSeriesSet(ss) + ss := querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + _, seriesSet, ws, err := expandSeriesSet(ss) testutil.Ok(t, err) + testutil.Equals(t, 0, len(ws)) testutil.Equals(t, map[string][]sample{}, seriesSet) - ss, _, err = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - testutil.Ok(t, err) - _, seriesSet, err = expandSeriesSet(ss) + ss = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + _, seriesSet, ws, err = expandSeriesSet(ss) testutil.Ok(t, err) + testutil.Equals(t, 0, len(ws)) testutil.Equals(t, map[string][]sample{}, seriesSet) // This commit is after the queriers are created, so should not be returned. @@ -2625,17 +2611,17 @@ func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) { testutil.Ok(t, err) // Nothing returned for querier created before the Add. - ss, _, err = querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - testutil.Ok(t, err) - _, seriesSet, err = expandSeriesSet(ss) + ss = querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + _, seriesSet, ws, err = expandSeriesSet(ss) testutil.Ok(t, err) + testutil.Equals(t, 0, len(ws)) testutil.Equals(t, map[string][]sample{}, seriesSet) // Series exists but has no samples for querier created after Add. - ss, _, err = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - testutil.Ok(t, err) - _, seriesSet, err = expandSeriesSet(ss) + ss = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + _, seriesSet, ws, err = expandSeriesSet(ss) testutil.Ok(t, err) + testutil.Equals(t, 0, len(ws)) testutil.Equals(t, map[string][]sample{`{foo="bar"}`: {}}, seriesSet) querierAfterCommit, err := db.Querier(context.Background(), 0, 1000000) @@ -2643,10 +2629,10 @@ func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) { defer querierAfterCommit.Close() // Samples are returned for querier created after Commit. - ss, _, err = querierAfterCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) - testutil.Ok(t, err) - _, seriesSet, err = expandSeriesSet(ss) + ss = querierAfterCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + _, seriesSet, ws, err = expandSeriesSet(ss) testutil.Ok(t, err) + testutil.Equals(t, 0, len(ws)) testutil.Equals(t, map[string][]sample{`{foo="bar"}`: {{t: 0, v: 0}}}, seriesSet) } diff --git a/tsdb/head_test.go b/tsdb/head_test.go index e717b07b77..f3071a5687 100644 --- a/tsdb/head_test.go +++ b/tsdb/head_test.go @@ -562,9 +562,7 @@ func TestHeadDeleteSimple(t *testing.T) { for _, h := range []*Head{head, reloadedHead} { q, err := NewBlockQuerier(h, h.MinTime(), h.MaxTime()) testutil.Ok(t, err) - actSeriesSet, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, lblDefault.Name, lblDefault.Value)) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + actSeriesSet := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, lblDefault.Name, lblDefault.Value)) testutil.Ok(t, q.Close()) expSeriesSet := newMockSeriesSet([]storage.Series{ newSeries(map[string]string{lblDefault.Name: lblDefault.Value}, func() []tsdbutil.Sample { @@ -583,6 +581,8 @@ func TestHeadDeleteSimple(t *testing.T) { if !eok { testutil.Ok(t, h.Close()) + testutil.Ok(t, actSeriesSet.Err()) + testutil.Equals(t, 0, len(actSeriesSet.Warnings())) continue Outer } expSeries := expSeriesSet.At() @@ -623,13 +623,15 @@ func TestDeleteUntilCurMax(t *testing.T) { // Test the series returns no samples. The series is cleared only after compaction. q, err := NewBlockQuerier(hb, 0, 100000) testutil.Ok(t, err) - res, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) testutil.Assert(t, res.Next(), "series is not present") s := res.At() it := s.Iterator() testutil.Assert(t, !it.Next(), "expected no samples") + for res.Next() { + } + testutil.Ok(t, res.Err()) + testutil.Equals(t, 0, len(res.Warnings())) // Add again and test for presence. app = hb.Appender() @@ -638,15 +640,17 @@ func TestDeleteUntilCurMax(t *testing.T) { testutil.Ok(t, app.Commit()) q, err = NewBlockQuerier(hb, 0, 100000) testutil.Ok(t, err) - res, ws, err = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + res = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b")) testutil.Assert(t, res.Next(), "series don't exist") exps := res.At() it = exps.Iterator() resSamples, err := expandSeriesIterator(it) testutil.Ok(t, err) testutil.Equals(t, []tsdbutil.Sample{sample{11, 1}}, resSamples) + for res.Next() { + } + testutil.Ok(t, res.Err()) + testutil.Equals(t, 0, len(res.Warnings())) } func TestDeletedSamplesAndSeriesStillInWALAfterCheckpoint(t *testing.T) { @@ -807,9 +811,7 @@ func TestDelete_e2e(t *testing.T) { q, err := NewBlockQuerier(hb, 0, 100000) testutil.Ok(t, err) defer q.Close() - ss, ws, err := q.Select(true, nil, del.ms...) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + ss := q.Select(true, nil, del.ms...) // Build the mockSeriesSet. matchedSeries := make([]storage.Series, 0, len(matched)) for _, m := range matched { @@ -850,6 +852,8 @@ func TestDelete_e2e(t *testing.T) { testutil.Equals(t, errExp, errRes) testutil.Equals(t, smplExp, smplRes) } + testutil.Ok(t, ss.Err()) + testutil.Equals(t, 0, len(ss.Warnings())) } } } @@ -1118,11 +1122,12 @@ func TestUncommittedSamplesNotLostOnTruncate(t *testing.T) { testutil.Ok(t, err) defer q.Close() - ss, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) - + ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1")) testutil.Equals(t, true, ss.Next()) + for ss.Next() { + } + testutil.Ok(t, ss.Err()) + testutil.Equals(t, 0, len(ss.Warnings())) } func TestRemoveSeriesAfterRollbackAndTruncate(t *testing.T) { @@ -1148,11 +1153,9 @@ func TestRemoveSeriesAfterRollbackAndTruncate(t *testing.T) { testutil.Ok(t, err) defer q.Close() - ss, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1")) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) - + ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1")) testutil.Equals(t, false, ss.Next()) + testutil.Equals(t, 0, len(ss.Warnings())) // Truncate again, this time the series should be deleted testutil.Ok(t, h.Truncate(2050)) @@ -1434,11 +1437,11 @@ func TestMemSeriesIsolation(t *testing.T) { testutil.Ok(t, err) defer querier.Close() - ss, _, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + ss := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar")) + _, seriesSet, ws, err := expandSeriesSet(ss) testutil.Ok(t, err) + testutil.Equals(t, 0, len(ws)) - _, seriesSet, err := expandSeriesSet(ss) - testutil.Ok(t, err) for _, series := range seriesSet { return int(series[len(series)-1].v) } @@ -1790,8 +1793,9 @@ func testHeadSeriesChunkRace(t *testing.T) { h.gc() wg.Done() }() - ss, _, err := q.Select(false, nil, matcher) - testutil.Ok(t, err) + ss := q.Select(false, nil, matcher) + for ss.Next() { + } testutil.Ok(t, ss.Err()) wg.Wait() } diff --git a/tsdb/querier.go b/tsdb/querier.go index 5af688b26d..9d7ac8c462 100644 --- a/tsdb/querier.go +++ b/tsdb/querier.go @@ -85,9 +85,9 @@ func (q *querier) lvals(qs []storage.Querier, n string) ([]string, storage.Warni return mergeStrings(s1, s2), ws, nil } -func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet { if len(q.blocks) == 0 { - return storage.EmptySeriesSet(), nil, nil + return storage.EmptySeriesSet() } if len(q.blocks) == 1 { // Sorting Head series is slow, and unneeded when only the @@ -96,18 +96,12 @@ func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*lab } ss := make([]storage.SeriesSet, len(q.blocks)) - var ws storage.Warnings for i, b := range q.blocks { // We have to sort if blocks > 1 as MergedSeriesSet requires it. - s, w, err := b.Select(true, hints, ms...) - ws = append(ws, w...) - if err != nil { - return nil, ws, err - } - ss[i] = s + ss[i] = b.Select(true, hints, ms...) } - return NewMergedSeriesSet(ss), ws, nil + return NewMergedSeriesSet(ss) } func (q *querier) Close() error { @@ -125,31 +119,23 @@ type verticalQuerier struct { querier } -func (q *verticalQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *verticalQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet { return q.sel(sortSeries, hints, q.blocks, ms) } -func (q *verticalQuerier) sel(sortSeries bool, hints *storage.SelectHints, qs []storage.Querier, ms []*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *verticalQuerier) sel(sortSeries bool, hints *storage.SelectHints, qs []storage.Querier, ms []*labels.Matcher) storage.SeriesSet { if len(qs) == 0 { - return storage.EmptySeriesSet(), nil, nil + return storage.EmptySeriesSet() } if len(qs) == 1 { return qs[0].Select(sortSeries, hints, ms...) } l := len(qs) / 2 - var ws storage.Warnings - a, w, err := q.sel(sortSeries, hints, qs[:l], ms) - ws = append(ws, w...) - if err != nil { - return nil, ws, err - } - b, w, err := q.sel(sortSeries, hints, qs[l:], ms) - ws = append(ws, w...) - if err != nil { - return nil, ws, err - } - return newMergedVerticalSeriesSet(a, b), ws, nil + return newMergedVerticalSeriesSet( + q.sel(sortSeries, hints, qs[:l], ms), + q.sel(sortSeries, hints, qs[l:], ms), + ) } // NewBlockQuerier returns a querier against the reader. @@ -189,7 +175,7 @@ type blockQuerier struct { mint, maxt int64 } -func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) { +func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet { var base storage.DeprecatedChunkSeriesSet var err error @@ -199,7 +185,7 @@ func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms .. base, err = LookupChunkSeries(q.index, q.tombstones, ms...) } if err != nil { - return nil, nil, err + return storage.ErrSeriesSet(err) } mint := q.mint @@ -218,7 +204,7 @@ func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms .. mint: mint, maxt: maxt, - }, nil, nil + } } func (q *blockQuerier) LabelValues(name string) ([]string, storage.Warnings, error) { @@ -501,6 +487,14 @@ func (s *mergedSeriesSet) Err() error { return s.err } +func (s *mergedSeriesSet) Warnings() storage.Warnings { + var ws storage.Warnings + for _, ss := range s.all { + ws = append(ws, ss.Warnings()...) + } + return ws +} + // nextAll is to call Next() for all SeriesSet. // Because the order of the SeriesSet slice will affect the results, // we need to use an buffer slice to hold the order. @@ -509,7 +503,10 @@ func (s *mergedSeriesSet) nextAll() { for _, ss := range s.all { if ss.Next() { s.buf = append(s.buf, ss) - } else if ss.Err() != nil { + continue + } + + if ss.Err() != nil { s.done = true s.err = ss.Err() break @@ -623,6 +620,13 @@ func (s *mergedVerticalSeriesSet) Err() error { return s.b.Err() } +func (s *mergedVerticalSeriesSet) Warnings() storage.Warnings { + var ws storage.Warnings + ws = append(ws, s.a.Warnings()...) + ws = append(ws, s.b.Warnings()...) + return ws +} + func (s *mergedVerticalSeriesSet) compare() int { if s.adone { return 1 @@ -848,8 +852,9 @@ func (s *blockSeriesSet) Next() bool { return false } -func (s *blockSeriesSet) At() storage.Series { return s.cur } -func (s *blockSeriesSet) Err() error { return s.err } +func (s *blockSeriesSet) At() storage.Series { return s.cur } +func (s *blockSeriesSet) Err() error { return s.err } +func (s *blockSeriesSet) Warnings() storage.Warnings { return nil } // chunkSeries is a series that is backed by a sequence of chunks holding // time series data. diff --git a/tsdb/querier_bench_test.go b/tsdb/querier_bench_test.go index 6d5f04a658..1f67551902 100644 --- a/tsdb/querier_bench_test.go +++ b/tsdb/querier_bench_test.go @@ -155,8 +155,7 @@ func BenchmarkQuerierSelect(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - ss, _, err := q.Select(sorted, nil, matcher) - testutil.Ok(b, err) + ss := q.Select(sorted, nil, matcher) for ss.Next() { } testutil.Ok(b, ss.Err()) diff --git a/tsdb/querier_test.go b/tsdb/querier_test.go index ea6a99ee1c..795fbb93f8 100644 --- a/tsdb/querier_test.go +++ b/tsdb/querier_test.go @@ -39,12 +39,14 @@ import ( type mockSeriesSet struct { next func() bool series func() storage.Series + ws func() storage.Warnings err func() error } -func (m *mockSeriesSet) Next() bool { return m.next() } -func (m *mockSeriesSet) At() storage.Series { return m.series() } -func (m *mockSeriesSet) Err() error { return m.err() } +func (m *mockSeriesSet) Next() bool { return m.next() } +func (m *mockSeriesSet) At() storage.Series { return m.series() } +func (m *mockSeriesSet) Err() error { return m.err() } +func (m *mockSeriesSet) Warnings() storage.Warnings { return m.ws() } func newMockSeriesSet(list []storage.Series) *mockSeriesSet { i := -1 @@ -57,11 +59,11 @@ func newMockSeriesSet(list []storage.Series) *mockSeriesSet { return list[i] }, err: func() error { return nil }, + ws: func() storage.Warnings { return nil }, } } func TestMergedSeriesSet(t *testing.T) { - cases := []struct { // The input sets in order (samples in series in b are strictly // after those in a). @@ -373,15 +375,14 @@ Outer: maxt: c.maxt, } - res, ws, err := querier.Select(false, nil, c.ms...) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + res := querier.Select(false, nil, c.ms...) for { eok, rok := c.exp.Next(), res.Next() testutil.Equals(t, eok, rok) if !eok { + testutil.Equals(t, 0, len(res.Warnings())) continue Outer } sexp := c.exp.At() @@ -536,15 +537,14 @@ Outer: maxt: c.maxt, } - res, ws, err := querier.Select(false, nil, c.ms...) - testutil.Ok(t, err) - testutil.Equals(t, 0, len(ws)) + res := querier.Select(false, nil, c.ms...) for { eok, rok := c.exp.Next(), res.Next() testutil.Equals(t, eok, rok) if !eok { + testutil.Equals(t, 0, len(res.Warnings())) continue Outer } sexp := c.exp.At() @@ -1654,7 +1654,7 @@ func BenchmarkQuerySeek(b *testing.B) { b.ResetTimer() b.ReportAllocs() - ss, ws, err := sq.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) + ss := sq.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*")) for ss.Next() { it := ss.At().Iterator() for t := mint; t <= maxt; t++ { @@ -1664,7 +1664,7 @@ func BenchmarkQuerySeek(b *testing.B) { } testutil.Ok(b, ss.Err()) testutil.Ok(b, err) - testutil.Equals(b, 0, len(ws)) + testutil.Equals(b, 0, len(ss.Warnings())) }) } } @@ -1792,9 +1792,11 @@ func BenchmarkSetMatcher(b *testing.B) { b.ResetTimer() b.ReportAllocs() for n := 0; n < b.N; n++ { - _, ws, err := que.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "test", c.pattern)) - testutil.Ok(b, err) - testutil.Equals(b, 0, len(ws)) + ss := que.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "test", c.pattern)) + for ss.Next() { + } + testutil.Ok(b, ss.Err()) + testutil.Equals(b, 0, len(ss.Warnings())) } }) } @@ -2252,9 +2254,7 @@ func benchQuery(b *testing.B, expExpansions int, q storage.Querier, selectors la b.ResetTimer() b.ReportAllocs() for i := 0; i < b.N; i++ { - ss, ws, err := q.Select(false, nil, selectors...) - testutil.Ok(b, err) - testutil.Equals(b, 0, len(ws)) + ss := q.Select(false, nil, selectors...) var actualExpansions int for ss.Next() { s := ss.At() @@ -2264,6 +2264,8 @@ func benchQuery(b *testing.B, expExpansions int, q storage.Querier, selectors la } actualExpansions++ } + testutil.Ok(b, ss.Err()) + testutil.Equals(b, 0, len(ss.Warnings())) testutil.Equals(b, expExpansions, actualExpansions) testutil.Ok(b, ss.Err()) } diff --git a/web/api/v1/api.go b/web/api/v1/api.go index 8a6e3a9005..ac40c398e7 100644 --- a/web/api/v1/api.go +++ b/web/api/v1/api.go @@ -596,13 +596,8 @@ func (api *API) series(r *http.Request) (result apiFuncResult) { } var sets []storage.SeriesSet - var warnings storage.Warnings for _, mset := range matcherSets { - s, wrn, err := q.Select(false, nil, mset...) - warnings = append(warnings, wrn...) - if err != nil { - return apiFuncResult{nil, &apiError{errorExec, err}, warnings, closer} - } + s := q.Select(false, nil, mset...) sets = append(sets, s) } @@ -611,6 +606,8 @@ func (api *API) series(r *http.Request) (result apiFuncResult) { for set.Next() { metrics = append(metrics, set.At().Labels()) } + + warnings := set.Warnings() if set.Err() != nil { return apiFuncResult{nil, &apiError{errorExec, set.Err()}, warnings, closer} } @@ -1226,12 +1223,9 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { return } for i, query := range req.Queries { - err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error { + ws, err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) (storage.Warnings, error) { // The streaming API has to provide the series sorted. - set, _, err := querier.Select(true, hints, filteredMatchers...) - if err != nil { - return err - } + set := querier.Select(true, hints, filteredMatchers...) return remote.StreamChunkedReadResponses( remote.NewChunkedWriter(w, f), @@ -1241,6 +1235,9 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { api.remoteReadMaxBytesInFrame, ) }) + for _, w := range ws { + level.Warn(api.logger).Log("msg", "warnings on remote read query", "err", w.Error()) + } if err != nil { if httpErr, ok := err.(remote.HTTPError); ok { http.Error(w, httpErr.Error(), httpErr.Status()) @@ -1259,22 +1256,26 @@ func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) { Results: make([]*prompb.QueryResult, len(req.Queries)), } for i, query := range req.Queries { - err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error { - set, _, err := querier.Select(false, hints, filteredMatchers...) - if err != nil { - return err - } + ws, err := api.remoteReadQuery(ctx, query, externalLabels, func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) (storage.Warnings, error) { + set := querier.Select(false, hints, filteredMatchers...) - resp.Results[i], err = remote.ToQueryResult(set, api.remoteReadSampleLimit) + var ( + ws storage.Warnings + err error + ) + resp.Results[i], ws, err = remote.ToQueryResult(set, api.remoteReadSampleLimit) if err != nil { - return err + return ws, err } for _, ts := range resp.Results[i].Timeseries { ts.Labels = remote.MergeLabels(ts.Labels, sortedExternalLabels) } - return nil + return ws, nil }) + for _, w := range ws { + level.Warn(api.logger).Log("msg", "warnings on remote read query", "err", w.Error()) + } if err != nil { if httpErr, ok := err.(remote.HTTPError); ok { http.Error(w, httpErr.Error(), httpErr.Status()) @@ -1318,15 +1319,15 @@ func filterExtLabelsFromMatchers(pbMatchers []*prompb.LabelMatcher, externalLabe return filteredMatchers, nil } -func (api *API) remoteReadQuery(ctx context.Context, query *prompb.Query, externalLabels map[string]string, seriesHandleFn func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) error) error { +func (api *API) remoteReadQuery(ctx context.Context, query *prompb.Query, externalLabels map[string]string, seriesHandleFn func(querier storage.Querier, hints *storage.SelectHints, filteredMatchers []*labels.Matcher) (storage.Warnings, error)) (storage.Warnings, error) { filteredMatchers, err := filterExtLabelsFromMatchers(query.Matchers, externalLabels) if err != nil { - return err + return nil, err } querier, err := api.Queryable.Querier(ctx, query.StartTimestampMs, query.EndTimestampMs) if err != nil { - return err + return nil, err } defer func() { if err := querier.Close(); err != nil { diff --git a/web/api/v1/api_test.go b/web/api/v1/api_test.go index aa86c4124d..a70a107627 100644 --- a/web/api/v1/api_test.go +++ b/web/api/v1/api_test.go @@ -517,12 +517,8 @@ func setupRemote(s storage.Storage) *httptest.Server { } defer querier.Close() - set, _, err := querier.Select(false, hints, matchers...) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } - resp.Results[i], err = remote.ToQueryResult(set, 1e6) + set := querier.Select(false, hints, matchers...) + resp.Results[i], _, err = remote.ToQueryResult(set, 1e6) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return diff --git a/web/federate.go b/web/federate.go index 4815a5a166..ba601bce9e 100644 --- a/web/federate.go +++ b/web/federate.go @@ -95,16 +95,7 @@ func (h *Handler) federation(w http.ResponseWriter, req *http.Request) { var sets []storage.SeriesSet for _, mset := range matcherSets { - s, wrns, err := q.Select(false, hints, mset...) - if wrns != nil { - level.Debug(h.logger).Log("msg", "Federation select returned warnings", "warnings", wrns) - federationWarnings.Add(float64(len(wrns))) - } - if err != nil { - federationErrors.Inc() - http.Error(w, err.Error(), http.StatusInternalServerError) - return - } + s := q.Select(false, hints, mset...) sets = append(sets, s) } @@ -142,6 +133,10 @@ func (h *Handler) federation(w http.ResponseWriter, req *http.Request) { Point: promql.Point{T: t, V: v}, }) } + if ws := set.Warnings(); len(ws) > 0 { + level.Debug(h.logger).Log("msg", "Federation select returned warnings", "warnings", ws) + federationWarnings.Add(float64(len(ws))) + } if set.Err() != nil { federationErrors.Inc() http.Error(w, set.Err().Error(), http.StatusInternalServerError) From dff31c28266d128d0ad3da96eebc453e64f9310e Mon Sep 17 00:00:00 2001 From: Lisa Carpenter <48000707+licarpen@users.noreply.github.com> Date: Wed, 10 Jun 2020 07:44:07 -0700 Subject: [PATCH 12/31] preserve state on page reload for alert status filters and annotations with useLocalStorage(), add toggleAnnotations method, and add passing tests (#7374) Signed-off-by: Lisa Carpenter --- .../src/pages/alerts/AlertContents.test.tsx | 39 +++++++ .../src/pages/alerts/AlertContents.tsx | 18 ++-- .../__snapshots__/AlertContents.test.tsx.snap | 100 ++++++++++++++++++ 3 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 web/ui/react-app/src/pages/alerts/AlertContents.test.tsx create mode 100644 web/ui/react-app/src/pages/alerts/__snapshots__/AlertContents.test.tsx.snap diff --git a/web/ui/react-app/src/pages/alerts/AlertContents.test.tsx b/web/ui/react-app/src/pages/alerts/AlertContents.test.tsx new file mode 100644 index 0000000000..78b5a05513 --- /dev/null +++ b/web/ui/react-app/src/pages/alerts/AlertContents.test.tsx @@ -0,0 +1,39 @@ +import React from 'react'; +import { shallow } from 'enzyme'; +import AlertsContent from './AlertContents'; + +describe('AlertsContent', () => { + const defaultProps = { + groups: [], + statsCount: { + inactive: 0, + pending: 0, + firing: 0, + }, + }; + const wrapper = shallow(); + + it('matches a snapshot', () => { + expect(wrapper).toMatchSnapshot(); + }); + + [ + { selector: '#inactive-toggler', propName: 'inactive' }, + { selector: '#pending-toggler', propName: 'pending' }, + { selector: '#firing-toggler', propName: 'firing' }, + ].forEach(testCase => { + it(`toggles the ${testCase.propName} checkbox from true to false when clicked and back to true when clicked again`, () => { + wrapper.find(testCase.selector).invoke('onClick')(testCase.propName); + expect(wrapper.find(testCase.selector).prop('checked')).toBe(false); + wrapper.find(testCase.selector).invoke('onClick')(testCase.propName); + expect(wrapper.find(testCase.selector).prop('checked')).toBe(true); + }); + }); + + it('toggles the "annotations" checkbox from false to true when clicked and back to true when clicked again', () => { + wrapper.find('#show-annotations-toggler').invoke('onClick')(); + expect(wrapper.find('#show-annotations-toggler').prop('checked')).toBe(true); + wrapper.find('#show-annotations-toggler').invoke('onClick')(); + expect(wrapper.find('#show-annotations-toggler').prop('checked')).toBe(false); + }); +}); diff --git a/web/ui/react-app/src/pages/alerts/AlertContents.tsx b/web/ui/react-app/src/pages/alerts/AlertContents.tsx index 8f09e5f455..2cbfa71762 100644 --- a/web/ui/react-app/src/pages/alerts/AlertContents.tsx +++ b/web/ui/react-app/src/pages/alerts/AlertContents.tsx @@ -1,9 +1,10 @@ -import React, { FC, useState, Fragment } from 'react'; +import React, { FC, Fragment } from 'react'; import { Badge } from 'reactstrap'; import CollapsibleAlertPanel from './CollapsibleAlertPanel'; import Checkbox from '../../components/Checkbox'; import { isPresent } from '../../utils'; import { Rule } from '../../types/types'; +import { useLocalStorage } from '../../hooks/useLocalStorage'; export type RuleState = keyof RuleStatus; @@ -40,12 +41,12 @@ const stateColorTuples: Array<[RuleState, 'success' | 'warning' | 'danger']> = [ ]; const AlertsContent: FC = ({ groups = [], statsCount }) => { - const [filter, setFilter] = useState>({ + const [filter, setFilter] = useLocalStorage('alerts-status-filter', { firing: true, pending: true, inactive: true, }); - const [showAnnotations, setShowAnnotations] = useState(false); + const [showAnnotations, setShowAnnotations] = useLocalStorage('alerts-annotations-status', { checked: false }); const toggleFilter = (ruleState: RuleState) => () => { setFilter({ @@ -54,6 +55,10 @@ const AlertsContent: FC = ({ groups = [], statsCount }) => { }); }; + const toggleAnnotations = () => { + setShowAnnotations({ checked: !showAnnotations.checked }); + }; + return ( <>
@@ -62,7 +67,7 @@ const AlertsContent: FC = ({ groups = [], statsCount }) => { @@ -74,8 +79,9 @@ const AlertsContent: FC = ({ groups = [], statsCount }) => { })} setShowAnnotations(!showAnnotations)} + onClick={() => toggleAnnotations()} > Show annotations @@ -90,7 +96,7 @@ const AlertsContent: FC = ({ groups = [], statsCount }) => { {group.rules.map((rule, j) => { return ( filter[rule.state] && ( - + ) ); })} diff --git a/web/ui/react-app/src/pages/alerts/__snapshots__/AlertContents.test.tsx.snap b/web/ui/react-app/src/pages/alerts/__snapshots__/AlertContents.test.tsx.snap new file mode 100644 index 0000000000..fdbba2c9f8 --- /dev/null +++ b/web/ui/react-app/src/pages/alerts/__snapshots__/AlertContents.test.tsx.snap @@ -0,0 +1,100 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`AlertsContent matches a snapshot 1`] = ` + +
+ + + inactive + ( + 0 + ) + + + + + pending + ( + 0 + ) + + + + + firing + ( + 0 + ) + + + + + Show annotations + + +
+ +`; From 7b1c0d6b6684398ce2fee1bbf2deac2b6f96dc5e Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Thu, 11 Jun 2020 16:02:09 +0200 Subject: [PATCH 13/31] discovery/kubernetes: Fix incorrect premature break of reading results Previously `max` results stopped reading from results in tests prematurely, as it stopped when `max` number of items were received from the channel instead of `max` number of unique target groups received. This caused flaky tests where the same target group was received multiple times, as Kubernetes informers may emit the same event multiple times. Before this patch, running this test repeatedly failed eventually. After this patch I have run the test many thousand times without failure. ```bash go test -run TestEndpointsDiscoveryNamespaces -count 1000 -test.v ``` Signed-off-by: Frederic Branczyk --- discovery/kubernetes/kubernetes_test.go | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/discovery/kubernetes/kubernetes_test.go b/discovery/kubernetes/kubernetes_test.go index 803002e063..eb1f97fa26 100644 --- a/discovery/kubernetes/kubernetes_test.go +++ b/discovery/kubernetes/kubernetes_test.go @@ -107,14 +107,18 @@ func (d k8sDiscoveryTest) Run(t *testing.T) { // readResultWithTimeout reads all targegroups from channel with timeout. // It merges targegroups by source and sends the result to result channel. func readResultWithTimeout(t *testing.T, ch <-chan []*targetgroup.Group, max int, timeout time.Duration, resChan chan<- map[string]*targetgroup.Group) { - allTgs := make([][]*targetgroup.Group, 0) - + res := make(map[string]*targetgroup.Group) Loop: for { select { case tgs := <-ch: - allTgs = append(allTgs, tgs) - if len(allTgs) == max { + for _, tg := range tgs { + if tg == nil { + continue + } + res[tg.Source] = tg + } + if len(res) == max { // Reached max target groups we may get, break fast. break Loop } @@ -122,21 +126,11 @@ Loop: // Because we use queue, an object that is created then // deleted or updated may be processed only once. // So possibly we may skip events, timed out here. - t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(allTgs), max) + t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(res), max) break Loop } } - // Merge by source and sent it to channel. - res := make(map[string]*targetgroup.Group) - for _, tgs := range allTgs { - for _, tg := range tgs { - if tg == nil { - continue - } - res[tg.Source] = tg - } - } resChan <- res } From b5d61fb66cdcd096499102e3e7760ec432153532 Mon Sep 17 00:00:00 2001 From: Martin Lee Date: Thu, 11 Jun 2020 18:25:58 +0100 Subject: [PATCH 14/31] Add AMI to labels scraped during service discovery. (#7386) Signed-off-by: Martin Lee --- discovery/ec2/ec2.go | 2 ++ docs/configuration/configuration.md | 1 + 2 files changed, 3 insertions(+) diff --git a/discovery/ec2/ec2.go b/discovery/ec2/ec2.go index 1fdb9cc4e4..8d94a0c65e 100644 --- a/discovery/ec2/ec2.go +++ b/discovery/ec2/ec2.go @@ -38,6 +38,7 @@ import ( const ( ec2Label = model.MetaLabelPrefix + "ec2_" + ec2LabelAMI = ec2Label + "ami" ec2LabelAZ = ec2Label + "availability_zone" ec2LabelArch = ec2Label + "architecture" ec2LabelInstanceID = ec2Label + "instance_id" @@ -212,6 +213,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName) } + labels[ec2LabelAMI] = model.LabelValue(*inst.ImageId) labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone) labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name) labels[ec2LabelInstanceType] = model.LabelValue(*inst.InstanceType) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index a972b24925..7945fa49ae 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -426,6 +426,7 @@ the public IP address with relabeling. The following meta labels are available on targets during [relabeling](#relabel_config): +* `__meta_ec2_ami`: the EC2 Amazon Machine Image * `__meta_ec2_architecture`: the architecture of the instance * `__meta_ec2_availability_zone`: the availability zone in which the instance is running * `__meta_ec2_instance_id`: the EC2 instance ID From ab6203b7c7cc9454476151ff992680e82e690538 Mon Sep 17 00:00:00 2001 From: Krasimir Georgiev <8903888+krasi-georgiev@users.noreply.github.com> Date: Fri, 12 Jun 2020 13:29:26 +0300 Subject: [PATCH 15/31] add head compaction test (#7338) --- tsdb/db_test.go | 92 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/tsdb/db_test.go b/tsdb/db_test.go index e4fa3e7d05..daafbd963e 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -2858,3 +2858,95 @@ func TestChunkReader_ConcurrentReads(t *testing.T) { } testutil.Ok(t, r.Close()) } + +// TestCompactHead ensures that the head compaction +// creates a block that is ready for loading and +// does not cause data loss. +// This test: +// * opens a storage; +// * appends values; +// * compacts the head; and +// * queries the db to ensure the samples are present from the compacted head. +func TestCompactHead(t *testing.T) { + dbDir, err := ioutil.TempDir("", "testFlush") + testutil.Ok(t, err) + defer func() { testutil.Ok(t, os.RemoveAll(dbDir)) }() + + // Open a DB and append data to the WAL. + tsdbCfg := &Options{ + RetentionDuration: int64(time.Hour * 24 * 15 / time.Millisecond), + NoLockfile: true, + MinBlockDuration: int64(time.Hour * 2 / time.Millisecond), + MaxBlockDuration: int64(time.Hour * 2 / time.Millisecond), + WALCompression: true, + } + + db, err := Open(dbDir, log.NewNopLogger(), prometheus.NewRegistry(), tsdbCfg) + testutil.Ok(t, err) + app := db.Appender() + var expSamples []sample + maxt := 100 + for i := 0; i < maxt; i++ { + val := rand.Float64() + _, err := app.Add(labels.FromStrings("a", "b"), int64(i), val) + testutil.Ok(t, err) + expSamples = append(expSamples, sample{int64(i), val}) + } + testutil.Ok(t, app.Commit()) + + // Compact the Head to create a new block. + testutil.Ok(t, db.CompactHead(NewRangeHead(db.Head(), 0, int64(maxt)-1))) + testutil.Ok(t, db.Close()) + + // Delete everything but the new block and + // reopen the db to query it to ensure it includes the head data. + testutil.Ok(t, deleteNonBlocks(db.Dir())) + db, err = Open(dbDir, log.NewNopLogger(), prometheus.NewRegistry(), tsdbCfg) + testutil.Ok(t, err) + testutil.Equals(t, 1, len(db.Blocks())) + testutil.Equals(t, int64(maxt), db.Head().MinTime()) + defer func() { testutil.Ok(t, db.Close()) }() + querier, err := db.Querier(context.Background(), 0, int64(maxt)-1) + testutil.Ok(t, err) + defer func() { testutil.Ok(t, querier.Close()) }() + + seriesSet, _, err := querier.Select(false, nil, &labels.Matcher{Type: labels.MatchEqual, Name: "a", Value: "b"}) + testutil.Ok(t, err) + var actSamples []sample + + for seriesSet.Next() { + series := seriesSet.At().Iterator() + for series.Next() { + time, val := series.At() + actSamples = append(actSamples, sample{int64(time), val}) + } + testutil.Ok(t, series.Err()) + } + testutil.Equals(t, expSamples, actSamples) + testutil.Ok(t, seriesSet.Err()) +} + +func deleteNonBlocks(dbDir string) error { + dirs, err := ioutil.ReadDir(dbDir) + if err != nil { + return err + } + for _, dir := range dirs { + if ok := isBlockDir(dir); !ok { + if err := os.RemoveAll(filepath.Join(dbDir, dir.Name())); err != nil { + return err + } + } + } + dirs, err = ioutil.ReadDir(dbDir) + if err != nil { + return err + } + for _, dir := range dirs { + if ok := isBlockDir(dir); !ok { + return errors.Errorf("root folder:%v still hase non block directory:%v", dbDir, dir.Name()) + } + } + + return nil +} From f893786153a471e8833558768511f838f99cc72e Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Sun, 14 Jun 2020 18:37:23 +0200 Subject: [PATCH 16/31] Fix TSDB test failure (#7394) PR #7338 was not rebased on top of master and interface had changed. Signed-off-by: Julien Pivotto --- tsdb/db_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tsdb/db_test.go b/tsdb/db_test.go index daafbd963e..6b16203ef2 100644 --- a/tsdb/db_test.go +++ b/tsdb/db_test.go @@ -2910,8 +2910,7 @@ func TestCompactHead(t *testing.T) { testutil.Ok(t, err) defer func() { testutil.Ok(t, querier.Close()) }() - seriesSet, _, err := querier.Select(false, nil, &labels.Matcher{Type: labels.MatchEqual, Name: "a", Value: "b"}) - testutil.Ok(t, err) + seriesSet := querier.Select(false, nil, &labels.Matcher{Type: labels.MatchEqual, Name: "a", Value: "b"}) var actSamples []sample for seriesSet.Next() { From abb4fba3c2df96a81666c02eaf4c84e80fc20ce4 Mon Sep 17 00:00:00 2001 From: Hrishikesh Barman Date: Sun, 14 Jun 2020 23:29:56 +0530 Subject: [PATCH 17/31] Remove unused env var CLUSTER_NAME. (#7391) Signed-off-by: Hrishikesh Barman --- .github/workflows/funcbench.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/funcbench.yml b/.github/workflows/funcbench.yml index 92fc890b6b..3cb719cb22 100644 --- a/.github/workflows/funcbench.yml +++ b/.github/workflows/funcbench.yml @@ -9,7 +9,6 @@ jobs: AUTH_FILE: ${{ secrets.TEST_INFRA_GKE_AUTH }} BRANCH: ${{ github.event.client_payload.BRANCH }} BENCH_FUNC_REGEX: ${{ github.event.client_payload.BENCH_FUNC_REGEX }} - CLUSTER_NAME: test-infra GITHUB_TOKEN: ${{ secrets.PROMBOT_TOKEN }} GITHUB_ORG: prometheus GITHUB_REPO: prometheus From 7eaffa71802821087c8ee42dbda652453ec1f867 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Linas=20Med=C5=BEi=C5=ABnas?= Date: Mon, 15 Jun 2020 13:32:10 +0300 Subject: [PATCH 18/31] Fix off-by-one error in funcHistogramQuantile / ensureMonotonic (#7393) * Fix off-by-one error in funcHistogramQuantile / ensureMonotonic * Additional coverage for nonmonotonic histogram buckets Signed-off-by: Linas Medziunas --- promql/quantile.go | 2 +- promql/testdata/histograms.test | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/promql/quantile.go b/promql/quantile.go index 6e63efab6f..a25f8917c9 100644 --- a/promql/quantile.go +++ b/promql/quantile.go @@ -168,7 +168,7 @@ func coalesceBuckets(buckets buckets) buckets { func ensureMonotonic(buckets buckets) { max := buckets[0].count - for i := range buckets[1:] { + for i := 1; i < len(buckets); i++ { switch { case buckets[i].count > max: max = buckets[i].count diff --git a/promql/testdata/histograms.test b/promql/testdata/histograms.test index b7ec13682e..ec236576b4 100644 --- a/promql/testdata/histograms.test +++ b/promql/testdata/histograms.test @@ -151,21 +151,25 @@ eval instant at 50m histogram_quantile(0.5, rate(request_duration_seconds_bucket # A histogram with nonmonotonic bucket counts. This may happen when recording # rule evaluation or federation races scrape ingestion, causing some buckets -# counts to be derived from fewer samples. The wrong answer we want to avoid -# is for histogram_quantile(0.99, nonmonotonic_bucket) to return ~1000 instead -# of 1. +# counts to be derived from fewer samples. load 5m - nonmonotonic_bucket{le="0.1"} 0+1x10 - nonmonotonic_bucket{le="1"} 0+9x10 - nonmonotonic_bucket{le="10"} 0+8x10 - nonmonotonic_bucket{le="100"} 0+8x10 + nonmonotonic_bucket{le="0.1"} 0+2x10 + nonmonotonic_bucket{le="1"} 0+1x10 + nonmonotonic_bucket{le="10"} 0+5x10 + nonmonotonic_bucket{le="100"} 0+4x10 nonmonotonic_bucket{le="1000"} 0+9x10 - nonmonotonic_bucket{le="+Inf"} 0+9x10 + nonmonotonic_bucket{le="+Inf"} 0+8x10 # Nonmonotonic buckets +eval instant at 50m histogram_quantile(0.01, nonmonotonic_bucket) + {} 0.0045 + +eval instant at 50m histogram_quantile(0.5, nonmonotonic_bucket) + {} 8.5 + eval instant at 50m histogram_quantile(0.99, nonmonotonic_bucket) - {} 0.989875 + {} 979.75 # Buckets with different representations of the same upper bound. eval instant at 50m histogram_quantile(0.5, rate(mixed_bucket[5m])) From 268b4c29e101706cbff0f90f0882bf05608fc56c Mon Sep 17 00:00:00 2001 From: Jordan Neufeld Date: Mon, 15 Jun 2020 10:03:07 -0500 Subject: [PATCH 19/31] Support extended durations in promtool unit tests (Fixes #6285) (#6297) * Fixed evaluation_time duration parsing in promtool unit tests (Fixes #6285) Signed-off-by: Jordan Neufeld --- cmd/promtool/testdata/alerts.yml | 13 +++++++ cmd/promtool/testdata/unittest.yml | 21 +++++++++++ cmd/promtool/unittest.go | 58 +++++++++++++++--------------- cmd/promtool/unittest_test.go | 42 ++++++++++++++++++++++ 4 files changed, 106 insertions(+), 28 deletions(-) create mode 100644 cmd/promtool/testdata/alerts.yml create mode 100644 cmd/promtool/testdata/unittest.yml create mode 100644 cmd/promtool/unittest_test.go diff --git a/cmd/promtool/testdata/alerts.yml b/cmd/promtool/testdata/alerts.yml new file mode 100644 index 0000000000..29485cdcc3 --- /dev/null +++ b/cmd/promtool/testdata/alerts.yml @@ -0,0 +1,13 @@ +# This is the rules file. + +groups: + - name: example + rules: + - alert: InstanceDown + expr: up == 0 + for: 5m + labels: + severity: page + annotations: + summary: "Instance {{ $labels.instance }} down" + description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes." diff --git a/cmd/promtool/testdata/unittest.yml b/cmd/promtool/testdata/unittest.yml new file mode 100644 index 0000000000..f8fb2d758a --- /dev/null +++ b/cmd/promtool/testdata/unittest.yml @@ -0,0 +1,21 @@ +rule_files: + - alerts.yml + +evaluation_interval: 1m + +tests: + - interval: 1m + input_series: + - series: 'up{job="prometheus", instance="localhost:9090"}' + values: "0+0x1440" + alert_rule_test: + - eval_time: 1d + alertname: InstanceDown + exp_alerts: + - exp_labels: + severity: page + instance: localhost:9090 + job: prometheus + exp_annotations: + summary: "Instance localhost:9090 down" + description: "localhost:9090 of job prometheus has been down for more than 5 minutes." diff --git a/cmd/promtool/unittest.go b/cmd/promtool/unittest.go index 78867ce012..42368b3915 100644 --- a/cmd/promtool/unittest.go +++ b/cmd/promtool/unittest.go @@ -29,6 +29,7 @@ import ( "github.com/pkg/errors" yaml "gopkg.in/yaml.v2" + "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/promql" "github.com/prometheus/prometheus/promql/parser" @@ -76,15 +77,16 @@ func ruleUnitTest(filename string) []error { } if unitTestInp.EvaluationInterval == 0 { - unitTestInp.EvaluationInterval = 1 * time.Minute + unitTestInp.EvaluationInterval = model.Duration(1 * time.Minute) } // Bounds for evaluating the rules. mint := time.Unix(0, 0).UTC() maxd := unitTestInp.maxEvalTime() maxt := mint.Add(maxd) + evalInterval := time.Duration(unitTestInp.EvaluationInterval) // Rounding off to nearest Eval time (> maxt). - maxt = maxt.Add(unitTestInp.EvaluationInterval / 2).Round(unitTestInp.EvaluationInterval) + maxt = maxt.Add(evalInterval / 2).Round(evalInterval) // Giving number for groups mentioned in the file for ordering. // Lower number group should be evaluated before higher number group. @@ -99,7 +101,7 @@ func ruleUnitTest(filename string) []error { // Testing. var errs []error for _, t := range unitTestInp.Tests { - ers := t.test(mint, maxt, unitTestInp.EvaluationInterval, groupOrderMap, + ers := t.test(mint, maxt, evalInterval, groupOrderMap, unitTestInp.RuleFiles...) if ers != nil { errs = append(errs, ers...) @@ -114,10 +116,10 @@ func ruleUnitTest(filename string) []error { // unitTestFile holds the contents of a single unit test file. type unitTestFile struct { - RuleFiles []string `yaml:"rule_files"` - EvaluationInterval time.Duration `yaml:"evaluation_interval,omitempty"` - GroupEvalOrder []string `yaml:"group_eval_order"` - Tests []testGroup `yaml:"tests"` + RuleFiles []string `yaml:"rule_files"` + EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"` + GroupEvalOrder []string `yaml:"group_eval_order"` + Tests []testGroup `yaml:"tests"` } func (utf *unitTestFile) maxEvalTime() time.Duration { @@ -157,7 +159,7 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error { // testGroup is a group of input series and tests associated with it. type testGroup struct { - Interval time.Duration `yaml:"interval"` + Interval model.Duration `yaml:"interval"` InputSeries []series `yaml:"input_series"` AlertRuleTests []alertTestCase `yaml:"alert_rule_test,omitempty"` PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"` @@ -182,7 +184,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou Logger: log.NewNopLogger(), } m := rules.NewManager(opts) - groupsMap, ers := m.LoadGroups(tg.Interval, tg.ExternalLabels, ruleFiles...) + groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, ruleFiles...) if ers != nil { return ers } @@ -193,11 +195,11 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou // This avoids storing them in memory, as the number of evals might be high. // All the `eval_time` for which we have unit tests for alerts. - alertEvalTimesMap := map[time.Duration]struct{}{} + alertEvalTimesMap := map[model.Duration]struct{}{} // Map of all the eval_time+alertname combination present in the unit tests. - alertsInTest := make(map[time.Duration]map[string]struct{}) + alertsInTest := make(map[model.Duration]map[string]struct{}) // Map of all the unit tests for given eval_time. - alertTests := make(map[time.Duration][]alertTestCase) + alertTests := make(map[model.Duration][]alertTestCase) for _, alert := range tg.AlertRuleTests { alertEvalTimesMap[alert.EvalTime] = struct{}{} @@ -208,7 +210,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou alertTests[alert.EvalTime] = append(alertTests[alert.EvalTime], alert) } - alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap)) + alertEvalTimes := make([]model.Duration, 0, len(alertEvalTimesMap)) for k := range alertEvalTimesMap { alertEvalTimes = append(alertEvalTimes, k) } @@ -242,8 +244,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou } for { - if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= alertEvalTimes[curr] && - alertEvalTimes[curr] < ts.Add(evalInterval).Sub(mint)) { + if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) && + time.Duration(alertEvalTimes[curr]) < ts.Add(time.Duration(evalInterval)).Sub(mint)) { break } @@ -322,7 +324,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou // Checking promql expressions. Outer: for _, testCase := range tg.PromqlExprTests { - got, err := query(suite.Context(), testCase.Expr, mint.Add(testCase.EvalTime), + got, err := query(suite.Context(), testCase.Expr, mint.Add(time.Duration(testCase.EvalTime)), suite.QueryEngine(), suite.Queryable()) if err != nil { errs = append(errs, errors.Errorf(" expr: %q, time: %s, err: %s", testCase.Expr, @@ -373,15 +375,15 @@ Outer: // seriesLoadingString returns the input series in PromQL notation. func (tg *testGroup) seriesLoadingString() string { - result := "" - result += "load " + shortDuration(tg.Interval) + "\n" + + result := fmt.Sprintf("load %v\n", shortDuration(tg.Interval)) for _, is := range tg.InputSeries { - result += " " + is.Series + " " + is.Values + "\n" + result += fmt.Sprintf(" %v %v\n", is.Series, is.Values) } return result } -func shortDuration(d time.Duration) string { +func shortDuration(d model.Duration) string { s := d.String() if strings.HasSuffix(s, "m0s") { s = s[:len(s)-2] @@ -407,7 +409,7 @@ func orderedGroups(groupsMap map[string]*rules.Group, groupOrderMap map[string]i // maxEvalTime returns the max eval time among all alert and promql unit tests. func (tg *testGroup) maxEvalTime() time.Duration { - var maxd time.Duration + var maxd model.Duration for _, alert := range tg.AlertRuleTests { if alert.EvalTime > maxd { maxd = alert.EvalTime @@ -418,7 +420,7 @@ func (tg *testGroup) maxEvalTime() time.Duration { maxd = pet.EvalTime } } - return maxd + return time.Duration(maxd) } func query(ctx context.Context, qs string, t time.Time, engine *promql.Engine, qu storage.Queryable) (promql.Vector, error) { @@ -483,9 +485,9 @@ type series struct { } type alertTestCase struct { - EvalTime time.Duration `yaml:"eval_time"` - Alertname string `yaml:"alertname"` - ExpAlerts []alert `yaml:"exp_alerts"` + EvalTime model.Duration `yaml:"eval_time"` + Alertname string `yaml:"alertname"` + ExpAlerts []alert `yaml:"exp_alerts"` } type alert struct { @@ -494,9 +496,9 @@ type alert struct { } type promqlTestCase struct { - Expr string `yaml:"expr"` - EvalTime time.Duration `yaml:"eval_time"` - ExpSamples []sample `yaml:"exp_samples"` + Expr string `yaml:"expr"` + EvalTime model.Duration `yaml:"eval_time"` + ExpSamples []sample `yaml:"exp_samples"` } type sample struct { diff --git a/cmd/promtool/unittest_test.go b/cmd/promtool/unittest_test.go new file mode 100644 index 0000000000..0b1511afc7 --- /dev/null +++ b/cmd/promtool/unittest_test.go @@ -0,0 +1,42 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import "testing" + +func TestRulesUnitTest(t *testing.T) { + type args struct { + files []string + } + tests := []struct { + name string + args args + want int + }{ + { + name: "Passing Unit Tests", + args: args{ + files: []string{"./testdata/unittest.yml"}, + }, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := RulesUnitTest(tt.args.files...); got != tt.want { + t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want) + } + }) + } +} From f42ed03dc5be7290a913fe427cdf96f6d1bee792 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Mon, 15 Jun 2020 17:44:40 +0200 Subject: [PATCH 20/31] Optimized bstream reader used by XORChunk iterator (#7390) * Optimized bstream reader Signed-off-by: Marco Pracucci * Fixed linter Signed-off-by: Marco Pracucci * Added license to new file Signed-off-by: Marco Pracucci * Fixed type cast Signed-off-by: Marco Pracucci * Changed comments Signed-off-by: Marco Pracucci * Improved comments and rolledback no-op changes Signed-off-by: Marco Pracucci * Fixed race condition Signed-off-by: Marco Pracucci --- tsdb/chunkenc/bstream.go | 185 +++++++++++++++++++++------------- tsdb/chunkenc/bstream_test.go | 89 ++++++++++++++++ tsdb/chunkenc/xor.go | 42 ++++++-- 3 files changed, 237 insertions(+), 79 deletions(-) create mode 100644 tsdb/chunkenc/bstream_test.go diff --git a/tsdb/chunkenc/bstream.go b/tsdb/chunkenc/bstream.go index 0a02a73035..fdfc222119 100644 --- a/tsdb/chunkenc/bstream.go +++ b/tsdb/chunkenc/bstream.go @@ -49,10 +49,6 @@ type bstream struct { count uint8 // how many bits are valid in current byte } -func newBReader(b []byte) bstream { - return bstream{stream: b, count: 8} -} - func (b *bstream) bytes() []byte { return b.stream } @@ -111,90 +107,141 @@ func (b *bstream) writeBits(u uint64, nbits int) { } } -func (b *bstream) readBit() (bit, error) { - if len(b.stream) == 0 { +type bstreamReader struct { + stream []byte + streamOffset int // The offset from which read the next byte from the stream. + + buffer uint64 // The current buffer, filled from the stream, containing up to 8 bytes from which read bits. + valid uint8 // The number of bits valid to read (from left) in the current buffer. +} + +func newBReader(b []byte) bstreamReader { + return bstreamReader{ + stream: b, + } +} + +func (b *bstreamReader) readBit() (bit, error) { + if b.valid == 0 { + if !b.loadNextBuffer(1) { + return false, io.EOF + } + } + + return b.readBitFast() +} + +// readBitFast is like readBit but can return io.EOF if the internal buffer is empty. +// If it returns io.EOF, the caller should retry reading bits calling readBit(). +// This function must be kept small and a leaf in order to help the compiler inlining it +// and further improve performances. +func (b *bstreamReader) readBitFast() (bit, error) { + if b.valid == 0 { return false, io.EOF } - if b.count == 0 { - b.stream = b.stream[1:] + b.valid-- + bitmask := uint64(1) << b.valid + return (b.buffer & bitmask) != 0, nil +} - if len(b.stream) == 0 { - return false, io.EOF +func (b *bstreamReader) readBits(nbits uint8) (uint64, error) { + if b.valid == 0 { + if !b.loadNextBuffer(nbits) { + return 0, io.EOF } - b.count = 8 } - d := (b.stream[0] << (8 - b.count)) & 0x80 - b.count-- - return d != 0, nil -} + if nbits <= b.valid { + return b.readBitsFast(nbits) + } -func (b *bstream) ReadByte() (byte, error) { - return b.readByte() -} + // We have to read all remaining valid bits from the current buffer and a part from the next one. + bitmask := (uint64(1) << b.valid) - 1 + nbits -= b.valid + v := (b.buffer & bitmask) << nbits + b.valid = 0 -func (b *bstream) readByte() (byte, error) { - if len(b.stream) == 0 { + if !b.loadNextBuffer(nbits) { return 0, io.EOF } - if b.count == 0 { - b.stream = b.stream[1:] + bitmask = (uint64(1) << nbits) - 1 + v = v | ((b.buffer >> (b.valid - nbits)) & bitmask) + b.valid -= nbits - if len(b.stream) == 0 { - return 0, io.EOF - } - return b.stream[0], nil - } + return v, nil +} - if b.count == 8 { - b.count = 0 - return b.stream[0], nil - } - - byt := b.stream[0] << (8 - b.count) - b.stream = b.stream[1:] - - if len(b.stream) == 0 { +// readBitsFast is like readBits but can return io.EOF if the internal buffer is empty. +// If it returns io.EOF, the caller should retry reading bits calling readBits(). +// This function must be kept small and a leaf in order to help the compiler inlining it +// and further improve performances. +func (b *bstreamReader) readBitsFast(nbits uint8) (uint64, error) { + if nbits > b.valid { return 0, io.EOF } - // We just advanced the stream and can assume the shift to be 0. - byt |= b.stream[0] >> b.count + bitmask := (uint64(1) << nbits) - 1 + b.valid -= nbits - return byt, nil + return (b.buffer >> b.valid) & bitmask, nil } -func (b *bstream) readBits(nbits int) (uint64, error) { - var u uint64 - - for nbits >= 8 { - byt, err := b.readByte() - if err != nil { - return 0, err - } - - u = (u << 8) | uint64(byt) - nbits -= 8 +func (b *bstreamReader) ReadByte() (byte, error) { + v, err := b.readBits(8) + if err != nil { + return 0, err } - - if nbits == 0 { - return u, nil - } - - if nbits > int(b.count) { - u = (u << uint(b.count)) | uint64((b.stream[0]<<(8-b.count))>>(8-b.count)) - nbits -= int(b.count) - b.stream = b.stream[1:] - - if len(b.stream) == 0 { - return 0, io.EOF - } - b.count = 8 - } - - u = (u << uint(nbits)) | uint64((b.stream[0]<<(8-b.count))>>(8-uint(nbits))) - b.count -= uint8(nbits) - return u, nil + return byte(v), nil +} + +// loadNextBuffer loads the next bytes from the stream into the internal buffer. +// The input nbits is the minimum number of bits that must be read, but the implementation +// can read more (if possible) to improve performances. +func (b *bstreamReader) loadNextBuffer(nbits uint8) bool { + if b.streamOffset >= len(b.stream) { + return false + } + + // Handle the case there are more then 8 bytes in the buffer (most common case) + // in a optimized way. It's guaranteed that this branch will never read from the + // very last byte of the stream (which suffers race conditions due to concurrent + // writes). + if b.streamOffset+8 < len(b.stream) { + // This is ugly, but significantly faster. + b.buffer = + ((uint64(b.stream[b.streamOffset])) << 56) | + ((uint64(b.stream[b.streamOffset+1])) << 48) | + ((uint64(b.stream[b.streamOffset+2])) << 40) | + ((uint64(b.stream[b.streamOffset+3])) << 32) | + ((uint64(b.stream[b.streamOffset+4])) << 24) | + ((uint64(b.stream[b.streamOffset+5])) << 16) | + ((uint64(b.stream[b.streamOffset+6])) << 8) | + uint64(b.stream[b.streamOffset+7]) + + b.streamOffset += 8 + b.valid = 64 + return true + } + + // We're here if the are 8 or less bytes left in the stream. Since this reader needs + // to handle race conditions with concurrent writes happening on the very last byte + // we make sure to never over more than the minimum requested bits (rounded up to + // the next byte). The following code is slower but called less frequently. + nbytes := int((nbits / 8) + 1) + if b.streamOffset+nbytes > len(b.stream) { + nbytes = len(b.stream) - b.streamOffset + } + + buffer := uint64(0) + for i := 0; i < nbytes; i++ { + buffer = buffer | (uint64(b.stream[b.streamOffset+i]) << uint(8*(nbytes-i-1))) + } + + b.buffer = buffer + b.streamOffset += nbytes + b.valid = uint8(nbytes * 8) + + return true } diff --git a/tsdb/chunkenc/bstream_test.go b/tsdb/chunkenc/bstream_test.go new file mode 100644 index 0000000000..785a4d5dca --- /dev/null +++ b/tsdb/chunkenc/bstream_test.go @@ -0,0 +1,89 @@ +// Copyright 2017 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The code in this file was largely written by Damian Gryski as part of +// https://github.com/dgryski/go-tsz and published under the license below. +// It received minor modifications to suit Prometheus's needs. + +// Copyright (c) 2015,2016 Damian Gryski +// All rights reserved. + +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: + +// * Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +package chunkenc + +import ( + "testing" + + "github.com/prometheus/prometheus/util/testutil" +) + +func TestBstreamReader(t *testing.T) { + // Write to the bit stream. + w := bstream{} + for _, bit := range []bit{true, false} { + w.writeBit(bit) + } + for nbits := 1; nbits <= 64; nbits++ { + w.writeBits(uint64(nbits), nbits) + } + for v := 1; v < 10000; v += 123 { + w.writeBits(uint64(v), 29) + } + + // Read back. + r := newBReader(w.bytes()) + for _, bit := range []bit{true, false} { + v, err := r.readBitFast() + if err != nil { + v, err = r.readBit() + } + testutil.Ok(t, err) + testutil.Equals(t, bit, v) + } + for nbits := uint8(1); nbits <= 64; nbits++ { + v, err := r.readBitsFast(nbits) + if err != nil { + v, err = r.readBits(nbits) + } + testutil.Ok(t, err) + testutil.Equals(t, uint64(nbits), v, "nbits=%d", nbits) + } + for v := 1; v < 10000; v += 123 { + actual, err := r.readBitsFast(29) + if err != nil { + actual, err = r.readBits(29) + } + testutil.Ok(t, err) + testutil.Equals(t, uint64(v), actual, "v=%d", v) + } +} diff --git a/tsdb/chunkenc/xor.go b/tsdb/chunkenc/xor.go index 33c7f10404..e3b4f58b2a 100644 --- a/tsdb/chunkenc/xor.go +++ b/tsdb/chunkenc/xor.go @@ -240,7 +240,7 @@ func (a *xorAppender) writeVDelta(v float64) { } type xorIterator struct { - br bstream + br bstreamReader numTotal uint16 numRead uint16 @@ -328,7 +328,10 @@ func (it *xorIterator) Next() bool { // read delta-of-delta for i := 0; i < 4; i++ { d <<= 1 - bit, err := it.br.readBit() + bit, err := it.br.readBitFast() + if err != nil { + bit, err = it.br.readBit() + } if err != nil { it.err = err return false @@ -350,6 +353,7 @@ func (it *xorIterator) Next() bool { case 0x0e: sz = 20 case 0x0f: + // Do not use fast because it's very unlikely it will succeed. bits, err := it.br.readBits(64) if err != nil { it.err = err @@ -360,7 +364,10 @@ func (it *xorIterator) Next() bool { } if sz != 0 { - bits, err := it.br.readBits(int(sz)) + bits, err := it.br.readBitsFast(sz) + if err != nil { + bits, err = it.br.readBits(sz) + } if err != nil { it.err = err return false @@ -379,7 +386,10 @@ func (it *xorIterator) Next() bool { } func (it *xorIterator) readValue() bool { - bit, err := it.br.readBit() + bit, err := it.br.readBitFast() + if err != nil { + bit, err = it.br.readBit() + } if err != nil { it.err = err return false @@ -388,7 +398,10 @@ func (it *xorIterator) readValue() bool { if bit == zero { // it.val = it.val } else { - bit, err := it.br.readBit() + bit, err := it.br.readBitFast() + if err != nil { + bit, err = it.br.readBit() + } if err != nil { it.err = err return false @@ -397,14 +410,20 @@ func (it *xorIterator) readValue() bool { // reuse leading/trailing zero bits // it.leading, it.trailing = it.leading, it.trailing } else { - bits, err := it.br.readBits(5) + bits, err := it.br.readBitsFast(5) + if err != nil { + bits, err = it.br.readBits(5) + } if err != nil { it.err = err return false } it.leading = uint8(bits) - bits, err = it.br.readBits(6) + bits, err = it.br.readBitsFast(6) + if err != nil { + bits, err = it.br.readBits(6) + } if err != nil { it.err = err return false @@ -417,14 +436,17 @@ func (it *xorIterator) readValue() bool { it.trailing = 64 - it.leading - mbits } - mbits := int(64 - it.leading - it.trailing) - bits, err := it.br.readBits(mbits) + mbits := 64 - it.leading - it.trailing + bits, err := it.br.readBitsFast(mbits) + if err != nil { + bits, err = it.br.readBits(mbits) + } if err != nil { it.err = err return false } vbits := math.Float64bits(it.val) - vbits ^= (bits << it.trailing) + vbits ^= bits << it.trailing it.val = math.Float64frombits(vbits) } From 3c753aba5f3ac334e87c1203ddc6c27cd0005bf3 Mon Sep 17 00:00:00 2001 From: Alex Vandiver Date: Mon, 15 Jun 2020 22:13:27 -0700 Subject: [PATCH 21/31] Add missing newline before inline-code block (#7401) Sections with three backticks require a blank line before them. Signed-off-by: Alex Vandiver --- docs/configuration/configuration.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 7945fa49ae..d01134b2c0 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -1048,6 +1048,7 @@ The following meta labels are available on targets during [relabeling](#relabel_ * `__meta_triton_machine_id`: the UUID of the target See below for the configuration options for Triton discovery: + ```yaml # The information to access the Triton discovery API. From 6e7554639b0edbfe88efc8a44a8a78a10f72e556 Mon Sep 17 00:00:00 2001 From: Manuel Fontan Date: Tue, 16 Jun 2020 10:32:06 +0100 Subject: [PATCH 22/31] Update Readme since jsonnetfmt is available in the jsonnet go implementation since v0.16.0 Signed-off-by: Manuel Fontan --- documentation/prometheus-mixin/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/prometheus-mixin/README.md b/documentation/prometheus-mixin/README.md index c44e70ca0c..a28d91068c 100644 --- a/documentation/prometheus-mixin/README.md +++ b/documentation/prometheus-mixin/README.md @@ -10,12 +10,12 @@ To use them, you need to have `jsonnet` (v0.13+) and `jb` installed. If you have a working Go development environment, it's easiest to run the following: ```bash $ go get github.com/google/go-jsonnet/cmd/jsonnet +$ go get github.com/google/go-jsonnet/cmd/jsonnetfmt $ go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb ``` _Note: The make targets `lint` and `fmt` need the `jsonnetfmt` binary, which is -currently not included in the Go implementation of `jsonnet`. For the time -being, you have to install the [C++ version of +available from [v.0.16.0](https://github.com/google/jsonnet/releases/tag/v0.16.0) in the Go implementation of `jsonnet`. If your jsonnet version is older than 0.16.0 you have to either upgrade or install the [C++ version of jsonnetfmt](https://github.com/google/jsonnet) if you want to use `make lint` or `make fmt`._ From d5a8f2afc4bfc4466c40c78baac5f48196a27706 Mon Sep 17 00:00:00 2001 From: Nicole J <43453127+njingco@users.noreply.github.com> Date: Tue, 16 Jun 2020 07:11:41 -0700 Subject: [PATCH 23/31] Added the remote read histogram (#7334) change remote read queries total metric to a histogram and add read requests counter with status code Signed-off-by: njingco --- storage/remote/client.go | 19 +++++++++++++++++++ storage/remote/read.go | 19 ++++++++++--------- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/storage/remote/client.go b/storage/remote/client.go index 4dce3ec50e..7ff957168c 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -21,6 +21,7 @@ import ( "io" "io/ioutil" "net/http" + "strconv" "strings" "time" @@ -28,6 +29,7 @@ import ( "github.com/golang/snappy" "github.com/opentracing/opentracing-go" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" config_util "github.com/prometheus/common/config" "github.com/prometheus/common/model" "github.com/prometheus/common/version" @@ -40,6 +42,16 @@ const maxErrMsgLen = 256 var userAgent = fmt.Sprintf("Prometheus/%s", version.Version) +var remoteReadQueriesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "read_queries_total", + Help: "The total number of remote read queries.", + }, + []string{remoteName, endpoint, "code"}, +) + // Client allows reading and writing from/to a remote HTTP endpoint. type Client struct { remoteName string // Used to differentiate clients in metrics. @@ -55,6 +67,10 @@ type ClientConfig struct { HTTPClientConfig config_util.HTTPClientConfig } +func init() { + prometheus.MustRegister(remoteReadQueriesTotal) +} + // NewClient creates a new Client. func NewClient(remoteName string, conf *ClientConfig) (*Client, error) { httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage", false) @@ -193,6 +209,9 @@ func (c *Client) Read(ctx context.Context, query *prompb.Query) (*prompb.QueryRe httpResp.Body.Close() }() + remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String(), strconv.Itoa(httpResp.StatusCode)) + remoteReadTotalCounter.Inc() + compressed, err = ioutil.ReadAll(httpResp.Body) if err != nil { return nil, errors.Wrap(err, fmt.Sprintf("error reading response. HTTP status code: %s", httpResp.Status)) diff --git a/storage/remote/read.go b/storage/remote/read.go index f2fd212458..0b53ed7e74 100644 --- a/storage/remote/read.go +++ b/storage/remote/read.go @@ -16,6 +16,7 @@ package remote import ( "context" "fmt" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pkg/labels" @@ -32,26 +33,26 @@ var remoteReadQueries = prometheus.NewGaugeVec( []string{remoteName, endpoint}, ) -var remoteReadQueriesTotal = prometheus.NewCounterVec( - prometheus.CounterOpts{ +var remoteReadQueriesHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ Namespace: namespace, Subsystem: subsystem, - Name: "remote_read_queries_total", - Help: "The total number of remote read queries.", + Name: "read_request_duration_seconds", + Help: "Histogram of the latency for remote read requests.", + Buckets: append(prometheus.DefBuckets, 25, 60), }, []string{remoteName, endpoint}, ) func init() { - prometheus.MustRegister(remoteReadQueries) - prometheus.MustRegister(remoteReadQueriesTotal) + prometheus.MustRegister(remoteReadQueries, remoteReadQueriesHistogram) } // QueryableClient returns a storage.Queryable which queries the given // Client to select series sets. func QueryableClient(c *Client) storage.Queryable { remoteReadQueries.WithLabelValues(c.remoteName, c.url.String()) - remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String()) + remoteReadQueriesHistogram.WithLabelValues(c.remoteName, c.url.String()) return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) { return &querier{ @@ -81,8 +82,8 @@ func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers . remoteReadGauge.Inc() defer remoteReadGauge.Dec() - remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(q.client.remoteName, q.client.url.String()) - remoteReadTotalCounter.Inc() + remoteReadQueriesHistogram := remoteReadQueriesHistogram.WithLabelValues(q.client.remoteName, q.client.url.String()) + remoteReadQueriesHistogram.Observe(time.Since(time.Now()).Seconds()) res, err := q.client.Read(q.ctx, query) if err != nil { From 7b24bb311631f4eaa5d794b849770f436fc75b5a Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Tue, 16 Jun 2020 23:09:05 +0200 Subject: [PATCH 24/31] Docs: normalize bearer_token_file type (#7408) Signed-off-by: Julien Pivotto --- docs/configuration/configuration.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index d01134b2c0..904a2b136f 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -173,7 +173,7 @@ basic_auth: # Sets the `Authorization` header on every scrape request with the bearer token # read from the configured file. It is mutually exclusive with `bearer_token`. -[ bearer_token_file: /path/to/bearer/token/file ] +[ bearer_token_file: ] # Configures the scrape request's TLS settings. tls_config: @@ -873,7 +873,7 @@ namespaces: # if you just want to monitor small subset of pods in large cluster it's recommended to use selectors. # Decision, if selectors should be used or not depends on the particular situation. [ selectors: - [ - role: + [ - role: [ label: ] [ field: ] ]] ``` @@ -942,7 +942,7 @@ basic_auth: # Sets the `Authorization` header on every request with the bearer token # read from the configured file. It is mutually exclusive with `bearer_token` and other authentication mechanisms. # NOTE: The current version of DC/OS marathon (v1.11.0) does not support standard Bearer token authentication. Use `auth_token_file` instead. -[ bearer_token_file: /path/to/bearer/token/file ] +[ bearer_token_file: ] # TLS configuration for connecting to marathon servers tls_config: @@ -1237,7 +1237,7 @@ basic_auth: # Sets the `Authorization` header on every request with the bearer token # read from the configured file. It is mutually exclusive with `bearer_token`. -[ bearer_token_file: /path/to/bearer/token/file ] +[ bearer_token_file: ] # Configures the scrape request's TLS settings. tls_config: @@ -1338,7 +1338,7 @@ basic_auth: # Sets the `Authorization` header on every remote write request with the bearer token # read from the configured file. It is mutually exclusive with `bearer_token`. -[ bearer_token_file: /path/to/bearer/token/file ] +[ bearer_token_file: ] # Configures the remote write request's TLS settings. tls_config: @@ -1410,7 +1410,7 @@ basic_auth: # Sets the `Authorization` header on every remote read request with the bearer token # read from the configured file. It is mutually exclusive with `bearer_token`. -[ bearer_token_file: /path/to/bearer/token/file ] +[ bearer_token_file: ] # Configures the remote read request's TLS settings. tls_config: From 2f1204937173b947feda210ebc75019176f790a7 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Wed, 17 Jun 2020 16:40:00 +0200 Subject: [PATCH 25/31] tsdb: improve logs when encountering corruption (#7308) * tsdb: improve logs when encountering corruption Signed-off-by: Simon Pasquier * Wrap corrupted block errors Signed-off-by: Simon Pasquier * Add file path to head chunks Signed-off-by: Simon Pasquier --- tsdb/chunks/head_chunks.go | 8 ++++---- tsdb/db.go | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/tsdb/chunks/head_chunks.go b/tsdb/chunks/head_chunks.go index a7edbe982c..5e3fc61017 100644 --- a/tsdb/chunks/head_chunks.go +++ b/tsdb/chunks/head_chunks.go @@ -190,23 +190,23 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) { lastSeq := chkFileIndices[0] for _, seq := range chkFileIndices[1:] { if seq != lastSeq+1 { - return errors.Errorf("found unsequential head chunk files %d and %d", lastSeq, seq) + return errors.Errorf("found unsequential head chunk files %s (index: %d) and %s (index: %d)", files[lastSeq], lastSeq, files[seq], seq) } lastSeq = seq } for i, b := range cdm.mmappedChunkFiles { if b.byteSlice.Len() < HeadChunkFileHeaderSize { - return errors.Wrapf(errInvalidSize, "invalid head chunk file header in file %d", i) + return errors.Wrapf(errInvalidSize, "%s: invalid head chunk file header", files[i]) } // Verify magic number. if m := binary.BigEndian.Uint32(b.byteSlice.Range(0, MagicChunksSize)); m != MagicHeadChunks { - return errors.Errorf("invalid magic number %x", m) + return errors.Errorf("%s: invalid magic number %x", files[i], m) } // Verify chunk format version. if v := int(b.byteSlice.Range(MagicChunksSize, MagicChunksSize+ChunksFormatVersionSize)[0]); v != chunksFormatV1 { - return errors.Errorf("invalid chunk format version %d", v) + return errors.Errorf("%s: invalid chunk format version %d", files[i], v) } cdm.size += int64(b.byteSlice.Len()) diff --git a/tsdb/db.go b/tsdb/db.go index 1997dca2d6..a2892b6d9e 100644 --- a/tsdb/db.go +++ b/tsdb/db.go @@ -441,10 +441,14 @@ func (db *DBReadOnly) Blocks() ([]BlockReader, error) { if len(corrupted) > 0 { for _, b := range loadable { if err := b.Close(); err != nil { - level.Warn(db.logger).Log("msg", "Closing a block", err) + level.Warn(db.logger).Log("msg", "Closing block failed", "err", err, "block", b) } } - return nil, errors.Errorf("unexpected corrupted block:%v", corrupted) + var merr tsdb_errors.MultiError + for ulid, err := range corrupted { + merr.Add(errors.Wrapf(err, "corrupted block %s", ulid.String())) + } + return nil, merr.Err() } if len(loadable) == 0 { @@ -880,7 +884,11 @@ func (db *DB) reload() (err error) { block.Close() } } - return fmt.Errorf("unexpected corrupted block:%v", corrupted) + var merr tsdb_errors.MultiError + for ulid, err := range corrupted { + merr.Add(errors.Wrapf(err, "corrupted block %s", ulid.String())) + } + return merr.Err() } // All deletable blocks should not be loaded. @@ -1054,7 +1062,7 @@ func (db *DB) deleteBlocks(blocks map[ulid.ULID]*Block) error { for ulid, block := range blocks { if block != nil { if err := block.Close(); err != nil { - level.Warn(db.logger).Log("msg", "Closing block failed", "err", err) + level.Warn(db.logger).Log("msg", "Closing block failed", "err", err, "block", ulid) } } if err := os.RemoveAll(filepath.Join(db.dir, ulid.String())); err != nil { From d634785944e1416245c97c7d176601d9e57dbd50 Mon Sep 17 00:00:00 2001 From: Simon Pasquier Date: Wed, 17 Jun 2020 17:08:21 +0200 Subject: [PATCH 26/31] tsdb/docs: fix head chunks directory + link from README (#7309) Signed-off-by: Simon Pasquier --- tsdb/docs/format/README.md | 1 + tsdb/docs/format/head_chunks.md | 11 +++++++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tsdb/docs/format/README.md b/tsdb/docs/format/README.md index 39db06fc57..5bd3d98823 100644 --- a/tsdb/docs/format/README.md +++ b/tsdb/docs/format/README.md @@ -2,5 +2,6 @@ * [Index](index.md) * [Chunks](chunks.md) +* [Head Chunks](head_chunks.md) * [Tombstones](tombstones.md) * [Wal](wal.md) diff --git a/tsdb/docs/format/head_chunks.md b/tsdb/docs/format/head_chunks.md index 5a320fa761..ed138f4b6f 100644 --- a/tsdb/docs/format/head_chunks.md +++ b/tsdb/docs/format/head_chunks.md @@ -1,7 +1,7 @@ # Head Chunks on Disk Format -The following describes the format of a chunks file, -which is created in the `wal/chunks/` inside the data directory. +The following describes the format of a chunks file, which is created in the +`chunks_head/` directory inside the data directory. Chunks in the files are referenced from the index by uint64 composed of in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes). @@ -27,10 +27,13 @@ in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes). # Chunk -Unlike chunks in the on-disk blocks, here we additionally store series reference that the chunks belongs to and the mint/maxt of the chunks. This is because we don't have an index associated with these chunks, hence these meta information are used while replaying the chunks. +Unlike chunks in the on-disk blocks, here we additionally store series +reference that the chunks belongs to and the mint/maxt of the chunks. This is +because we don't have an index associated with these chunks, hence these meta +information are used while replaying the chunks. ``` ┌─────────────────────┬───────────────────────┬───────────────────────┬───────────────────┬───────────────┬──────────────┬────────────────┐ | series ref <8 byte> | mint <8 byte, uint64> | maxt <8 byte, uint64> | encoding <1 byte> | len | data │ CRC32 <4 byte> │ └─────────────────────┴───────────────────────┴───────────────────────┴───────────────────┴───────────────┴──────────────┴────────────────┘ -``` \ No newline at end of file +``` From cbb1c263054dbc3b6c9252340e1f6a8e3b52d357 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Wed, 17 Jun 2020 16:51:32 +0200 Subject: [PATCH 27/31] Improve repo sync script * Change the Makefile.common update script to be more generic. * Sync CODE_OF_CONDUCT.md and LICENSE files. * Rename code-of-conduct.md to match github naming convention[0] [0]: https://help.github.com/en/github/building-a-strong-community/adding-a-code-of-conduct-to-your-project Signed-off-by: Ben Kochie --- .circleci/config.yml | 6 +-- code-of-conduct.md => CODE_OF_CONDUCT.md | 0 .../{sync_makefiles.sh => sync_repo_files.sh} | 48 ++++++++++++------- 3 files changed, 35 insertions(+), 19 deletions(-) rename code-of-conduct.md => CODE_OF_CONDUCT.md (100%) rename scripts/{sync_makefiles.sh => sync_repo_files.sh} (66%) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53669bf72d..6ad8cb8427 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -87,11 +87,11 @@ jobs: - setup_remote_docker - run: ./fuzzit.sh fuzzing - makefile_sync: + repo_sync: executor: golang steps: - checkout - - run: ./scripts/sync_makefiles.sh + - run: ./scripts/sync_repo_files.sh workflows: version: 2 @@ -143,7 +143,7 @@ workflows: only: - master jobs: - - makefile_sync: + - repo_sync: context: org-context - fuzzit_fuzzing: context: org-context diff --git a/code-of-conduct.md b/CODE_OF_CONDUCT.md similarity index 100% rename from code-of-conduct.md rename to CODE_OF_CONDUCT.md diff --git a/scripts/sync_makefiles.sh b/scripts/sync_repo_files.sh similarity index 66% rename from scripts/sync_makefiles.sh rename to scripts/sync_repo_files.sh index 8def333b33..f2630b5553 100755 --- a/scripts/sync_makefiles.sh +++ b/scripts/sync_repo_files.sh @@ -7,10 +7,10 @@ set -uo pipefail git_mail="prometheus-team@googlegroups.com" git_user="prombot" -branch="makefile_common" -commit_msg="makefile: update Makefile.common with newer version" -pr_title="Synchronize Makefile.common from prometheus/prometheus" -pr_msg="Propagating changes from master Makefile.common located in prometheus/prometheus." +branch="repo_sync" +commit_msg="Update common Prometheus files" +pr_title="Synchronize common files from prometheus/prometheus" +pr_msg="Propagating changes from prometheus/prometheus default branch." orgs="prometheus prometheus-community" GITHUB_TOKEN="${GITHUB_TOKEN:-}" @@ -19,11 +19,13 @@ if [ -z "${GITHUB_TOKEN}" ]; then exit 1 fi +# List of files that should be synced. +SYNC_FILES="CODE_OF_CONDUCT.md LICENSE Makefile.common" + # Go to the root of the repo cd "$(git rev-parse --show-cdup)" || exit 1 -source_makefile="$(pwd)/Makefile.common" -source_checksum="$(sha256sum Makefile.common | cut -d' ' -f1)" +source_dir="$(pwd)" tmp_dir="$(mktemp -d)" trap 'rm -rf "${tmp_dir}"' EXIT @@ -55,14 +57,25 @@ process_repo() { local org_repo="$1" echo -e "\e[32mAnalyzing '${org_repo}'\e[0m" - target_makefile="$(curl -s --fail "https://raw.githubusercontent.com/${org_repo}/master/Makefile.common")" - if [ -z "${target_makefile}" ]; then - echo "Makefile.common doesn't exist in ${org_repo}" - return - fi - target_checksum="$(echo "${target_makefile}" | sha256sum | cut -d' ' -f1)" - if [ "${source_checksum}" == "${target_checksum}" ]; then - echo "Makefile.common is already in sync." + local needs_update=0 + for source_file in ${SYNC_FILES}; do + source_checksum="$(sha256sum "${souce_dir}/${source_file}" | cut -d' ' -f1)" + + target_file="$(curl -s --fail "https://raw.githubusercontent.com/${org_repo}/master/${source_file}")" + if [[ -z "${target_makefile}" ]]; then + echo "${source_file} doesn't exist in ${org_repo}" + continue + fi + target_checksum="$(echo "${target_file}" | sha256sum | cut -d' ' -f1)" + if [ "${source_checksum}" == "${target_checksum}" ]; then + echo "${source_file} is already in sync." + continue + fi + needs_update=1 + done + + if [[ "${needs_update}" -eq 0 ]] ; then + echo "No files need sync." return fi @@ -71,8 +84,11 @@ process_repo() { cd "${tmp_dir}/${org_repo}" || return 1 git checkout -b "${branch}" || return 1 - # Replace Makefile.common in target repo by one from prometheus/prometheus - cp -f "${source_makefile}" ./ + # Update the files in target repo by one from prometheus/prometheus. + for source_file in ${SYNC_FILES}; do + cp -f "${source_dir}/${source_file}" "./${source_file}" + done + if [ -n "$(git status --porcelain)" ]; then git config user.email "${git_mail}" git config user.name "${git_user}" From 3b529ddbce93df7598237833e2dbbe57b22ba560 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Thu, 18 Jun 2020 11:19:39 +0200 Subject: [PATCH 28/31] Cleanup bstream_test.go based on post-merge feedback received on #7390 (#7413) * Fixed bstream test license Signed-off-by: Marco Pracucci * Simplified bstreamReader.loadNextBuffer() Signed-off-by: Marco Pracucci * Fixed date in license Signed-off-by: Marco Pracucci --- tsdb/chunkenc/bstream.go | 17 +++++------------ tsdb/chunkenc/bstream_test.go | 30 +----------------------------- 2 files changed, 6 insertions(+), 41 deletions(-) diff --git a/tsdb/chunkenc/bstream.go b/tsdb/chunkenc/bstream.go index fdfc222119..ad8077c27e 100644 --- a/tsdb/chunkenc/bstream.go +++ b/tsdb/chunkenc/bstream.go @@ -41,7 +41,10 @@ package chunkenc -import "io" +import ( + "encoding/binary" + "io" +) // bstream is a stream of bits. type bstream struct { @@ -209,17 +212,7 @@ func (b *bstreamReader) loadNextBuffer(nbits uint8) bool { // very last byte of the stream (which suffers race conditions due to concurrent // writes). if b.streamOffset+8 < len(b.stream) { - // This is ugly, but significantly faster. - b.buffer = - ((uint64(b.stream[b.streamOffset])) << 56) | - ((uint64(b.stream[b.streamOffset+1])) << 48) | - ((uint64(b.stream[b.streamOffset+2])) << 40) | - ((uint64(b.stream[b.streamOffset+3])) << 32) | - ((uint64(b.stream[b.streamOffset+4])) << 24) | - ((uint64(b.stream[b.streamOffset+5])) << 16) | - ((uint64(b.stream[b.streamOffset+6])) << 8) | - uint64(b.stream[b.streamOffset+7]) - + b.buffer = binary.BigEndian.Uint64(b.stream[b.streamOffset:]) b.streamOffset += 8 b.valid = 64 return true diff --git a/tsdb/chunkenc/bstream_test.go b/tsdb/chunkenc/bstream_test.go index 785a4d5dca..40425b1caf 100644 --- a/tsdb/chunkenc/bstream_test.go +++ b/tsdb/chunkenc/bstream_test.go @@ -1,4 +1,4 @@ -// Copyright 2017 The Prometheus Authors +// Copyright 2020 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at @@ -11,34 +11,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -// The code in this file was largely written by Damian Gryski as part of -// https://github.com/dgryski/go-tsz and published under the license below. -// It received minor modifications to suit Prometheus's needs. - -// Copyright (c) 2015,2016 Damian Gryski -// All rights reserved. - -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are met: - -// * Redistributions of source code must retain the above copyright notice, -// this list of conditions and the following disclaimer. -// -// * Redistributions in binary form must reproduce the above copyright notice, -// this list of conditions and the following disclaimer in the documentation -// and/or other materials provided with the distribution. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - package chunkenc import ( From fc3aab8de3ddab079c35b3c1ddfc97e5f1aa6a33 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Thu, 18 Jun 2020 15:12:35 +0530 Subject: [PATCH 29/31] Update NOTICE for the XORChunk (#7415) Signed-off-by: Ganesh Vernekar --- NOTICE | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/NOTICE b/NOTICE index 30ce2a8263..5e4f509896 100644 --- a/NOTICE +++ b/NOTICE @@ -86,6 +86,11 @@ https://github.com/samuel/go-zookeeper Copyright (c) 2013, Samuel Stauffer See https://github.com/samuel/go-zookeeper/blob/master/LICENSE for license details. +Time series compression algorithm from Facebook's Gorilla paper +https://github.com/dgryski/go-tsz +Copyright (c) 2015,2016 Damian Gryski +See https://github.com/dgryski/go-tsz/blob/master/LICENSE for license details. + We also use code from a large number of npm packages. For details, see: - https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/package.json - https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/package-lock.json From c61141ce516031f4298ce4c797b6e4611d8297e2 Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Thu, 18 Jun 2020 17:04:41 +0200 Subject: [PATCH 30/31] Add DigitalOcean service discovery (#7407) Signed-off-by: Julien Pivotto --- discovery/config/config.go | 8 + discovery/digitalocean/digitalocean.go | 193 ++++ discovery/digitalocean/digitalocean_test.go | 125 +++ discovery/digitalocean/mock_test.go | 644 +++++++++++++ discovery/manager.go | 6 + docs/configuration/configuration.md | 58 ++ .../examples/prometheus-digitalocean.yml | 25 + go.mod | 1 + go.sum | 4 + .../github.com/digitalocean/godo/.gitignore | 1 + .../github.com/digitalocean/godo/.whitesource | 8 + .../github.com/digitalocean/godo/1-click.go | 52 ++ .../github.com/digitalocean/godo/CHANGELOG.md | 255 ++++++ .../digitalocean/godo/CONTRIBUTING.md | 54 ++ .../github.com/digitalocean/godo/LICENSE.txt | 55 ++ vendor/github.com/digitalocean/godo/README.md | 139 +++ .../github.com/digitalocean/godo/account.go | 60 ++ vendor/github.com/digitalocean/godo/action.go | 108 +++ .../github.com/digitalocean/godo/balance.go | 52 ++ .../digitalocean/godo/billing_history.go | 72 ++ vendor/github.com/digitalocean/godo/cdn.go | 218 +++++ .../digitalocean/godo/certificates.go | 130 +++ .../github.com/digitalocean/godo/databases.go | 845 ++++++++++++++++++ vendor/github.com/digitalocean/godo/doc.go | 2 + .../github.com/digitalocean/godo/domains.go | 341 +++++++ .../digitalocean/godo/droplet_actions.go | 329 +++++++ .../github.com/digitalocean/godo/droplets.go | 592 ++++++++++++ vendor/github.com/digitalocean/godo/errors.go | 24 + .../github.com/digitalocean/godo/firewalls.go | 271 ++++++ .../digitalocean/godo/floating_ips.go | 143 +++ .../digitalocean/godo/floating_ips_actions.go | 109 +++ vendor/github.com/digitalocean/godo/go.mod | 16 + vendor/github.com/digitalocean/godo/go.sum | 42 + vendor/github.com/digitalocean/godo/godo.go | 432 +++++++++ .../digitalocean/godo/image_actions.go | 102 +++ vendor/github.com/digitalocean/godo/images.go | 245 +++++ .../github.com/digitalocean/godo/invoices.go | 225 +++++ vendor/github.com/digitalocean/godo/keys.go | 230 +++++ .../digitalocean/godo/kubernetes.go | 697 +++++++++++++++ vendor/github.com/digitalocean/godo/links.go | 83 ++ .../digitalocean/godo/load_balancers.go | 324 +++++++ vendor/github.com/digitalocean/godo/meta.go | 6 + .../github.com/digitalocean/godo/projects.go | 310 +++++++ .../github.com/digitalocean/godo/regions.go | 68 ++ .../github.com/digitalocean/godo/registry.go | 253 ++++++ vendor/github.com/digitalocean/godo/sizes.go | 72 ++ .../github.com/digitalocean/godo/snapshots.go | 142 +++ .../github.com/digitalocean/godo/storage.go | 261 ++++++ .../digitalocean/godo/storage_actions.go | 132 +++ .../github.com/digitalocean/godo/strings.go | 102 +++ vendor/github.com/digitalocean/godo/tags.go | 247 +++++ .../github.com/digitalocean/godo/timestamp.go | 35 + vendor/github.com/digitalocean/godo/vpcs.go | 201 +++++ .../github.com/google/go-querystring/LICENSE | 27 + .../google/go-querystring/query/encode.go | 320 +++++++ vendor/modules.txt | 4 + 56 files changed, 9500 insertions(+) create mode 100644 discovery/digitalocean/digitalocean.go create mode 100644 discovery/digitalocean/digitalocean_test.go create mode 100644 discovery/digitalocean/mock_test.go create mode 100644 documentation/examples/prometheus-digitalocean.yml create mode 100644 vendor/github.com/digitalocean/godo/.gitignore create mode 100644 vendor/github.com/digitalocean/godo/.whitesource create mode 100644 vendor/github.com/digitalocean/godo/1-click.go create mode 100644 vendor/github.com/digitalocean/godo/CHANGELOG.md create mode 100644 vendor/github.com/digitalocean/godo/CONTRIBUTING.md create mode 100644 vendor/github.com/digitalocean/godo/LICENSE.txt create mode 100644 vendor/github.com/digitalocean/godo/README.md create mode 100644 vendor/github.com/digitalocean/godo/account.go create mode 100644 vendor/github.com/digitalocean/godo/action.go create mode 100644 vendor/github.com/digitalocean/godo/balance.go create mode 100644 vendor/github.com/digitalocean/godo/billing_history.go create mode 100644 vendor/github.com/digitalocean/godo/cdn.go create mode 100644 vendor/github.com/digitalocean/godo/certificates.go create mode 100644 vendor/github.com/digitalocean/godo/databases.go create mode 100644 vendor/github.com/digitalocean/godo/doc.go create mode 100644 vendor/github.com/digitalocean/godo/domains.go create mode 100644 vendor/github.com/digitalocean/godo/droplet_actions.go create mode 100644 vendor/github.com/digitalocean/godo/droplets.go create mode 100644 vendor/github.com/digitalocean/godo/errors.go create mode 100644 vendor/github.com/digitalocean/godo/firewalls.go create mode 100644 vendor/github.com/digitalocean/godo/floating_ips.go create mode 100644 vendor/github.com/digitalocean/godo/floating_ips_actions.go create mode 100644 vendor/github.com/digitalocean/godo/go.mod create mode 100644 vendor/github.com/digitalocean/godo/go.sum create mode 100644 vendor/github.com/digitalocean/godo/godo.go create mode 100644 vendor/github.com/digitalocean/godo/image_actions.go create mode 100644 vendor/github.com/digitalocean/godo/images.go create mode 100644 vendor/github.com/digitalocean/godo/invoices.go create mode 100644 vendor/github.com/digitalocean/godo/keys.go create mode 100644 vendor/github.com/digitalocean/godo/kubernetes.go create mode 100644 vendor/github.com/digitalocean/godo/links.go create mode 100644 vendor/github.com/digitalocean/godo/load_balancers.go create mode 100644 vendor/github.com/digitalocean/godo/meta.go create mode 100644 vendor/github.com/digitalocean/godo/projects.go create mode 100644 vendor/github.com/digitalocean/godo/regions.go create mode 100644 vendor/github.com/digitalocean/godo/registry.go create mode 100644 vendor/github.com/digitalocean/godo/sizes.go create mode 100644 vendor/github.com/digitalocean/godo/snapshots.go create mode 100644 vendor/github.com/digitalocean/godo/storage.go create mode 100644 vendor/github.com/digitalocean/godo/storage_actions.go create mode 100644 vendor/github.com/digitalocean/godo/strings.go create mode 100644 vendor/github.com/digitalocean/godo/tags.go create mode 100644 vendor/github.com/digitalocean/godo/timestamp.go create mode 100644 vendor/github.com/digitalocean/godo/vpcs.go create mode 100644 vendor/github.com/google/go-querystring/LICENSE create mode 100644 vendor/github.com/google/go-querystring/query/encode.go diff --git a/discovery/config/config.go b/discovery/config/config.go index 820de1f7c5..1648655700 100644 --- a/discovery/config/config.go +++ b/discovery/config/config.go @@ -18,6 +18,7 @@ import ( "github.com/prometheus/prometheus/discovery/azure" "github.com/prometheus/prometheus/discovery/consul" + "github.com/prometheus/prometheus/discovery/digitalocean" "github.com/prometheus/prometheus/discovery/dns" "github.com/prometheus/prometheus/discovery/ec2" "github.com/prometheus/prometheus/discovery/file" @@ -40,6 +41,8 @@ type ServiceDiscoveryConfig struct { FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"` // List of Consul service discovery configurations. ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"` + // List of DigitalOcean service discovery configurations. + DigitalOceanSDConfigs []*digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"` // List of Serverset service discovery configurations. ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"` // NerveSDConfigs is a list of Nerve service discovery configurations. @@ -72,6 +75,11 @@ func (c *ServiceDiscoveryConfig) Validate() error { return errors.New("empty or null section in consul_sd_configs") } } + for _, cfg := range c.DigitalOceanSDConfigs { + if cfg == nil { + return errors.New("empty or null section in digitalocean_sd_configs") + } + } for _, cfg := range c.DNSSDConfigs { if cfg == nil { return errors.New("empty or null section in dns_sd_configs") diff --git a/discovery/digitalocean/digitalocean.go b/discovery/digitalocean/digitalocean.go new file mode 100644 index 0000000000..a80a45d528 --- /dev/null +++ b/discovery/digitalocean/digitalocean.go @@ -0,0 +1,193 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package digitalocean + +import ( + "context" + "fmt" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/digitalocean/godo" + "github.com/go-kit/kit/log" + config_util "github.com/prometheus/common/config" + "github.com/prometheus/common/model" + "github.com/prometheus/common/version" + + "github.com/prometheus/prometheus/discovery/refresh" + "github.com/prometheus/prometheus/discovery/targetgroup" +) + +const ( + doLabel = model.MetaLabelPrefix + "digitalocean_" + doLabelID = doLabel + "droplet_id" + doLabelName = doLabel + "droplet_name" + doLabelImage = doLabel + "image" + doLabelPrivateIPv4 = doLabel + "private_ipv4" + doLabelPublicIPv4 = doLabel + "public_ipv4" + doLabelPublicIPv6 = doLabel + "public_ipv6" + doLabelRegion = doLabel + "region" + doLabelSize = doLabel + "size" + doLabelStatus = doLabel + "status" + doLabelFeatures = doLabel + "features" + doLabelTags = doLabel + "tags" + separator = "," +) + +// DefaultSDConfig is the default DigitalOcean SD configuration. +var DefaultSDConfig = SDConfig{ + Port: 80, + RefreshInterval: model.Duration(60 * time.Second), +} + +// SDConfig is the configuration for DigitalOcean based service discovery. +type SDConfig struct { + HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"` + + RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultSDConfig + type plain SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + return nil +} + +// Discovery periodically performs DigitalOcean requests. It implements +// the Discoverer interface. +type Discovery struct { + *refresh.Discovery + client *godo.Client + port int +} + +// NewDiscovery returns a new Discovery which periodically refreshes its targets. +func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) { + d := &Discovery{ + port: conf.Port, + } + + rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "digitalocean_sd", false) + if err != nil { + return nil, err + } + + d.client, err = godo.New( + &http.Client{ + Transport: rt, + Timeout: 5 * time.Duration(conf.RefreshInterval), + }, + godo.SetUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)), + ) + if err != nil { + return nil, fmt.Errorf("error setting up digital ocean agent: %w", err) + } + + d.Discovery = refresh.NewDiscovery( + logger, + "digitalocean", + time.Duration(conf.RefreshInterval), + d.refresh, + ) + return d, nil +} + +func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) { + tg := &targetgroup.Group{ + Source: "DigitalOcean", + } + + droplets, err := d.listDroplets() + if err != nil { + return nil, err + } + for _, droplet := range droplets { + if droplet.Networks == nil || len(droplet.Networks.V4) == 0 { + continue + } + + privateIPv4, err := droplet.PrivateIPv4() + if err != nil { + return nil, fmt.Errorf("error while reading private IPv4 of droplet %d: %w", droplet.ID, err) + } + publicIPv4, err := droplet.PublicIPv4() + if err != nil { + return nil, fmt.Errorf("error while reading public IPv4 of droplet %d: %w", droplet.ID, err) + } + publicIPv6, err := droplet.PublicIPv6() + if err != nil { + return nil, fmt.Errorf("error while reading public IPv6 of droplet %d: %w", droplet.ID, err) + } + + labels := model.LabelSet{ + doLabelID: model.LabelValue(fmt.Sprintf("%d", droplet.ID)), + doLabelName: model.LabelValue(droplet.Name), + doLabelImage: model.LabelValue(droplet.Image.Slug), + doLabelPrivateIPv4: model.LabelValue(privateIPv4), + doLabelPublicIPv4: model.LabelValue(publicIPv4), + doLabelPublicIPv6: model.LabelValue(publicIPv6), + doLabelRegion: model.LabelValue(droplet.Region.Slug), + doLabelSize: model.LabelValue(droplet.SizeSlug), + doLabelStatus: model.LabelValue(droplet.Status), + } + + addr := net.JoinHostPort(publicIPv4, strconv.FormatUint(uint64(d.port), 10)) + labels[model.AddressLabel] = model.LabelValue(addr) + + if len(droplet.Features) > 0 { + // We surround the separated list with the separator as well. This way regular expressions + // in relabeling rules don't have to consider feature positions. + features := separator + strings.Join(droplet.Features, separator) + separator + labels[doLabelFeatures] = model.LabelValue(features) + } + + if len(droplet.Tags) > 0 { + // We surround the separated list with the separator as well. This way regular expressions + // in relabeling rules don't have to consider tag positions. + tags := separator + strings.Join(droplet.Tags, separator) + separator + labels[doLabelTags] = model.LabelValue(tags) + } + + tg.Targets = append(tg.Targets, labels) + } + return []*targetgroup.Group{tg}, nil +} + +func (d *Discovery) listDroplets() ([]godo.Droplet, error) { + var ( + droplets []godo.Droplet + opts = &godo.ListOptions{Page: 1} + ) + for { + paginatedDroplets, resp, err := d.client.Droplets.List(context.Background(), opts) + if err != nil { + return nil, fmt.Errorf("error while listing droplets page %d: %w", opts.Page, err) + } + droplets = append(droplets, paginatedDroplets...) + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + opts.Page++ + } + return droplets, nil +} diff --git a/discovery/digitalocean/digitalocean_test.go b/discovery/digitalocean/digitalocean_test.go new file mode 100644 index 0000000000..86024a5dfe --- /dev/null +++ b/discovery/digitalocean/digitalocean_test.go @@ -0,0 +1,125 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package digitalocean + +import ( + "context" + "fmt" + "net/url" + "testing" + + "github.com/go-kit/kit/log" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/util/testutil" +) + +type DigitalOceanSDTestSuite struct { + Mock *SDMock +} + +func (s *DigitalOceanSDTestSuite) TearDownSuite() { + s.Mock.ShutdownServer() +} + +func (s *DigitalOceanSDTestSuite) SetupTest(t *testing.T) { + s.Mock = NewSDMock(t) + s.Mock.Setup() + + s.Mock.HandleDropletsList() +} + +func TestDigitalOceanSDRefresh(t *testing.T) { + sdmock := &DigitalOceanSDTestSuite{} + sdmock.SetupTest(t) + t.Cleanup(sdmock.TearDownSuite) + + cfg := DefaultSDConfig + cfg.HTTPClientConfig.BearerToken = tokenID + d, err := NewDiscovery(&cfg, log.NewNopLogger()) + testutil.Ok(t, err) + endpoint, err := url.Parse(sdmock.Mock.Endpoint()) + testutil.Ok(t, err) + d.client.BaseURL = endpoint + + ctx := context.Background() + tgs, err := d.refresh(ctx) + testutil.Ok(t, err) + + testutil.Equals(t, 1, len(tgs)) + + tg := tgs[0] + testutil.Assert(t, tg != nil, "tg should not be nil") + testutil.Assert(t, tg.Targets != nil, "tg.targets should not be nil") + testutil.Equals(t, 4, len(tg.Targets)) + + for i, lbls := range []model.LabelSet{ + { + "__address__": model.LabelValue("104.236.32.182:80"), + "__meta_digitalocean_droplet_id": model.LabelValue("3164444"), + "__meta_digitalocean_droplet_name": model.LabelValue("example.com"), + "__meta_digitalocean_image": model.LabelValue("ubuntu-16-04-x64"), + "__meta_digitalocean_private_ipv4": model.LabelValue(""), + "__meta_digitalocean_public_ipv4": model.LabelValue("104.236.32.182"), + "__meta_digitalocean_public_ipv6": model.LabelValue("2604:A880:0800:0010:0000:0000:02DD:4001"), + "__meta_digitalocean_region": model.LabelValue("nyc3"), + "__meta_digitalocean_size": model.LabelValue("s-1vcpu-1gb"), + "__meta_digitalocean_status": model.LabelValue("active"), + "__meta_digitalocean_features": model.LabelValue(",backups,ipv6,virtio,"), + }, + { + "__address__": model.LabelValue("104.131.186.241:80"), + "__meta_digitalocean_droplet_id": model.LabelValue("3164494"), + "__meta_digitalocean_droplet_name": model.LabelValue("prometheus"), + "__meta_digitalocean_image": model.LabelValue("ubuntu-16-04-x64"), + "__meta_digitalocean_private_ipv4": model.LabelValue(""), + "__meta_digitalocean_public_ipv4": model.LabelValue("104.131.186.241"), + "__meta_digitalocean_public_ipv6": model.LabelValue(""), + "__meta_digitalocean_region": model.LabelValue("nyc3"), + "__meta_digitalocean_size": model.LabelValue("s-1vcpu-1gb"), + "__meta_digitalocean_status": model.LabelValue("active"), + "__meta_digitalocean_tags": model.LabelValue(",monitor,"), + "__meta_digitalocean_features": model.LabelValue(",virtio,"), + }, + { + "__address__": model.LabelValue("167.172.111.118:80"), + "__meta_digitalocean_droplet_id": model.LabelValue("175072239"), + "__meta_digitalocean_droplet_name": model.LabelValue("prometheus-demo-old"), + "__meta_digitalocean_image": model.LabelValue("ubuntu-18-04-x64"), + "__meta_digitalocean_private_ipv4": model.LabelValue("10.135.64.211"), + "__meta_digitalocean_public_ipv4": model.LabelValue("167.172.111.118"), + "__meta_digitalocean_public_ipv6": model.LabelValue(""), + "__meta_digitalocean_region": model.LabelValue("fra1"), + "__meta_digitalocean_size": model.LabelValue("s-1vcpu-1gb"), + "__meta_digitalocean_status": model.LabelValue("off"), + "__meta_digitalocean_features": model.LabelValue(",ipv6,private_networking,"), + }, + { + "__address__": model.LabelValue("138.65.56.69:80"), + "__meta_digitalocean_droplet_id": model.LabelValue("176011507"), + "__meta_digitalocean_droplet_name": model.LabelValue("prometheus-demo"), + "__meta_digitalocean_image": model.LabelValue("ubuntu-18-04-x64"), + "__meta_digitalocean_private_ipv4": model.LabelValue("10.135.64.212"), + "__meta_digitalocean_public_ipv4": model.LabelValue("138.65.56.69"), + "__meta_digitalocean_public_ipv6": model.LabelValue("2a03:b0c0:3:f0::cf2:4"), + "__meta_digitalocean_region": model.LabelValue("fra1"), + "__meta_digitalocean_size": model.LabelValue("s-1vcpu-1gb"), + "__meta_digitalocean_status": model.LabelValue("active"), + "__meta_digitalocean_features": model.LabelValue(",ipv6,private_networking,"), + }, + } { + t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) { + testutil.Equals(t, lbls, tg.Targets[i]) + }) + } +} diff --git a/discovery/digitalocean/mock_test.go b/discovery/digitalocean/mock_test.go new file mode 100644 index 0000000000..8f5a934ae2 --- /dev/null +++ b/discovery/digitalocean/mock_test.go @@ -0,0 +1,644 @@ +// Copyright 2020 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package digitalocean + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "testing" +) + +// SDMock is the interface for the DigitalOcean mock +type SDMock struct { + t *testing.T + Server *httptest.Server + Mux *http.ServeMux +} + +// NewSDMock returns a new SDMock. +func NewSDMock(t *testing.T) *SDMock { + return &SDMock{ + t: t, + } +} + +// Endpoint returns the URI to the mock server +func (m *SDMock) Endpoint() string { + return m.Server.URL + "/" +} + +// Setup creates the mock server +func (m *SDMock) Setup() { + m.Mux = http.NewServeMux() + m.Server = httptest.NewServer(m.Mux) +} + +// ShutdownServer creates the mock server +func (m *SDMock) ShutdownServer() { + m.Server.Close() +} + +const tokenID = "3c9a75a2-24fd-4508-b4f2-11f18aa97411" + +// HandleDropletsList mocks droplet list. +func (m *SDMock) HandleDropletsList() { + m.Mux.HandleFunc("/v2/droplets", func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) { + w.WriteHeader(http.StatusForbidden) + return + } + + w.Header().Add("content-type", "application/json; charset=utf-8") + w.Header().Add("ratelimit-limit", "1200") + w.Header().Add("ratelimit-remaining", "965") + w.Header().Add("ratelimit-reset", "1415984218") + w.WriteHeader(http.StatusAccepted) + + page := 1 + if pageQuery, ok := r.URL.Query()["page"]; ok { + var err error + page, err = strconv.Atoi(pageQuery[0]) + if err != nil { + panic(err) + } + } + fmt.Fprint(w, []string{` +{ + "droplets": [ + { + "id": 3164444, + "name": "example.com", + "memory": 1024, + "vcpus": 1, + "disk": 25, + "locked": false, + "status": "active", + "kernel": { + "id": 2233, + "name": "Ubuntu 14.04 x64 vmlinuz-3.13.0-37-generic", + "version": "3.13.0-37-generic" + }, + "created_at": "2014-11-14T16:29:21Z", + "features": [ + "backups", + "ipv6", + "virtio" + ], + "backup_ids": [ + 7938002 + ], + "snapshot_ids": [ + + ], + "image": { + "id": 6918990, + "name": "14.04 x64", + "distribution": "Ubuntu", + "slug": "ubuntu-16-04-x64", + "public": true, + "regions": [ + "nyc1", + "ams1", + "sfo1", + "nyc2", + "ams2", + "sgp1", + "lon1", + "nyc3", + "ams3", + "nyc3" + ], + "created_at": "2014-10-17T20:24:33Z", + "type": "snapshot", + "min_disk_size": 20, + "size_gigabytes": 2.34 + }, + "volume_ids": [ + + ], + "size": { + }, + "size_slug": "s-1vcpu-1gb", + "networks": { + "v4": [ + { + "ip_address": "104.236.32.182", + "netmask": "255.255.192.0", + "gateway": "104.236.0.1", + "type": "public" + } + ], + "v6": [ + { + "ip_address": "2604:A880:0800:0010:0000:0000:02DD:4001", + "netmask": 64, + "gateway": "2604:A880:0800:0010:0000:0000:0000:0001", + "type": "public" + } + ] + }, + "region": { + "name": "New York 3", + "slug": "nyc3", + "sizes": [ + + ], + "features": [ + "virtio", + "private_networking", + "backups", + "ipv6", + "metadata" + ], + "available": null + }, + "tags": [ + + ], + "vpc_uuid": "f9b0769c-e118-42fb-a0c4-fed15ef69662" + }, + { + "id": 3164494, + "name": "prometheus", + "memory": 1024, + "vcpus": 1, + "disk": 25, + "locked": false, + "status": "active", + "kernel": { + "id": 2233, + "name": "Ubuntu 14.04 x64 vmlinuz-3.13.0-37-generic", + "version": "3.13.0-37-generic" + }, + "created_at": "2014-11-14T16:36:31Z", + "features": [ + "virtio" + ], + "backup_ids": [ + + ], + "snapshot_ids": [ + 7938206 + ], + "image": { + "id": 6918990, + "name": "14.04 x64", + "distribution": "Ubuntu", + "slug": "ubuntu-16-04-x64", + "public": true, + "regions": [ + "nyc1", + "ams1", + "sfo1", + "nyc2", + "ams2", + "sgp1", + "lon1", + "nyc3", + "ams3", + "nyc3" + ], + "created_at": "2014-10-17T20:24:33Z", + "type": "snapshot", + "min_disk_size": 20, + "size_gigabytes": 2.34 + }, + "volume_ids": [ + + ], + "size": { + }, + "size_slug": "s-1vcpu-1gb", + "networks": { + "v4": [ + { + "ip_address": "104.131.186.241", + "netmask": "255.255.240.0", + "gateway": "104.131.176.1", + "type": "public" + } + ] + }, + "region": { + "name": "New York 3", + "slug": "nyc3", + "sizes": [ + "s-1vcpu-1gb", + "s-1vcpu-2gb", + "s-1vcpu-3gb", + "s-2vcpu-2gb", + "s-3vcpu-1gb", + "s-2vcpu-4gb", + "s-4vcpu-8gb", + "s-6vcpu-16gb", + "s-8vcpu-32gb", + "s-12vcpu-48gb", + "s-16vcpu-64gb", + "s-20vcpu-96gb", + "s-24vcpu-128gb", + "s-32vcpu-192gb" + ], + "features": [ + "virtio", + "private_networking", + "backups", + "ipv6", + "metadata" + ], + "available": true + }, + "tags": [ + "monitor" + ], + "vpc_uuid": "f9b0769c-e118-42fb-a0c4-fed15ef69662" + } + ], + "links": { + "pages": { + "next": "https://api.digitalocean.com/v2/droplets?page=2&per_page=2", + "last": "https://api.digitalocean.com/v2/droplets?page=2&per_page=2" + } + }, + "meta": { + "total": 4 + } +} + `, + ` +{ + "droplets": [ + { + "id": 175072239, + "name": "prometheus-demo-old", + "memory": 1024, + "vcpus": 1, + "disk": 25, + "locked": false, + "status": "off", + "kernel": null, + "created_at": "2020-01-10T16:47:39Z", + "features": [ + "ipv6", + "private_networking" + ], + "backup_ids": [], + "next_backup_window": null, + "snapshot_ids": [], + "image": { + "id": 53893572, + "name": "18.04.3 (LTS) x64", + "distribution": "Ubuntu", + "slug": "ubuntu-18-04-x64", + "public": true, + "regions": [ + "nyc3", + "nyc1", + "sfo1", + "nyc2", + "ams2", + "sgp1", + "lon1", + "nyc3", + "ams3", + "fra1", + "tor1", + "sfo2", + "blr1", + "sfo3" + ], + "created_at": "2019-10-22T01:38:19Z", + "min_disk_size": 20, + "type": "base", + "size_gigabytes": 2.36, + "description": "Ubuntu 18.04 x64 20191022", + "tags": [], + "status": "available" + }, + "volume_ids": [], + "size": { + "slug": "s-1vcpu-1gb", + "memory": 1024, + "vcpus": 1, + "disk": 25, + "transfer": 1, + "price_monthly": 5, + "price_hourly": 0.00744, + "regions": [ + "ams2", + "ams3", + "blr1", + "fra1", + "lon1", + "nyc1", + "nyc2", + "nyc3", + "sfo1", + "sfo2", + "sfo3", + "sgp1", + "tor1" + ], + "available": true + }, + "size_slug": "s-1vcpu-1gb", + "networks": { + "v4": [ + { + "ip_address": "167.172.111.118", + "netmask": "255.255.240.0", + "gateway": "167.172.176.1", + "type": "public" + }, + { + "ip_address": "10.135.64.211", + "netmask": "255.255.0.0", + "gateway": "10.135.0.1", + "type": "private" + } + ], + "v6": [ + ] + }, + "region": { + "name": "Frankfurt 1", + "slug": "fra1", + "features": [ + "private_networking", + "backups", + "ipv6", + "metadata", + "install_agent", + "storage", + "image_transfer" + ], + "available": true, + "sizes": [ + "s-1vcpu-1gb", + "512mb", + "s-1vcpu-2gb", + "1gb", + "s-3vcpu-1gb", + "s-2vcpu-2gb", + "s-1vcpu-3gb", + "s-2vcpu-4gb", + "2gb", + "s-4vcpu-8gb", + "m-1vcpu-8gb", + "c-2", + "4gb", + "g-2vcpu-8gb", + "gd-2vcpu-8gb", + "m-16gb", + "s-6vcpu-16gb", + "c-4", + "8gb", + "m-2vcpu-16gb", + "m3-2vcpu-16gb", + "g-4vcpu-16gb", + "gd-4vcpu-16gb", + "m6-2vcpu-16gb", + "m-32gb", + "s-8vcpu-32gb", + "c-8", + "16gb", + "m-4vcpu-32gb", + "m3-4vcpu-32gb", + "g-8vcpu-32gb", + "s-12vcpu-48gb", + "gd-8vcpu-32gb", + "m6-4vcpu-32gb", + "m-64gb", + "s-16vcpu-64gb", + "c-16", + "32gb", + "m-8vcpu-64gb", + "m3-8vcpu-64gb", + "g-16vcpu-64gb", + "s-20vcpu-96gb", + "48gb", + "gd-16vcpu-64gb", + "m6-8vcpu-64gb", + "m-128gb", + "s-24vcpu-128gb", + "c-32", + "64gb", + "m-16vcpu-128gb", + "m3-16vcpu-128gb", + "s-32vcpu-192gb", + "m-24vcpu-192gb", + "m-224gb", + "m6-16vcpu-128gb", + "m3-24vcpu-192gb", + "m6-24vcpu-192gb" + ] + }, + "tags": [] + }, + { + "id": 176011507, + "name": "prometheus-demo", + "memory": 1024, + "vcpus": 1, + "disk": 25, + "locked": false, + "status": "active", + "kernel": null, + "created_at": "2020-01-17T12:06:26Z", + "features": [ + "ipv6", + "private_networking" + ], + "backup_ids": [], + "next_backup_window": null, + "snapshot_ids": [], + "image": { + "id": 53893572, + "name": "18.04.3 (LTS) x64", + "distribution": "Ubuntu", + "slug": "ubuntu-18-04-x64", + "public": true, + "regions": [ + "nyc3", + "nyc1", + "sfo1", + "nyc2", + "ams2", + "sgp1", + "lon1", + "nyc3", + "ams3", + "fra1", + "tor1", + "sfo2", + "blr1", + "sfo3" + ], + "created_at": "2019-10-22T01:38:19Z", + "min_disk_size": 20, + "type": "base", + "size_gigabytes": 2.36, + "description": "Ubuntu 18.04 x64 20191022", + "tags": [], + "status": "available" + }, + "volume_ids": [], + "size": { + "slug": "s-1vcpu-1gb", + "memory": 1024, + "vcpus": 1, + "disk": 25, + "transfer": 1, + "price_monthly": 5, + "price_hourly": 0.00744, + "regions": [ + "ams2", + "ams3", + "blr1", + "fra1", + "lon1", + "nyc1", + "nyc2", + "nyc3", + "sfo1", + "sfo2", + "sfo3", + "sgp1", + "tor1" + ], + "available": true + }, + "size_slug": "s-1vcpu-1gb", + "networks": { + "v4": [ + { + "ip_address": "138.65.56.69", + "netmask": "255.255.240.0", + "gateway": "138.65.64.1", + "type": "public" + }, + { + "ip_address": "154.245.26.111", + "netmask": "255.255.252.0", + "gateway": "154.245.24.1", + "type": "public" + }, + { + "ip_address": "10.135.64.212", + "netmask": "255.255.0.0", + "gateway": "10.135.0.1", + "type": "private" + } + ], + "v6": [ + { + "ip_address": "2a03:b0c0:3:f0::cf2:4", + "netmask": 64, + "gateway": "2a03:b0c0:3:f0::1", + "type": "public" + } + ] + }, + "region": { + "name": "Frankfurt 1", + "slug": "fra1", + "features": [ + "private_networking", + "backups", + "ipv6", + "metadata", + "install_agent", + "storage", + "image_transfer" + ], + "available": true, + "sizes": [ + "s-1vcpu-1gb", + "512mb", + "s-1vcpu-2gb", + "1gb", + "s-3vcpu-1gb", + "s-2vcpu-2gb", + "s-1vcpu-3gb", + "s-2vcpu-4gb", + "2gb", + "s-4vcpu-8gb", + "m-1vcpu-8gb", + "c-2", + "4gb", + "g-2vcpu-8gb", + "gd-2vcpu-8gb", + "m-16gb", + "s-6vcpu-16gb", + "c-4", + "8gb", + "m-2vcpu-16gb", + "m3-2vcpu-16gb", + "g-4vcpu-16gb", + "gd-4vcpu-16gb", + "m6-2vcpu-16gb", + "m-32gb", + "s-8vcpu-32gb", + "c-8", + "16gb", + "m-4vcpu-32gb", + "m3-4vcpu-32gb", + "g-8vcpu-32gb", + "s-12vcpu-48gb", + "gd-8vcpu-32gb", + "m6-4vcpu-32gb", + "m-64gb", + "s-16vcpu-64gb", + "c-16", + "32gb", + "m-8vcpu-64gb", + "m3-8vcpu-64gb", + "g-16vcpu-64gb", + "s-20vcpu-96gb", + "48gb", + "gd-16vcpu-64gb", + "m6-8vcpu-64gb", + "m-128gb", + "s-24vcpu-128gb", + "c-32", + "64gb", + "m-16vcpu-128gb", + "m3-16vcpu-128gb", + "s-32vcpu-192gb", + "m-24vcpu-192gb", + "m-224gb", + "m6-16vcpu-128gb", + "m3-24vcpu-192gb", + "m6-24vcpu-192gb" + ] + }, + "tags": [] + } + ], + "links": { + "pages": { + "first": "https://api.digitalocean.com/v2/droplets?page=1&per_page=2", + "prev": "https://api.digitalocean.com/v2/droplets?page=1&per_page=2" + } + }, + "meta": { + "total": 4 + } +} +`, + }[page-1], + ) + }) +} diff --git a/discovery/manager.go b/discovery/manager.go index 66c0057a35..32ef5348dd 100644 --- a/discovery/manager.go +++ b/discovery/manager.go @@ -29,6 +29,7 @@ import ( "github.com/prometheus/prometheus/discovery/azure" "github.com/prometheus/prometheus/discovery/consul" + "github.com/prometheus/prometheus/discovery/digitalocean" "github.com/prometheus/prometheus/discovery/dns" "github.com/prometheus/prometheus/discovery/ec2" "github.com/prometheus/prometheus/discovery/file" @@ -369,6 +370,11 @@ func (m *Manager) registerProviders(cfg sd_config.ServiceDiscoveryConfig, setNam return consul.NewDiscovery(c, log.With(m.logger, "discovery", "consul")) }) } + for _, c := range cfg.DigitalOceanSDConfigs { + add(c, func() (Discoverer, error) { + return digitalocean.NewDiscovery(c, log.With(m.logger, "discovery", "digitalocean")) + }) + } for _, c := range cfg.MarathonSDConfigs { add(c, func() (Discoverer, error) { return marathon.NewDiscovery(*c, log.With(m.logger, "discovery", "marathon")) diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 904a2b136f..4cc3dd5d63 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -190,6 +190,10 @@ azure_sd_configs: consul_sd_configs: [ - ... ] +# List of DigitalOcean service discovery configurations. +digitalocean_sd_configs: + [ - ... ] + # List of DNS service discovery configurations. dns_sd_configs: [ - ... ] @@ -386,6 +390,60 @@ users with thousands of services it can be more efficient to use the Consul API directly which has basic support for filtering nodes (currently by node metadata and a single tag). +### `` + +DigitalOcean SD configurations allow retrieving scrape targets from [DigitalOcean's](https://www.digitalocean.com/) +Droplets API. +This service discovery uses the public IPv4 address by default, by that can be +changed with relabelling, as demonstrated in [the Prometheus digitalocean-sd +configuration file](/documentation/examples/prometheus-digitalocean.yml). + +The following meta labels are available on targets during [relabeling](#relabel_config): + +* `__meta_digitalocean_droplet_id`: the id of the droplet +* `__meta_digitalocean_droplet_name`: the name of the droplet +* `__meta_digitalocean_image`: the image name of the droplet +* `__meta_digitalocean_private_ipv4`: the private IPv4 of the droplet +* `__meta_digitalocean_public_ipv4`: the public IPv4 of the droplet +* `__meta_digitalocean_public_ipv6`: the public IPv6 of the droplet +* `__meta_digitalocean_region`: the region of the droplet +* `__meta_digitalocean_size`: the size of the droplet +* `__meta_digitalocean_status`: the status of the droplet +* `__meta_digitalocean_features`: the comma-separated list of features of the droplet +* `__meta_digitalocean_tags`: the comma-separated list of tags of the droplet + +```yaml +# Authentication information used to authenticate to the API server. +# Note that `basic_auth`, `bearer_token` and `bearer_token_file` options are +# mutually exclusive. +# password and password_file are mutually exclusive. + +# Optional HTTP basic authentication information, not currently supported by DigitalOcean. +basic_auth: + [ username: ] + [ password: ] + [ password_file: ] + +# Optional bearer token authentication information. +[ bearer_token: ] + +# Optional bearer token file authentication information. +[ bearer_token_file: ] + +# Optional proxy URL. +[ proxy_url: ] + +# TLS configuration. +tls_config: + [ ] + +# The port to scrape metrics from. +[ port: | default = 80 ] + +# The time after which the droplets are refreshed. +[ refresh_interval: | default = 60s ] +``` + ### `` A DNS-based service discovery configuration allows specifying a set of DNS diff --git a/documentation/examples/prometheus-digitalocean.yml b/documentation/examples/prometheus-digitalocean.yml new file mode 100644 index 0000000000..bb8ee1744c --- /dev/null +++ b/documentation/examples/prometheus-digitalocean.yml @@ -0,0 +1,25 @@ +# A example scrape configuration for running Prometheus with +# DigitalOcean. + +scrape_configs: + + # Make Prometheus scrape itself for metrics. + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + + # Discover Node Exporter instances to scrape. + - job_name: 'node' + + digitalocean_sd_configs: + - bearer_token: "" + relabel_configs: + # Only scrape targets that have a tag 'monitoring'. + - source_labels: [__meta_digitalocean_tags] + regex: '.*,monitoring,.*' + action: keep + + # Use the public IPv6 address and port 9100 to scrape the target. + - source_labels: [__meta_digitalocean_public_ipv6] + target_label: __address__ + replacement: '[$1]:9100' diff --git a/go.mod b/go.mod index cdc21de8f3..57eaf512d4 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/cespare/xxhash v1.1.0 github.com/davecgh/go-spew v1.1.1 github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b + github.com/digitalocean/godo v1.37.0 github.com/edsrzf/mmap-go v1.0.0 github.com/go-kit/kit v0.10.0 github.com/go-logfmt/logfmt v0.5.0 diff --git a/go.sum b/go.sum index ec22f4580f..0e921a4a8d 100644 --- a/go.sum +++ b/go.sum @@ -150,6 +150,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b h1:Yqiad0+sloMPdd/0Fg22actpFx0dekpzt1xJmVNVkU0= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/digitalocean/godo v1.37.0 h1:NEj5ne2cvLBHo1GJY1DNN/iEt9ipa72CMwwAjKEA530= +github.com/digitalocean/godo v1.37.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -362,6 +364,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= diff --git a/vendor/github.com/digitalocean/godo/.gitignore b/vendor/github.com/digitalocean/godo/.gitignore new file mode 100644 index 0000000000..48b8bf9072 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/.gitignore @@ -0,0 +1 @@ +vendor/ diff --git a/vendor/github.com/digitalocean/godo/.whitesource b/vendor/github.com/digitalocean/godo/.whitesource new file mode 100644 index 0000000000..e0aaa3e9eb --- /dev/null +++ b/vendor/github.com/digitalocean/godo/.whitesource @@ -0,0 +1,8 @@ +{ + "checkRunSettings": { + "vulnerableCheckRunConclusionLevel": "failure" + }, + "issueSettings": { + "minSeverityLevel": "LOW" + } +} \ No newline at end of file diff --git a/vendor/github.com/digitalocean/godo/1-click.go b/vendor/github.com/digitalocean/godo/1-click.go new file mode 100644 index 0000000000..fab04fe510 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/1-click.go @@ -0,0 +1,52 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const oneClickBasePath = "v2/1-clicks" + +// OneClickService is an interface for interacting with 1-clicks with the +// DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2/#1-click-applications +type OneClickService interface { + List(context.Context, string) ([]*OneClick, *Response, error) +} + +var _ OneClickService = &OneClickServiceOp{} + +// OneClickServiceOp interfaces with 1-click endpoints in the DigitalOcean API. +type OneClickServiceOp struct { + client *Client +} + +// OneClick is the structure of a 1-click +type OneClick struct { + Slug string `json:"slug"` + Type string `json:"type"` +} + +// OneClicksRoot is the root of the json payload that contains a list of 1-clicks +type OneClicksRoot struct { + List []*OneClick `json:"1_clicks"` +} + +// List returns a list of the available 1-click applications. +func (ocs *OneClickServiceOp) List(ctx context.Context, oneClickType string) ([]*OneClick, *Response, error) { + path := fmt.Sprintf(`%s?type=%s`, oneClickBasePath, oneClickType) + + req, err := ocs.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(OneClicksRoot) + resp, err := ocs.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.List, resp, nil +} diff --git a/vendor/github.com/digitalocean/godo/CHANGELOG.md b/vendor/github.com/digitalocean/godo/CHANGELOG.md new file mode 100644 index 0000000000..c826205fac --- /dev/null +++ b/vendor/github.com/digitalocean/godo/CHANGELOG.md @@ -0,0 +1,255 @@ +# Change Log + +## [v1.37.0] - 2020-06-01 + +- #336 registry: URL encode repository names when building URLs. @adamwg +- #335 Add 1-click service and request. @scottcrawford03 + +## [v1.36.0] - 2020-05-12 + +- #331 Expose expiry_seconds for Registry.DockerCredentials. @andrewsomething + +## [v1.35.1] - 2020-04-21 + +- #328 Update vulnerable x/crypto dependency - @bentranter + +## [v1.35.0] - 2020-04-20 + +- #326 Add TagCount field to registry/Repository - @nicktate +- #325 Add DOCR EA routes - @nicktate +- #324 Upgrade godo to Go 1.14 - @bentranter + +## [v1.34.0] - 2020-03-30 + +- #320 Add VPC v3 attributes - @viola + +## [v1.33.1] - 2020-03-23 + +- #318 upgrade github.com/stretchr/objx past 0.1.1 - @hilary + +## [v1.33.0] - 2020-03-20 + +- #310 Add BillingHistory service and List endpoint - @rbutler +- #316 load balancers: add new enable_backend_keepalive field - @anitgandhi + +## [v1.32.0] - 2020-03-04 + +- #311 Add reset database user auth method - @zbarahal-do + +## [v1.31.0] - 2020-02-28 + +- #305 invoices: GetPDF and GetCSV methods - @rbutler +- #304 Add NewFromToken convenience method to init client - @bentranter +- #301 invoices: Get, Summary, and List methods - @rbutler +- #299 Fix param expiry_seconds for kubernetes.GetCredentials request - @velp + +## [v1.30.0] - 2020-02-03 + +- #295 registry: support the created_at field - @adamwg +- #293 doks: node pool labels - @snormore + +## [v1.29.0] - 2019-12-13 + +- #288 Add Balance Get method - @rbutler +- #286,#289 Deserialize meta field - @timoreimann + +## [v1.28.0] - 2019-12-04 + +- #282 Add valid Redis eviction policy constants - @bentranter +- #281 Remove databases info from top-level godoc string - @bentranter +- #280 Fix VolumeSnapshotResourceType value volumesnapshot -> volume_snapshot - @aqche + +## [v1.27.0] - 2019-11-18 + +- #278 add mysql user auth settings for database users - @gregmankes + +## [v1.26.0] - 2019-11-13 + +- #272 dbaas: get and set mysql sql mode - @mikejholly + +## [v1.25.0] - 2019-11-13 + +- #275 registry/docker-credentials: add support for the read/write parameter - @kamaln7 +- #273 implement the registry/docker-credentials endpoint - @kamaln7 +- #271 Add registry resource - @snormore + +## [v1.24.1] - 2019-11-04 + +- #264 Update isLast to check p.Next - @aqche + +## [v1.24.0] - 2019-10-30 + +- #267 Return []DatabaseFirewallRule in addition to raw response. - @andrewsomething + +## [v1.23.1] - 2019-10-30 + +- #265 add support for getting/setting firewall rules - @gregmankes +- #262 remove ResolveReference call - @mdanzinger +- #261 Update CONTRIBUTING.md - @mdanzinger + +## [v1.22.0] - 2019-09-24 + +- #259 Add Kubernetes GetCredentials method - @snormore + +## [v1.21.1] - 2019-09-19 + +- #257 Upgrade to Go 1.13 - @bentranter + +## [v1.21.0] - 2019-09-16 + +- #255 Add DropletID to Kubernetes Node instance - @snormore +- #254 Add tags to Database, DatabaseReplica - @Zyqsempai + +## [v1.20.0] - 2019-09-06 + +- #252 Add Kubernetes autoscale config fields - @snormore +- #251 Support unset fields on Kubernetes cluster and node pool updates - @snormore +- #250 Add Kubernetes GetUser method - @snormore + +## [v1.19.0] - 2019-07-19 + +- #244 dbaas: add private-network-uuid field to create request + +## [v1.18.0] - 2019-07-17 + +- #241 Databases: support for custom VPC UUID on migrate @mikejholly +- #240 Add the ability to get URN for a Database @stack72 +- #236 Fix omitempty typos in JSON struct tags @amccarthy1 + +## [v1.17.0] - 2019-06-21 + +- #238 Add support for Redis eviction policy in Databases @mikejholly + +## [v1.16.0] - 2019-06-04 + +- #233 Add Kubernetes DeleteNode method, deprecate RecycleNodePoolNodes @bouk + +## [v1.15.0] - 2019-05-13 + +- #231 Add private connection fields to Databases - @mikejholly +- #223 Introduce Go modules - @andreiavrammsd + +## [v1.14.0] - 2019-05-13 + +- #229 Add support for upgrading Kubernetes clusters - @adamwg + +## [v1.13.0] - 2019-04-19 + +- #213 Add tagging support for volume snapshots - @jcodybaker + +## [v1.12.0] - 2019-04-18 + +- #224 Add maintenance window support for Kubernetes- @fatih + +## [v1.11.1] - 2019-04-04 + +- #222 Fix Create Database Pools json fields - @sunny-b + +## [v1.11.0] - 2019-04-03 + +- #220 roll out vpc functionality - @jheimann + +## [v1.10.1] - 2019-03-27 + +- #219 Fix Database Pools json field - @sunny-b + +## [v1.10.0] - 2019-03-20 + +- #215 Add support for Databases - @mikejholly + +## [v1.9.0] - 2019-03-18 + +- #214 add support for enable_proxy_protocol. - @mregmi + +## [v1.8.0] - 2019-03-13 + +- #210 Expose tags on storage volume create/list/get. - @jcodybaker + +## [v1.7.5] - 2019-03-04 + +- #207 Add support for custom subdomains for Spaces CDN [beta] - @xornivore + +## [v1.7.4] - 2019-02-08 + +- #202 Allow tagging volumes - @mchitten + +## [v1.7.3] - 2018-12-18 + +- #196 Expose tag support for creating Load Balancers. + +## [v1.7.2] - 2018-12-04 + +- #192 Exposes more options for Kubernetes clusters. + +## [v1.7.1] - 2018-11-27 + +- #190 Expose constants for the state of Kubernetes clusters. + +## [v1.7.0] - 2018-11-13 + +- #188 Kubernetes support [beta] - @aybabtme + +## [v1.6.0] - 2018-10-16 + +- #185 Projects support [beta] - @mchitten + +## [v1.5.0] - 2018-10-01 + +- #181 Adding tagging images support - @hugocorbucci + +## [v1.4.2] - 2018-08-30 + +- #178 Allowing creating domain records with weight of 0 - @TFaga +- #177 Adding `VolumeLimit` to account - @lxfontes + +## [v1.4.1] - 2018-08-23 + +- #176 Fix cdn flush cache API endpoint - @sunny-b + +## [v1.4.0] - 2018-08-22 + +- #175 Add support for Spaces CDN - @sunny-b + +## [v1.3.0] - 2018-05-24 + +- #170 Add support for volume formatting - @adamwg + +## [v1.2.0] - 2018-05-08 + +- #166 Remove support for Go 1.6 - @iheanyi +- #165 Add support for Let's Encrypt Certificates - @viola + +## [v1.1.3] - 2018-03-07 + +- #156 Handle non-json errors from the API - @aknuds1 +- #158 Update droplet example to use latest instance type - @dan-v + +## [v1.1.2] - 2018-03-06 + +- #157 storage: list volumes should handle only name or only region params - @andrewsykim +- #154 docs: replace first example with fully-runnable example - @xmudrii +- #152 Handle flags & tag properties of domain record - @jaymecd + +## [v1.1.1] - 2017-09-29 + +- #151 Following user agent field recommendations - @joonas +- #148 AsRequest method to create load balancers requests - @lukegb + +## [v1.1.0] - 2017-06-06 + +### Added +- #145 Add FirewallsService for managing Firewalls with the DigitalOcean API. - @viola +- #139 Add TTL field to the Domains. - @xmudrii + +### Fixed +- #143 Fix oauth2.NoContext depreciation. - @jbowens +- #141 Fix DropletActions on tagged resources. - @xmudrii + +## [v1.0.0] - 2017-03-10 + +### Added +- #130 Add Convert to ImageActionsService. - @xmudrii +- #126 Add CertificatesService for managing certificates with the DigitalOcean API. - @viola +- #125 Add LoadBalancersService for managing load balancers with the DigitalOcean API. - @viola +- #122 Add GetVolumeByName to StorageService. - @protochron +- #113 Add context.Context to all calls. - @aybabtme diff --git a/vendor/github.com/digitalocean/godo/CONTRIBUTING.md b/vendor/github.com/digitalocean/godo/CONTRIBUTING.md new file mode 100644 index 0000000000..33f03132a8 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/CONTRIBUTING.md @@ -0,0 +1,54 @@ +# Contributing + +We love contributions! You are welcome to open a pull request, but it's a good idea to +open an issue and discuss your idea with us first. + +Once you are ready to open a PR, please keep the following guidelines in mind: + +1. Code should be `go fmt` compliant. +1. Types, structs and funcs should be documented. +1. Tests pass. + +## Getting set up + +`godo` uses go modules. Just fork this repo, clone your fork and off you go! + +## Running tests + +When working on code in this repository, tests can be run via: + +```sh +go test -mod=vendor . +``` + +## Versioning + +Godo follows [semver](https://www.semver.org) versioning semantics. +New functionality should be accompanied by increment to the minor +version number. Any code merged to master is subject to release. + +## Releasing + +Releasing a new version of godo is currently a manual process. + +Submit a separate pull request for the version change from the pull +request with your changes. + +1. Update the `CHANGELOG.md` with your changes. If a version header + for the next (unreleased) version does not exist, create one. + Include one bullet point for each piece of new functionality in the + release, including the pull request ID, description, and author(s). + +``` +## [v1.8.0] - 2019-03-13 + +- #210 Expose tags on storage volume create/list/get. - @jcodybaker +- #123 Update test dependencies - @digitalocean +``` + +2. Update the `libraryVersion` number in `godo.go`. +3. Make a pull request with these changes. This PR should be separate from the PR containing the godo changes. +4. Once the pull request has been merged, [draft a new release](https://github.com/digitalocean/godo/releases/new). +5. Update the `Tag version` and `Release title` field with the new godo version. Be sure the version has a `v` prefixed in both places. Ex `v1.8.0`. +6. Copy the changelog bullet points to the description field. +7. Publish the release. diff --git a/vendor/github.com/digitalocean/godo/LICENSE.txt b/vendor/github.com/digitalocean/godo/LICENSE.txt new file mode 100644 index 0000000000..43c5d2eee7 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/LICENSE.txt @@ -0,0 +1,55 @@ +Copyright (c) 2014-2016 The godo AUTHORS. All rights reserved. + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +====================== +Portions of the client are based on code at: +https://github.com/google/go-github/ + +Copyright (c) 2013 The go-github AUTHORS. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + diff --git a/vendor/github.com/digitalocean/godo/README.md b/vendor/github.com/digitalocean/godo/README.md new file mode 100644 index 0000000000..cadeb69edb --- /dev/null +++ b/vendor/github.com/digitalocean/godo/README.md @@ -0,0 +1,139 @@ +# Godo + +[![Build Status](https://travis-ci.org/digitalocean/godo.svg)](https://travis-ci.org/digitalocean/godo) +[![GoDoc](https://godoc.org/github.com/digitalocean/godo?status.svg)](https://godoc.org/github.com/digitalocean/godo) + +Godo is a Go client library for accessing the DigitalOcean V2 API. + +You can view the client API docs here: [http://godoc.org/github.com/digitalocean/godo](http://godoc.org/github.com/digitalocean/godo) + +You can view DigitalOcean API docs here: [https://developers.digitalocean.com/documentation/v2/](https://developers.digitalocean.com/documentation/v2/) + +## Install +```sh +go get github.com/digitalocean/godo@vX.Y.Z +``` + +where X.Y.Z is the [version](https://github.com/digitalocean/godo/releases) you need. + +or +```sh +go get github.com/digitalocean/godo +``` +for non Go modules usage or latest version. + +## Usage + +```go +import "github.com/digitalocean/godo" +``` + +Create a new DigitalOcean client, then use the exposed services to +access different parts of the DigitalOcean API. + +### Authentication + +Currently, Personal Access Token (PAT) is the only method of +authenticating with the API. You can manage your tokens +at the DigitalOcean Control Panel [Applications Page](https://cloud.digitalocean.com/settings/applications). + +You can then use your token to create a new client: + +```go +package main + +import ( + "github.com/digitalocean/godo" +) + +func main() { + client := godo.NewFromToken("my-digitalocean-api-token") +} +``` + +If you need to provide a `context.Context` to your new client, you should use [`godo.NewClient`](https://godoc.org/github.com/digitalocean/godo#NewClient) to manually construct a client instead. + +## Examples + + +To create a new Droplet: + +```go +dropletName := "super-cool-droplet" + +createRequest := &godo.DropletCreateRequest{ + Name: dropletName, + Region: "nyc3", + Size: "s-1vcpu-1gb", + Image: godo.DropletCreateImage{ + Slug: "ubuntu-14-04-x64", + }, +} + +ctx := context.TODO() + +newDroplet, _, err := client.Droplets.Create(ctx, createRequest) + +if err != nil { + fmt.Printf("Something bad happened: %s\n\n", err) + return err +} +``` + +### Pagination + +If a list of items is paginated by the API, you must request pages individually. For example, to fetch all Droplets: + +```go +func DropletList(ctx context.Context, client *godo.Client) ([]godo.Droplet, error) { + // create a list to hold our droplets + list := []godo.Droplet{} + + // create options. initially, these will be blank + opt := &godo.ListOptions{} + for { + droplets, resp, err := client.Droplets.List(ctx, opt) + if err != nil { + return nil, err + } + + // append the current page's droplets to our list + for _, d := range droplets { + list = append(list, d) + } + + // if we are at the last page, break out the for loop + if resp.Links == nil || resp.Links.IsLastPage() { + break + } + + page, err := resp.Links.CurrentPage() + if err != nil { + return nil, err + } + + // set the page we want for the next request + opt.Page = page + 1 + } + + return list, nil +} +``` + +## Versioning + +Each version of the client is tagged and the version is updated accordingly. + +To see the list of past versions, run `git tag`. + + +## Documentation + +For a comprehensive list of examples, check out the [API documentation](https://developers.digitalocean.com/documentation/v2/). + +For details on all the functionality in this library, see the [GoDoc](http://godoc.org/github.com/digitalocean/godo) documentation. + + +## Contributing + +We love pull requests! Please see the [contribution guidelines](CONTRIBUTING.md). diff --git a/vendor/github.com/digitalocean/godo/account.go b/vendor/github.com/digitalocean/godo/account.go new file mode 100644 index 0000000000..7d3e105d34 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/account.go @@ -0,0 +1,60 @@ +package godo + +import ( + "context" + "net/http" +) + +// AccountService is an interface for interfacing with the Account +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2/#account +type AccountService interface { + Get(context.Context) (*Account, *Response, error) +} + +// AccountServiceOp handles communication with the Account related methods of +// the DigitalOcean API. +type AccountServiceOp struct { + client *Client +} + +var _ AccountService = &AccountServiceOp{} + +// Account represents a DigitalOcean Account +type Account struct { + DropletLimit int `json:"droplet_limit,omitempty"` + FloatingIPLimit int `json:"floating_ip_limit,omitempty"` + VolumeLimit int `json:"volume_limit,omitempty"` + Email string `json:"email,omitempty"` + UUID string `json:"uuid,omitempty"` + EmailVerified bool `json:"email_verified,omitempty"` + Status string `json:"status,omitempty"` + StatusMessage string `json:"status_message,omitempty"` +} + +type accountRoot struct { + Account *Account `json:"account"` +} + +func (r Account) String() string { + return Stringify(r) +} + +// Get DigitalOcean account info +func (s *AccountServiceOp) Get(ctx context.Context) (*Account, *Response, error) { + + path := "v2/account" + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(accountRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Account, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/action.go b/vendor/github.com/digitalocean/godo/action.go new file mode 100644 index 0000000000..e3176005ba --- /dev/null +++ b/vendor/github.com/digitalocean/godo/action.go @@ -0,0 +1,108 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const ( + actionsBasePath = "v2/actions" + + // ActionInProgress is an in progress action status + ActionInProgress = "in-progress" + + //ActionCompleted is a completed action status + ActionCompleted = "completed" +) + +// ActionsService handles communction with action related methods of the +// DigitalOcean API: https://developers.digitalocean.com/documentation/v2#actions +type ActionsService interface { + List(context.Context, *ListOptions) ([]Action, *Response, error) + Get(context.Context, int) (*Action, *Response, error) +} + +// ActionsServiceOp handles communition with the image action related methods of the +// DigitalOcean API. +type ActionsServiceOp struct { + client *Client +} + +var _ ActionsService = &ActionsServiceOp{} + +type actionsRoot struct { + Actions []Action `json:"actions"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type actionRoot struct { + Event *Action `json:"action"` +} + +// Action represents a DigitalOcean Action +type Action struct { + ID int `json:"id"` + Status string `json:"status"` + Type string `json:"type"` + StartedAt *Timestamp `json:"started_at"` + CompletedAt *Timestamp `json:"completed_at"` + ResourceID int `json:"resource_id"` + ResourceType string `json:"resource_type"` + Region *Region `json:"region,omitempty"` + RegionSlug string `json:"region_slug,omitempty"` +} + +// List all actions +func (s *ActionsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Action, *Response, error) { + path := actionsBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Actions, resp, err +} + +// Get an action by ID. +func (s *ActionsServiceOp) Get(ctx context.Context, id int) (*Action, *Response, error) { + if id < 1 { + return nil, nil, NewArgError("id", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", actionsBasePath, id) + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (a Action) String() string { + return Stringify(a) +} diff --git a/vendor/github.com/digitalocean/godo/balance.go b/vendor/github.com/digitalocean/godo/balance.go new file mode 100644 index 0000000000..4da6978361 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/balance.go @@ -0,0 +1,52 @@ +package godo + +import ( + "context" + "net/http" + "time" +) + +// BalanceService is an interface for interfacing with the Balance +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2/#balance +type BalanceService interface { + Get(context.Context) (*Balance, *Response, error) +} + +// BalanceServiceOp handles communication with the Balance related methods of +// the DigitalOcean API. +type BalanceServiceOp struct { + client *Client +} + +var _ BalanceService = &BalanceServiceOp{} + +// Balance represents a DigitalOcean Balance +type Balance struct { + MonthToDateBalance string `json:"month_to_date_balance"` + AccountBalance string `json:"account_balance"` + MonthToDateUsage string `json:"month_to_date_usage"` + GeneratedAt time.Time `json:"generated_at"` +} + +func (r Balance) String() string { + return Stringify(r) +} + +// Get DigitalOcean balance info +func (s *BalanceServiceOp) Get(ctx context.Context) (*Balance, *Response, error) { + path := "v2/customers/my/balance" + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(Balance) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/billing_history.go b/vendor/github.com/digitalocean/godo/billing_history.go new file mode 100644 index 0000000000..a510100065 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/billing_history.go @@ -0,0 +1,72 @@ +package godo + +import ( + "context" + "net/http" + "time" +) + +const billingHistoryBasePath = "v2/customers/my/billing_history" + +// BillingHistoryService is an interface for interfacing with the BillingHistory +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2/#billing_history +type BillingHistoryService interface { + List(context.Context, *ListOptions) (*BillingHistory, *Response, error) +} + +// BillingHistoryServiceOp handles communication with the BillingHistory related methods of +// the DigitalOcean API. +type BillingHistoryServiceOp struct { + client *Client +} + +var _ BillingHistoryService = &BillingHistoryServiceOp{} + +// BillingHistory represents a DigitalOcean Billing History +type BillingHistory struct { + BillingHistory []BillingHistoryEntry `json:"billing_history"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// BillingHistoryEntry represents an entry in a customer's Billing History +type BillingHistoryEntry struct { + Description string `json:"description"` + Amount string `json:"amount"` + InvoiceID *string `json:"invoice_id"` + InvoiceUUID *string `json:"invoice_uuid"` + Date time.Time `json:"date"` + Type string `json:"type"` +} + +func (b BillingHistory) String() string { + return Stringify(b) +} + +// List the Billing History for a customer +func (s *BillingHistoryServiceOp) List(ctx context.Context, opt *ListOptions) (*BillingHistory, *Response, error) { + path, err := addOptions(billingHistoryBasePath, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(BillingHistory) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/cdn.go b/vendor/github.com/digitalocean/godo/cdn.go new file mode 100644 index 0000000000..4c97d1174d --- /dev/null +++ b/vendor/github.com/digitalocean/godo/cdn.go @@ -0,0 +1,218 @@ +package godo + +import ( + "context" + "fmt" + "net/http" + "time" +) + +const cdnBasePath = "v2/cdn/endpoints" + +// CDNService is an interface for managing Spaces CDN with the DigitalOcean API. +type CDNService interface { + List(context.Context, *ListOptions) ([]CDN, *Response, error) + Get(context.Context, string) (*CDN, *Response, error) + Create(context.Context, *CDNCreateRequest) (*CDN, *Response, error) + UpdateTTL(context.Context, string, *CDNUpdateTTLRequest) (*CDN, *Response, error) + UpdateCustomDomain(context.Context, string, *CDNUpdateCustomDomainRequest) (*CDN, *Response, error) + FlushCache(context.Context, string, *CDNFlushCacheRequest) (*Response, error) + Delete(context.Context, string) (*Response, error) +} + +// CDNServiceOp handles communication with the CDN related methods of the +// DigitalOcean API. +type CDNServiceOp struct { + client *Client +} + +var _ CDNService = &CDNServiceOp{} + +// CDN represents a DigitalOcean CDN +type CDN struct { + ID string `json:"id"` + Origin string `json:"origin"` + Endpoint string `json:"endpoint"` + CreatedAt time.Time `json:"created_at"` + TTL uint32 `json:"ttl"` + CertificateID string `json:"certificate_id,omitempty"` + CustomDomain string `json:"custom_domain,omitempty"` +} + +// CDNRoot represents a response from the DigitalOcean API +type cdnRoot struct { + Endpoint *CDN `json:"endpoint"` +} + +type cdnsRoot struct { + Endpoints []CDN `json:"endpoints"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// CDNCreateRequest represents a request to create a CDN. +type CDNCreateRequest struct { + Origin string `json:"origin"` + TTL uint32 `json:"ttl"` + CustomDomain string `json:"custom_domain,omitempty"` + CertificateID string `json:"certificate_id,omitempty"` +} + +// CDNUpdateTTLRequest represents a request to update the ttl of a CDN. +type CDNUpdateTTLRequest struct { + TTL uint32 `json:"ttl"` +} + +// CDNUpdateCustomDomainRequest represents a request to update the custom domain of a CDN. +type CDNUpdateCustomDomainRequest struct { + CustomDomain string `json:"custom_domain"` + CertificateID string `json:"certificate_id"` +} + +// CDNFlushCacheRequest represents a request to flush cache of a CDN. +type CDNFlushCacheRequest struct { + Files []string `json:"files"` +} + +// List all CDN endpoints +func (c CDNServiceOp) List(ctx context.Context, opt *ListOptions) ([]CDN, *Response, error) { + path, err := addOptions(cdnBasePath, opt) + if err != nil { + return nil, nil, err + } + + req, err := c.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(cdnsRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Endpoints, resp, err +} + +// Get individual CDN. It requires a non-empty cdn id. +func (c CDNServiceOp) Get(ctx context.Context, id string) (*CDN, *Response, error) { + if len(id) == 0 { + return nil, nil, NewArgError("id", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s", cdnBasePath, id) + + req, err := c.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(cdnRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Endpoint, resp, err +} + +// Create a new CDN +func (c CDNServiceOp) Create(ctx context.Context, createRequest *CDNCreateRequest) (*CDN, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + req, err := c.client.NewRequest(ctx, http.MethodPost, cdnBasePath, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(cdnRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Endpoint, resp, err +} + +// UpdateTTL updates the ttl of an individual CDN +func (c CDNServiceOp) UpdateTTL(ctx context.Context, id string, updateRequest *CDNUpdateTTLRequest) (*CDN, *Response, error) { + return c.update(ctx, id, updateRequest) +} + +// UpdateCustomDomain sets or removes the custom domain of an individual CDN +func (c CDNServiceOp) UpdateCustomDomain(ctx context.Context, id string, updateRequest *CDNUpdateCustomDomainRequest) (*CDN, *Response, error) { + return c.update(ctx, id, updateRequest) +} + +func (c CDNServiceOp) update(ctx context.Context, id string, updateRequest interface{}) (*CDN, *Response, error) { + if updateRequest == nil { + return nil, nil, NewArgError("updateRequest", "cannot be nil") + } + + if len(id) == 0 { + return nil, nil, NewArgError("id", "cannot be an empty string") + } + path := fmt.Sprintf("%s/%s", cdnBasePath, id) + + req, err := c.client.NewRequest(ctx, http.MethodPut, path, updateRequest) + if err != nil { + return nil, nil, err + } + + root := new(cdnRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Endpoint, resp, err +} + +// FlushCache flushes the cache of an individual CDN. Requires a non-empty slice of file paths and/or wildcards +func (c CDNServiceOp) FlushCache(ctx context.Context, id string, flushCacheRequest *CDNFlushCacheRequest) (*Response, error) { + if flushCacheRequest == nil { + return nil, NewArgError("flushCacheRequest", "cannot be nil") + } + + if len(id) == 0 { + return nil, NewArgError("id", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s/cache", cdnBasePath, id) + + req, err := c.client.NewRequest(ctx, http.MethodDelete, path, flushCacheRequest) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(ctx, req, nil) + + return resp, err +} + +// Delete an individual CDN +func (c CDNServiceOp) Delete(ctx context.Context, id string) (*Response, error) { + if len(id) == 0 { + return nil, NewArgError("id", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s", cdnBasePath, id) + + req, err := c.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := c.client.Do(ctx, req, nil) + + return resp, err +} diff --git a/vendor/github.com/digitalocean/godo/certificates.go b/vendor/github.com/digitalocean/godo/certificates.go new file mode 100644 index 0000000000..9a6bdb2daa --- /dev/null +++ b/vendor/github.com/digitalocean/godo/certificates.go @@ -0,0 +1,130 @@ +package godo + +import ( + "context" + "net/http" + "path" +) + +const certificatesBasePath = "/v2/certificates" + +// CertificatesService is an interface for managing certificates with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2/#certificates +type CertificatesService interface { + Get(context.Context, string) (*Certificate, *Response, error) + List(context.Context, *ListOptions) ([]Certificate, *Response, error) + Create(context.Context, *CertificateRequest) (*Certificate, *Response, error) + Delete(context.Context, string) (*Response, error) +} + +// Certificate represents a DigitalOcean certificate configuration. +type Certificate struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + DNSNames []string `json:"dns_names,omitempty"` + NotAfter string `json:"not_after,omitempty"` + SHA1Fingerprint string `json:"sha1_fingerprint,omitempty"` + Created string `json:"created_at,omitempty"` + State string `json:"state,omitempty"` + Type string `json:"type,omitempty"` +} + +// CertificateRequest represents configuration for a new certificate. +type CertificateRequest struct { + Name string `json:"name,omitempty"` + DNSNames []string `json:"dns_names,omitempty"` + PrivateKey string `json:"private_key,omitempty"` + LeafCertificate string `json:"leaf_certificate,omitempty"` + CertificateChain string `json:"certificate_chain,omitempty"` + Type string `json:"type,omitempty"` +} + +type certificateRoot struct { + Certificate *Certificate `json:"certificate"` +} + +type certificatesRoot struct { + Certificates []Certificate `json:"certificates"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// CertificatesServiceOp handles communication with certificates methods of the DigitalOcean API. +type CertificatesServiceOp struct { + client *Client +} + +var _ CertificatesService = &CertificatesServiceOp{} + +// Get an existing certificate by its identifier. +func (c *CertificatesServiceOp) Get(ctx context.Context, cID string) (*Certificate, *Response, error) { + urlStr := path.Join(certificatesBasePath, cID) + + req, err := c.client.NewRequest(ctx, http.MethodGet, urlStr, nil) + if err != nil { + return nil, nil, err + } + + root := new(certificateRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Certificate, resp, nil +} + +// List all certificates. +func (c *CertificatesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Certificate, *Response, error) { + urlStr, err := addOptions(certificatesBasePath, opt) + if err != nil { + return nil, nil, err + } + + req, err := c.client.NewRequest(ctx, http.MethodGet, urlStr, nil) + if err != nil { + return nil, nil, err + } + + root := new(certificatesRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Certificates, resp, nil +} + +// Create a new certificate with provided configuration. +func (c *CertificatesServiceOp) Create(ctx context.Context, cr *CertificateRequest) (*Certificate, *Response, error) { + req, err := c.client.NewRequest(ctx, http.MethodPost, certificatesBasePath, cr) + if err != nil { + return nil, nil, err + } + + root := new(certificateRoot) + resp, err := c.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Certificate, resp, nil +} + +// Delete a certificate by its identifier. +func (c *CertificatesServiceOp) Delete(ctx context.Context, cID string) (*Response, error) { + urlStr := path.Join(certificatesBasePath, cID) + + req, err := c.client.NewRequest(ctx, http.MethodDelete, urlStr, nil) + if err != nil { + return nil, err + } + + return c.client.Do(ctx, req, nil) +} diff --git a/vendor/github.com/digitalocean/godo/databases.go b/vendor/github.com/digitalocean/godo/databases.go new file mode 100644 index 0000000000..42d83ee0c4 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/databases.go @@ -0,0 +1,845 @@ +package godo + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" +) + +const ( + databaseBasePath = "/v2/databases" + databaseSinglePath = databaseBasePath + "/%s" + databaseResizePath = databaseBasePath + "/%s/resize" + databaseMigratePath = databaseBasePath + "/%s/migrate" + databaseMaintenancePath = databaseBasePath + "/%s/maintenance" + databaseBackupsPath = databaseBasePath + "/%s/backups" + databaseUsersPath = databaseBasePath + "/%s/users" + databaseUserPath = databaseBasePath + "/%s/users/%s" + databaseResetUserAuthPath = databaseUserPath + "/reset_auth" + databaseDBPath = databaseBasePath + "/%s/dbs/%s" + databaseDBsPath = databaseBasePath + "/%s/dbs" + databasePoolPath = databaseBasePath + "/%s/pools/%s" + databasePoolsPath = databaseBasePath + "/%s/pools" + databaseReplicaPath = databaseBasePath + "/%s/replicas/%s" + databaseReplicasPath = databaseBasePath + "/%s/replicas" + databaseEvictionPolicyPath = databaseBasePath + "/%s/eviction_policy" + databaseSQLModePath = databaseBasePath + "/%s/sql_mode" + databaseFirewallRulesPath = databaseBasePath + "/%s/firewall" +) + +// SQL Mode constants allow for MySQL-specific SQL flavor configuration. +const ( + SQLModeAllowInvalidDates = "ALLOW_INVALID_DATES" + SQLModeANSIQuotes = "ANSI_QUOTES" + SQLModeHighNotPrecedence = "HIGH_NOT_PRECEDENCE" + SQLModeIgnoreSpace = "IGNORE_SPACE" + SQLModeNoAuthCreateUser = "NO_AUTO_CREATE_USER" + SQLModeNoAutoValueOnZero = "NO_AUTO_VALUE_ON_ZERO" + SQLModeNoBackslashEscapes = "NO_BACKSLASH_ESCAPES" + SQLModeNoDirInCreate = "NO_DIR_IN_CREATE" + SQLModeNoEngineSubstitution = "NO_ENGINE_SUBSTITUTION" + SQLModeNoFieldOptions = "NO_FIELD_OPTIONS" + SQLModeNoKeyOptions = "NO_KEY_OPTIONS" + SQLModeNoTableOptions = "NO_TABLE_OPTIONS" + SQLModeNoUnsignedSubtraction = "NO_UNSIGNED_SUBTRACTION" + SQLModeNoZeroDate = "NO_ZERO_DATE" + SQLModeNoZeroInDate = "NO_ZERO_IN_DATE" + SQLModeOnlyFullGroupBy = "ONLY_FULL_GROUP_BY" + SQLModePadCharToFullLength = "PAD_CHAR_TO_FULL_LENGTH" + SQLModePipesAsConcat = "PIPES_AS_CONCAT" + SQLModeRealAsFloat = "REAL_AS_FLOAT" + SQLModeStrictAllTables = "STRICT_ALL_TABLES" + SQLModeStrictTransTables = "STRICT_TRANS_TABLES" + SQLModeANSI = "ANSI" + SQLModeDB2 = "DB2" + SQLModeMaxDB = "MAXDB" + SQLModeMSSQL = "MSSQL" + SQLModeMYSQL323 = "MYSQL323" + SQLModeMYSQL40 = "MYSQL40" + SQLModeOracle = "ORACLE" + SQLModePostgreSQL = "POSTGRESQL" + SQLModeTraditional = "TRADITIONAL" +) + +// SQL Auth constants allow for MySQL-specific user auth plugins +const ( + SQLAuthPluginNative = "mysql_native_password" + SQLAuthPluginCachingSHA2 = "caching_sha2_password" +) + +// Redis eviction policies supported by the managed Redis product. +const ( + EvictionPolicyNoEviction = "noeviction" + EvictionPolicyAllKeysLRU = "allkeys_lru" + EvictionPolicyAllKeysRandom = "allkeys_random" + EvictionPolicyVolatileLRU = "volatile_lru" + EvictionPolicyVolatileRandom = "volatile_random" + EvictionPolicyVolatileTTL = "volatile_ttl" +) + +// The DatabasesService provides access to the DigitalOcean managed database +// suite of products through the public API. Customers can create new database +// clusters, migrate them between regions, create replicas and interact with +// their configurations. Each database service is refered to as a Database. A +// SQL database service can have multiple databases residing in the system. To +// help make these entities distinct from Databases in godo, we refer to them +// here as DatabaseDBs. +// +// See: https://developers.digitalocean.com/documentation/v2#databases +type DatabasesService interface { + List(context.Context, *ListOptions) ([]Database, *Response, error) + Get(context.Context, string) (*Database, *Response, error) + Create(context.Context, *DatabaseCreateRequest) (*Database, *Response, error) + Delete(context.Context, string) (*Response, error) + Resize(context.Context, string, *DatabaseResizeRequest) (*Response, error) + Migrate(context.Context, string, *DatabaseMigrateRequest) (*Response, error) + UpdateMaintenance(context.Context, string, *DatabaseUpdateMaintenanceRequest) (*Response, error) + ListBackups(context.Context, string, *ListOptions) ([]DatabaseBackup, *Response, error) + GetUser(context.Context, string, string) (*DatabaseUser, *Response, error) + ListUsers(context.Context, string, *ListOptions) ([]DatabaseUser, *Response, error) + CreateUser(context.Context, string, *DatabaseCreateUserRequest) (*DatabaseUser, *Response, error) + DeleteUser(context.Context, string, string) (*Response, error) + ResetUserAuth(context.Context, string, string, *DatabaseResetUserAuthRequest) (*DatabaseUser, *Response, error) + ListDBs(context.Context, string, *ListOptions) ([]DatabaseDB, *Response, error) + CreateDB(context.Context, string, *DatabaseCreateDBRequest) (*DatabaseDB, *Response, error) + GetDB(context.Context, string, string) (*DatabaseDB, *Response, error) + DeleteDB(context.Context, string, string) (*Response, error) + ListPools(context.Context, string, *ListOptions) ([]DatabasePool, *Response, error) + CreatePool(context.Context, string, *DatabaseCreatePoolRequest) (*DatabasePool, *Response, error) + GetPool(context.Context, string, string) (*DatabasePool, *Response, error) + DeletePool(context.Context, string, string) (*Response, error) + GetReplica(context.Context, string, string) (*DatabaseReplica, *Response, error) + ListReplicas(context.Context, string, *ListOptions) ([]DatabaseReplica, *Response, error) + CreateReplica(context.Context, string, *DatabaseCreateReplicaRequest) (*DatabaseReplica, *Response, error) + DeleteReplica(context.Context, string, string) (*Response, error) + GetEvictionPolicy(context.Context, string) (string, *Response, error) + SetEvictionPolicy(context.Context, string, string) (*Response, error) + GetSQLMode(context.Context, string) (string, *Response, error) + SetSQLMode(context.Context, string, ...string) (*Response, error) + GetFirewallRules(context.Context, string) ([]DatabaseFirewallRule, *Response, error) + UpdateFirewallRules(context.Context, string, *DatabaseUpdateFirewallRulesRequest) (*Response, error) +} + +// DatabasesServiceOp handles communication with the Databases related methods +// of the DigitalOcean API. +type DatabasesServiceOp struct { + client *Client +} + +var _ DatabasesService = &DatabasesServiceOp{} + +// Database represents a DigitalOcean managed database product. These managed databases +// are usually comprised of a cluster of database nodes, a primary and 0 or more replicas. +// The EngineSlug is a string which indicates the type of database service. Some examples are +// "pg", "mysql" or "redis". A Database also includes connection information and other +// properties of the service like region, size and current status. +type Database struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + EngineSlug string `json:"engine,omitempty"` + VersionSlug string `json:"version,omitempty"` + Connection *DatabaseConnection `json:"connection,omitempty"` + PrivateConnection *DatabaseConnection `json:"private_connection,omitempty"` + Users []DatabaseUser `json:"users,omitempty"` + NumNodes int `json:"num_nodes,omitempty"` + SizeSlug string `json:"size,omitempty"` + DBNames []string `json:"db_names,omitempty"` + RegionSlug string `json:"region,omitempty"` + Status string `json:"status,omitempty"` + MaintenanceWindow *DatabaseMaintenanceWindow `json:"maintenance_window,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + PrivateNetworkUUID string `json:"private_network_uuid,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +// DatabaseConnection represents a database connection +type DatabaseConnection struct { + URI string `json:"uri,omitempty"` + Database string `json:"database,omitempty"` + Host string `json:"host,omitempty"` + Port int `json:"port,omitempty"` + User string `json:"user,omitempty"` + Password string `json:"password,omitempty"` + SSL bool `json:"ssl,omitempty"` +} + +// DatabaseUser represents a user in the database +type DatabaseUser struct { + Name string `json:"name,omitempty"` + Role string `json:"role,omitempty"` + Password string `json:"password,omitempty"` + MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"` +} + +// DatabaseMySQLUserSettings contains MySQL-specific user settings +type DatabaseMySQLUserSettings struct { + AuthPlugin string `json:"auth_plugin"` +} + +// DatabaseMaintenanceWindow represents the maintenance_window of a database +// cluster +type DatabaseMaintenanceWindow struct { + Day string `json:"day,omitempty"` + Hour string `json:"hour,omitempty"` + Pending bool `json:"pending,omitempty"` + Description []string `json:"description,omitempty"` +} + +// DatabaseBackup represents a database backup. +type DatabaseBackup struct { + CreatedAt time.Time `json:"created_at,omitempty"` + SizeGigabytes float64 `json:"size_gigabytes,omitempty"` +} + +// DatabaseCreateRequest represents a request to create a database cluster +type DatabaseCreateRequest struct { + Name string `json:"name,omitempty"` + EngineSlug string `json:"engine,omitempty"` + Version string `json:"version,omitempty"` + SizeSlug string `json:"size,omitempty"` + Region string `json:"region,omitempty"` + NumNodes int `json:"num_nodes,omitempty"` + PrivateNetworkUUID string `json:"private_network_uuid"` + Tags []string `json:"tags,omitempty"` +} + +// DatabaseResizeRequest can be used to initiate a database resize operation. +type DatabaseResizeRequest struct { + SizeSlug string `json:"size,omitempty"` + NumNodes int `json:"num_nodes,omitempty"` +} + +// DatabaseMigrateRequest can be used to initiate a database migrate operation. +type DatabaseMigrateRequest struct { + Region string `json:"region,omitempty"` + PrivateNetworkUUID string `json:"private_network_uuid"` +} + +// DatabaseUpdateMaintenanceRequest can be used to update the database's maintenance window. +type DatabaseUpdateMaintenanceRequest struct { + Day string `json:"day,omitempty"` + Hour string `json:"hour,omitempty"` +} + +// DatabaseDB represents an engine-specific database created within a database cluster. For SQL +// databases like PostgreSQL or MySQL, a "DB" refers to a database created on the RDBMS. For instance, +// a PostgreSQL database server can contain many database schemas, each with it's own settings, access +// permissions and data. ListDBs will return all databases present on the server. +type DatabaseDB struct { + Name string `json:"name"` +} + +// DatabaseReplica represents a read-only replica of a particular database +type DatabaseReplica struct { + Name string `json:"name"` + Connection *DatabaseConnection `json:"connection"` + PrivateConnection *DatabaseConnection `json:"private_connection,omitempty"` + Region string `json:"region"` + Status string `json:"status"` + CreatedAt time.Time `json:"created_at"` + PrivateNetworkUUID string `json:"private_network_uuid,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +// DatabasePool represents a database connection pool +type DatabasePool struct { + User string `json:"user"` + Name string `json:"name"` + Size int `json:"size"` + Database string `json:"db"` + Mode string `json:"mode"` + Connection *DatabaseConnection `json:"connection"` + PrivateConnection *DatabaseConnection `json:"private_connection,omitempty"` +} + +// DatabaseCreatePoolRequest is used to create a new database connection pool +type DatabaseCreatePoolRequest struct { + User string `json:"user"` + Name string `json:"name"` + Size int `json:"size"` + Database string `json:"db"` + Mode string `json:"mode"` +} + +// DatabaseCreateUserRequest is used to create a new database user +type DatabaseCreateUserRequest struct { + Name string `json:"name"` + MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"` +} + +// DatabaseResetUserAuth request is used to reset a users DB auth +type DatabaseResetUserAuthRequest struct { + MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"` +} + +// DatabaseCreateDBRequest is used to create a new engine-specific database within the cluster +type DatabaseCreateDBRequest struct { + Name string `json:"name"` +} + +// DatabaseCreateReplicaRequest is used to create a new read-only replica +type DatabaseCreateReplicaRequest struct { + Name string `json:"name"` + Region string `json:"region"` + Size string `json:"size"` + PrivateNetworkUUID string `json:"private_network_uuid"` + Tags []string `json:"tags,omitempty"` +} + +// DatabaseUpdateFirewallRulesRequest is used to set the firewall rules for a database +type DatabaseUpdateFirewallRulesRequest struct { + Rules []*DatabaseFirewallRule `json:"rules"` +} + +// DatabaseFirewallRule is a rule describing an inbound source to a database +type DatabaseFirewallRule struct { + UUID string `json:"uuid"` + ClusterUUID string `json:"cluster_uuid"` + Type string `json:"type"` + Value string `json:"value"` + CreatedAt time.Time `json:"created_at"` +} + +type databaseUserRoot struct { + User *DatabaseUser `json:"user"` +} + +type databaseUsersRoot struct { + Users []DatabaseUser `json:"users"` +} + +type databaseDBRoot struct { + DB *DatabaseDB `json:"db"` +} + +type databaseDBsRoot struct { + DBs []DatabaseDB `json:"dbs"` +} + +type databasesRoot struct { + Databases []Database `json:"databases"` +} + +type databaseRoot struct { + Database *Database `json:"database"` +} + +type databaseBackupsRoot struct { + Backups []DatabaseBackup `json:"backups"` +} + +type databasePoolRoot struct { + Pool *DatabasePool `json:"pool"` +} + +type databasePoolsRoot struct { + Pools []DatabasePool `json:"pools"` +} + +type databaseReplicaRoot struct { + Replica *DatabaseReplica `json:"replica"` +} + +type databaseReplicasRoot struct { + Replicas []DatabaseReplica `json:"replicas"` +} + +type evictionPolicyRoot struct { + EvictionPolicy string `json:"eviction_policy"` +} + +type sqlModeRoot struct { + SQLMode string `json:"sql_mode"` +} + +type databaseFirewallRuleRoot struct { + Rules []DatabaseFirewallRule `json:"rules"` +} + +// URN returns a URN identifier for the database +func (d Database) URN() string { + return ToURN("dbaas", d.ID) +} + +// List returns a list of the Databases visible with the caller's API token +func (svc *DatabasesServiceOp) List(ctx context.Context, opts *ListOptions) ([]Database, *Response, error) { + path := databaseBasePath + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databasesRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Databases, resp, nil +} + +// Get retrieves the details of a database cluster +func (svc *DatabasesServiceOp) Get(ctx context.Context, databaseID string) (*Database, *Response, error) { + path := fmt.Sprintf(databaseSinglePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Database, resp, nil +} + +// Create creates a database cluster +func (svc *DatabasesServiceOp) Create(ctx context.Context, create *DatabaseCreateRequest) (*Database, *Response, error) { + path := databaseBasePath + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create) + if err != nil { + return nil, nil, err + } + root := new(databaseRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Database, resp, nil +} + +// Delete deletes a database cluster. There is no way to recover a cluster once +// it has been destroyed. +func (svc *DatabasesServiceOp) Delete(ctx context.Context, databaseID string) (*Response, error) { + path := fmt.Sprintf("%s/%s", databaseBasePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// Resize resizes a database cluster by number of nodes or size +func (svc *DatabasesServiceOp) Resize(ctx context.Context, databaseID string, resize *DatabaseResizeRequest) (*Response, error) { + path := fmt.Sprintf(databaseResizePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, resize) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// Migrate migrates a database cluster to a new region +func (svc *DatabasesServiceOp) Migrate(ctx context.Context, databaseID string, migrate *DatabaseMigrateRequest) (*Response, error) { + path := fmt.Sprintf(databaseMigratePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, migrate) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// UpdateMaintenance updates the maintenance window on a cluster +func (svc *DatabasesServiceOp) UpdateMaintenance(ctx context.Context, databaseID string, maintenance *DatabaseUpdateMaintenanceRequest) (*Response, error) { + path := fmt.Sprintf(databaseMaintenancePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, maintenance) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// ListBackups returns a list of the current backups of a database +func (svc *DatabasesServiceOp) ListBackups(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseBackup, *Response, error) { + path := fmt.Sprintf(databaseBackupsPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseBackupsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Backups, resp, nil +} + +// GetUser returns the database user identified by userID +func (svc *DatabasesServiceOp) GetUser(ctx context.Context, databaseID, userID string) (*DatabaseUser, *Response, error) { + path := fmt.Sprintf(databaseUserPath, databaseID, userID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseUserRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.User, resp, nil +} + +// ListUsers returns all database users for the database +func (svc *DatabasesServiceOp) ListUsers(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseUser, *Response, error) { + path := fmt.Sprintf(databaseUsersPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseUsersRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Users, resp, nil +} + +// CreateUser will create a new database user +func (svc *DatabasesServiceOp) CreateUser(ctx context.Context, databaseID string, createUser *DatabaseCreateUserRequest) (*DatabaseUser, *Response, error) { + path := fmt.Sprintf(databaseUsersPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createUser) + if err != nil { + return nil, nil, err + } + root := new(databaseUserRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.User, resp, nil +} + +func (svc *DatabasesServiceOp) ResetUserAuth(ctx context.Context, databaseID, userID string, resetAuth *DatabaseResetUserAuthRequest) (*DatabaseUser, *Response, error) { + path := fmt.Sprintf(databaseResetUserAuthPath, databaseID, userID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, resetAuth) + if err != nil { + return nil, nil, err + } + root := new(databaseUserRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.User, resp, nil +} + +// DeleteUser will delete an existing database user +func (svc *DatabasesServiceOp) DeleteUser(ctx context.Context, databaseID, userID string) (*Response, error) { + path := fmt.Sprintf(databaseUserPath, databaseID, userID) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// ListDBs returns all databases for a given database cluster +func (svc *DatabasesServiceOp) ListDBs(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseDB, *Response, error) { + path := fmt.Sprintf(databaseDBsPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseDBsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.DBs, resp, nil +} + +// GetDB returns a single database by name +func (svc *DatabasesServiceOp) GetDB(ctx context.Context, databaseID, name string) (*DatabaseDB, *Response, error) { + path := fmt.Sprintf(databaseDBPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseDBRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.DB, resp, nil +} + +// CreateDB will create a new database +func (svc *DatabasesServiceOp) CreateDB(ctx context.Context, databaseID string, createDB *DatabaseCreateDBRequest) (*DatabaseDB, *Response, error) { + path := fmt.Sprintf(databaseDBsPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createDB) + if err != nil { + return nil, nil, err + } + root := new(databaseDBRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.DB, resp, nil +} + +// DeleteDB will delete an existing database +func (svc *DatabasesServiceOp) DeleteDB(ctx context.Context, databaseID, name string) (*Response, error) { + path := fmt.Sprintf(databaseDBPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// ListPools returns all connection pools for a given database cluster +func (svc *DatabasesServiceOp) ListPools(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabasePool, *Response, error) { + path := fmt.Sprintf(databasePoolsPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databasePoolsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Pools, resp, nil +} + +// GetPool returns a single database connection pool by name +func (svc *DatabasesServiceOp) GetPool(ctx context.Context, databaseID, name string) (*DatabasePool, *Response, error) { + path := fmt.Sprintf(databasePoolPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databasePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Pool, resp, nil +} + +// CreatePool will create a new database connection pool +func (svc *DatabasesServiceOp) CreatePool(ctx context.Context, databaseID string, createPool *DatabaseCreatePoolRequest) (*DatabasePool, *Response, error) { + path := fmt.Sprintf(databasePoolsPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createPool) + if err != nil { + return nil, nil, err + } + root := new(databasePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Pool, resp, nil +} + +// DeletePool will delete an existing database connection pool +func (svc *DatabasesServiceOp) DeletePool(ctx context.Context, databaseID, name string) (*Response, error) { + path := fmt.Sprintf(databasePoolPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// GetReplica returns a single database replica +func (svc *DatabasesServiceOp) GetReplica(ctx context.Context, databaseID, name string) (*DatabaseReplica, *Response, error) { + path := fmt.Sprintf(databaseReplicaPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseReplicaRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Replica, resp, nil +} + +// ListReplicas returns all read-only replicas for a given database cluster +func (svc *DatabasesServiceOp) ListReplicas(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseReplica, *Response, error) { + path := fmt.Sprintf(databaseReplicasPath, databaseID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(databaseReplicasRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Replicas, resp, nil +} + +// CreateReplica will create a new database connection pool +func (svc *DatabasesServiceOp) CreateReplica(ctx context.Context, databaseID string, createReplica *DatabaseCreateReplicaRequest) (*DatabaseReplica, *Response, error) { + path := fmt.Sprintf(databaseReplicasPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createReplica) + if err != nil { + return nil, nil, err + } + root := new(databaseReplicaRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Replica, resp, nil +} + +// DeleteReplica will delete an existing database replica +func (svc *DatabasesServiceOp) DeleteReplica(ctx context.Context, databaseID, name string) (*Response, error) { + path := fmt.Sprintf(databaseReplicaPath, databaseID, name) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// GetEvictionPolicy loads the eviction policy for a given Redis cluster. +func (svc *DatabasesServiceOp) GetEvictionPolicy(ctx context.Context, databaseID string) (string, *Response, error) { + path := fmt.Sprintf(databaseEvictionPolicyPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return "", nil, err + } + root := new(evictionPolicyRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return "", resp, err + } + return root.EvictionPolicy, resp, nil +} + +// SetEvictionPolicy updates the eviction policy for a given Redis cluster. +// +// The valid eviction policies are documented by the exported string constants +// with the prefix `EvictionPolicy`. +func (svc *DatabasesServiceOp) SetEvictionPolicy(ctx context.Context, databaseID, policy string) (*Response, error) { + path := fmt.Sprintf(databaseEvictionPolicyPath, databaseID) + root := &evictionPolicyRoot{EvictionPolicy: policy} + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, root) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// GetSQLMode loads the SQL Mode settings for a given MySQL cluster. +func (svc *DatabasesServiceOp) GetSQLMode(ctx context.Context, databaseID string) (string, *Response, error) { + path := fmt.Sprintf(databaseSQLModePath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return "", nil, err + } + root := &sqlModeRoot{} + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return "", resp, err + } + return root.SQLMode, resp, nil +} + +// SetSQLMode updates the SQL Mode settings for a given MySQL cluster. +func (svc *DatabasesServiceOp) SetSQLMode(ctx context.Context, databaseID string, sqlModes ...string) (*Response, error) { + path := fmt.Sprintf(databaseSQLModePath, databaseID) + root := &sqlModeRoot{SQLMode: strings.Join(sqlModes, ",")} + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, root) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// GetFirewallRules loads the inbound sources for a given cluster. +func (svc *DatabasesServiceOp) GetFirewallRules(ctx context.Context, databaseID string) ([]DatabaseFirewallRule, *Response, error) { + path := fmt.Sprintf(databaseFirewallRulesPath, databaseID) + root := new(databaseFirewallRuleRoot) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Rules, resp, nil +} + +// UpdateFirewallRules sets the inbound sources for a given cluster. +func (svc *DatabasesServiceOp) UpdateFirewallRules(ctx context.Context, databaseID string, firewallRulesReq *DatabaseUpdateFirewallRulesRequest) (*Response, error) { + path := fmt.Sprintf(databaseFirewallRulesPath, databaseID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, firewallRulesReq) + if err != nil { + return nil, err + } + return svc.client.Do(ctx, req, nil) +} diff --git a/vendor/github.com/digitalocean/godo/doc.go b/vendor/github.com/digitalocean/godo/doc.go new file mode 100644 index 0000000000..ac812e9740 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/doc.go @@ -0,0 +1,2 @@ +// Package godo is the DigtalOcean API v2 client for Go. +package godo diff --git a/vendor/github.com/digitalocean/godo/domains.go b/vendor/github.com/digitalocean/godo/domains.go new file mode 100644 index 0000000000..43c0424473 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/domains.go @@ -0,0 +1,341 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const domainsBasePath = "v2/domains" + +// DomainsService is an interface for managing DNS with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#domains and +// https://developers.digitalocean.com/documentation/v2#domain-records +type DomainsService interface { + List(context.Context, *ListOptions) ([]Domain, *Response, error) + Get(context.Context, string) (*Domain, *Response, error) + Create(context.Context, *DomainCreateRequest) (*Domain, *Response, error) + Delete(context.Context, string) (*Response, error) + + Records(context.Context, string, *ListOptions) ([]DomainRecord, *Response, error) + Record(context.Context, string, int) (*DomainRecord, *Response, error) + DeleteRecord(context.Context, string, int) (*Response, error) + EditRecord(context.Context, string, int, *DomainRecordEditRequest) (*DomainRecord, *Response, error) + CreateRecord(context.Context, string, *DomainRecordEditRequest) (*DomainRecord, *Response, error) +} + +// DomainsServiceOp handles communication with the domain related methods of the +// DigitalOcean API. +type DomainsServiceOp struct { + client *Client +} + +var _ DomainsService = &DomainsServiceOp{} + +// Domain represents a DigitalOcean domain +type Domain struct { + Name string `json:"name"` + TTL int `json:"ttl"` + ZoneFile string `json:"zone_file"` +} + +// domainRoot represents a response from the DigitalOcean API +type domainRoot struct { + Domain *Domain `json:"domain"` +} + +type domainsRoot struct { + Domains []Domain `json:"domains"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// DomainCreateRequest respresents a request to create a domain. +type DomainCreateRequest struct { + Name string `json:"name"` + IPAddress string `json:"ip_address,omitempty"` +} + +// DomainRecordRoot is the root of an individual Domain Record response +type domainRecordRoot struct { + DomainRecord *DomainRecord `json:"domain_record"` +} + +// DomainRecordsRoot is the root of a group of Domain Record responses +type domainRecordsRoot struct { + DomainRecords []DomainRecord `json:"domain_records"` + Links *Links `json:"links"` +} + +// DomainRecord represents a DigitalOcean DomainRecord +type DomainRecord struct { + ID int `json:"id,float64,omitempty"` + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Data string `json:"data,omitempty"` + Priority int `json:"priority"` + Port int `json:"port,omitempty"` + TTL int `json:"ttl,omitempty"` + Weight int `json:"weight"` + Flags int `json:"flags"` + Tag string `json:"tag,omitempty"` +} + +// DomainRecordEditRequest represents a request to update a domain record. +type DomainRecordEditRequest struct { + Type string `json:"type,omitempty"` + Name string `json:"name,omitempty"` + Data string `json:"data,omitempty"` + Priority int `json:"priority"` + Port int `json:"port,omitempty"` + TTL int `json:"ttl,omitempty"` + Weight int `json:"weight"` + Flags int `json:"flags"` + Tag string `json:"tag,omitempty"` +} + +func (d Domain) String() string { + return Stringify(d) +} + +func (d Domain) URN() string { + return ToURN("Domain", d.Name) +} + +// List all domains. +func (s DomainsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Domain, *Response, error) { + path := domainsBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(domainsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Domains, resp, err +} + +// Get individual domain. It requires a non-empty domain name. +func (s *DomainsServiceOp) Get(ctx context.Context, name string) (*Domain, *Response, error) { + if len(name) < 1 { + return nil, nil, NewArgError("name", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s", domainsBasePath, name) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(domainRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Domain, resp, err +} + +// Create a new domain +func (s *DomainsServiceOp) Create(ctx context.Context, createRequest *DomainCreateRequest) (*Domain, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + path := domainsBasePath + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(domainRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Domain, resp, err +} + +// Delete domain +func (s *DomainsServiceOp) Delete(ctx context.Context, name string) (*Response, error) { + if len(name) < 1 { + return nil, NewArgError("name", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s", domainsBasePath, name) + + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// Converts a DomainRecord to a string. +func (d DomainRecord) String() string { + return Stringify(d) +} + +// Converts a DomainRecordEditRequest to a string. +func (d DomainRecordEditRequest) String() string { + return Stringify(d) +} + +// Records returns a slice of DomainRecords for a domain +func (s *DomainsServiceOp) Records(ctx context.Context, domain string, opt *ListOptions) ([]DomainRecord, *Response, error) { + if len(domain) < 1 { + return nil, nil, NewArgError("domain", "cannot be an empty string") + } + + path := fmt.Sprintf("%s/%s/records", domainsBasePath, domain) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(domainRecordsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.DomainRecords, resp, err +} + +// Record returns the record id from a domain +func (s *DomainsServiceOp) Record(ctx context.Context, domain string, id int) (*DomainRecord, *Response, error) { + if len(domain) < 1 { + return nil, nil, NewArgError("domain", "cannot be an empty string") + } + + if id < 1 { + return nil, nil, NewArgError("id", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + record := new(domainRecordRoot) + resp, err := s.client.Do(ctx, req, record) + if err != nil { + return nil, resp, err + } + + return record.DomainRecord, resp, err +} + +// DeleteRecord deletes a record from a domain identified by id +func (s *DomainsServiceOp) DeleteRecord(ctx context.Context, domain string, id int) (*Response, error) { + if len(domain) < 1 { + return nil, NewArgError("domain", "cannot be an empty string") + } + + if id < 1 { + return nil, NewArgError("id", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id) + + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// EditRecord edits a record using a DomainRecordEditRequest +func (s *DomainsServiceOp) EditRecord(ctx context.Context, + domain string, + id int, + editRequest *DomainRecordEditRequest, +) (*DomainRecord, *Response, error) { + if len(domain) < 1 { + return nil, nil, NewArgError("domain", "cannot be an empty string") + } + + if id < 1 { + return nil, nil, NewArgError("id", "cannot be less than 1") + } + + if editRequest == nil { + return nil, nil, NewArgError("editRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id) + + req, err := s.client.NewRequest(ctx, http.MethodPut, path, editRequest) + if err != nil { + return nil, nil, err + } + + root := new(domainRecordRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.DomainRecord, resp, err +} + +// CreateRecord creates a record using a DomainRecordEditRequest +func (s *DomainsServiceOp) CreateRecord(ctx context.Context, + domain string, + createRequest *DomainRecordEditRequest) (*DomainRecord, *Response, error) { + if len(domain) < 1 { + return nil, nil, NewArgError("domain", "cannot be empty string") + } + + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s/records", domainsBasePath, domain) + req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest) + + if err != nil { + return nil, nil, err + } + + d := new(domainRecordRoot) + resp, err := s.client.Do(ctx, req, d) + if err != nil { + return nil, resp, err + } + + return d.DomainRecord, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/droplet_actions.go b/vendor/github.com/digitalocean/godo/droplet_actions.go new file mode 100644 index 0000000000..ddeacfc862 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/droplet_actions.go @@ -0,0 +1,329 @@ +package godo + +import ( + "context" + "fmt" + "net/http" + "net/url" +) + +// ActionRequest reprents DigitalOcean Action Request +type ActionRequest map[string]interface{} + +// DropletActionsService is an interface for interfacing with the Droplet actions +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#droplet-actions +type DropletActionsService interface { + Shutdown(context.Context, int) (*Action, *Response, error) + ShutdownByTag(context.Context, string) ([]Action, *Response, error) + PowerOff(context.Context, int) (*Action, *Response, error) + PowerOffByTag(context.Context, string) ([]Action, *Response, error) + PowerOn(context.Context, int) (*Action, *Response, error) + PowerOnByTag(context.Context, string) ([]Action, *Response, error) + PowerCycle(context.Context, int) (*Action, *Response, error) + PowerCycleByTag(context.Context, string) ([]Action, *Response, error) + Reboot(context.Context, int) (*Action, *Response, error) + Restore(context.Context, int, int) (*Action, *Response, error) + Resize(context.Context, int, string, bool) (*Action, *Response, error) + Rename(context.Context, int, string) (*Action, *Response, error) + Snapshot(context.Context, int, string) (*Action, *Response, error) + SnapshotByTag(context.Context, string, string) ([]Action, *Response, error) + EnableBackups(context.Context, int) (*Action, *Response, error) + EnableBackupsByTag(context.Context, string) ([]Action, *Response, error) + DisableBackups(context.Context, int) (*Action, *Response, error) + DisableBackupsByTag(context.Context, string) ([]Action, *Response, error) + PasswordReset(context.Context, int) (*Action, *Response, error) + RebuildByImageID(context.Context, int, int) (*Action, *Response, error) + RebuildByImageSlug(context.Context, int, string) (*Action, *Response, error) + ChangeKernel(context.Context, int, int) (*Action, *Response, error) + EnableIPv6(context.Context, int) (*Action, *Response, error) + EnableIPv6ByTag(context.Context, string) ([]Action, *Response, error) + EnablePrivateNetworking(context.Context, int) (*Action, *Response, error) + EnablePrivateNetworkingByTag(context.Context, string) ([]Action, *Response, error) + Get(context.Context, int, int) (*Action, *Response, error) + GetByURI(context.Context, string) (*Action, *Response, error) +} + +// DropletActionsServiceOp handles communication with the Droplet action related +// methods of the DigitalOcean API. +type DropletActionsServiceOp struct { + client *Client +} + +var _ DropletActionsService = &DropletActionsServiceOp{} + +// Shutdown a Droplet +func (s *DropletActionsServiceOp) Shutdown(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "shutdown"} + return s.doAction(ctx, id, request) +} + +// ShutdownByTag shuts down Droplets matched by a Tag. +func (s *DropletActionsServiceOp) ShutdownByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "shutdown"} + return s.doActionByTag(ctx, tag, request) +} + +// PowerOff a Droplet +func (s *DropletActionsServiceOp) PowerOff(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "power_off"} + return s.doAction(ctx, id, request) +} + +// PowerOffByTag powers off Droplets matched by a Tag. +func (s *DropletActionsServiceOp) PowerOffByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "power_off"} + return s.doActionByTag(ctx, tag, request) +} + +// PowerOn a Droplet +func (s *DropletActionsServiceOp) PowerOn(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "power_on"} + return s.doAction(ctx, id, request) +} + +// PowerOnByTag powers on Droplets matched by a Tag. +func (s *DropletActionsServiceOp) PowerOnByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "power_on"} + return s.doActionByTag(ctx, tag, request) +} + +// PowerCycle a Droplet +func (s *DropletActionsServiceOp) PowerCycle(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "power_cycle"} + return s.doAction(ctx, id, request) +} + +// PowerCycleByTag power cycles Droplets matched by a Tag. +func (s *DropletActionsServiceOp) PowerCycleByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "power_cycle"} + return s.doActionByTag(ctx, tag, request) +} + +// Reboot a Droplet +func (s *DropletActionsServiceOp) Reboot(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "reboot"} + return s.doAction(ctx, id, request) +} + +// Restore an image to a Droplet +func (s *DropletActionsServiceOp) Restore(ctx context.Context, id, imageID int) (*Action, *Response, error) { + requestType := "restore" + request := &ActionRequest{ + "type": requestType, + "image": float64(imageID), + } + return s.doAction(ctx, id, request) +} + +// Resize a Droplet +func (s *DropletActionsServiceOp) Resize(ctx context.Context, id int, sizeSlug string, resizeDisk bool) (*Action, *Response, error) { + requestType := "resize" + request := &ActionRequest{ + "type": requestType, + "size": sizeSlug, + "disk": resizeDisk, + } + return s.doAction(ctx, id, request) +} + +// Rename a Droplet +func (s *DropletActionsServiceOp) Rename(ctx context.Context, id int, name string) (*Action, *Response, error) { + requestType := "rename" + request := &ActionRequest{ + "type": requestType, + "name": name, + } + return s.doAction(ctx, id, request) +} + +// Snapshot a Droplet. +func (s *DropletActionsServiceOp) Snapshot(ctx context.Context, id int, name string) (*Action, *Response, error) { + requestType := "snapshot" + request := &ActionRequest{ + "type": requestType, + "name": name, + } + return s.doAction(ctx, id, request) +} + +// SnapshotByTag snapshots Droplets matched by a Tag. +func (s *DropletActionsServiceOp) SnapshotByTag(ctx context.Context, tag string, name string) ([]Action, *Response, error) { + requestType := "snapshot" + request := &ActionRequest{ + "type": requestType, + "name": name, + } + return s.doActionByTag(ctx, tag, request) +} + +// EnableBackups enables backups for a Droplet. +func (s *DropletActionsServiceOp) EnableBackups(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "enable_backups"} + return s.doAction(ctx, id, request) +} + +// EnableBackupsByTag enables backups for Droplets matched by a Tag. +func (s *DropletActionsServiceOp) EnableBackupsByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "enable_backups"} + return s.doActionByTag(ctx, tag, request) +} + +// DisableBackups disables backups for a Droplet. +func (s *DropletActionsServiceOp) DisableBackups(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "disable_backups"} + return s.doAction(ctx, id, request) +} + +// DisableBackupsByTag disables backups for Droplet matched by a Tag. +func (s *DropletActionsServiceOp) DisableBackupsByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "disable_backups"} + return s.doActionByTag(ctx, tag, request) +} + +// PasswordReset resets the password for a Droplet. +func (s *DropletActionsServiceOp) PasswordReset(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "password_reset"} + return s.doAction(ctx, id, request) +} + +// RebuildByImageID rebuilds a Droplet from an image with a given id. +func (s *DropletActionsServiceOp) RebuildByImageID(ctx context.Context, id, imageID int) (*Action, *Response, error) { + request := &ActionRequest{"type": "rebuild", "image": imageID} + return s.doAction(ctx, id, request) +} + +// RebuildByImageSlug rebuilds a Droplet from an Image matched by a given Slug. +func (s *DropletActionsServiceOp) RebuildByImageSlug(ctx context.Context, id int, slug string) (*Action, *Response, error) { + request := &ActionRequest{"type": "rebuild", "image": slug} + return s.doAction(ctx, id, request) +} + +// ChangeKernel changes the kernel for a Droplet. +func (s *DropletActionsServiceOp) ChangeKernel(ctx context.Context, id, kernelID int) (*Action, *Response, error) { + request := &ActionRequest{"type": "change_kernel", "kernel": kernelID} + return s.doAction(ctx, id, request) +} + +// EnableIPv6 enables IPv6 for a Droplet. +func (s *DropletActionsServiceOp) EnableIPv6(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "enable_ipv6"} + return s.doAction(ctx, id, request) +} + +// EnableIPv6ByTag enables IPv6 for Droplets matched by a Tag. +func (s *DropletActionsServiceOp) EnableIPv6ByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "enable_ipv6"} + return s.doActionByTag(ctx, tag, request) +} + +// EnablePrivateNetworking enables private networking for a Droplet. +func (s *DropletActionsServiceOp) EnablePrivateNetworking(ctx context.Context, id int) (*Action, *Response, error) { + request := &ActionRequest{"type": "enable_private_networking"} + return s.doAction(ctx, id, request) +} + +// EnablePrivateNetworkingByTag enables private networking for Droplets matched by a Tag. +func (s *DropletActionsServiceOp) EnablePrivateNetworkingByTag(ctx context.Context, tag string) ([]Action, *Response, error) { + request := &ActionRequest{"type": "enable_private_networking"} + return s.doActionByTag(ctx, tag, request) +} + +func (s *DropletActionsServiceOp) doAction(ctx context.Context, id int, request *ActionRequest) (*Action, *Response, error) { + if id < 1 { + return nil, nil, NewArgError("id", "cannot be less than 1") + } + + if request == nil { + return nil, nil, NewArgError("request", "request can't be nil") + } + + path := dropletActionPath(id) + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, request) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *DropletActionsServiceOp) doActionByTag(ctx context.Context, tag string, request *ActionRequest) ([]Action, *Response, error) { + if tag == "" { + return nil, nil, NewArgError("tag", "cannot be empty") + } + + if request == nil { + return nil, nil, NewArgError("request", "request can't be nil") + } + + path := dropletActionPathByTag(tag) + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, request) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Actions, resp, err +} + +// Get an action for a particular Droplet by id. +func (s *DropletActionsServiceOp) Get(ctx context.Context, dropletID, actionID int) (*Action, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + if actionID < 1 { + return nil, nil, NewArgError("actionID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", dropletActionPath(dropletID), actionID) + return s.get(ctx, path) +} + +// GetByURI gets an action for a particular Droplet by id. +func (s *DropletActionsServiceOp) GetByURI(ctx context.Context, rawurl string) (*Action, *Response, error) { + u, err := url.Parse(rawurl) + if err != nil { + return nil, nil, err + } + + return s.get(ctx, u.Path) + +} + +func (s *DropletActionsServiceOp) get(ctx context.Context, path string) (*Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err + +} + +func dropletActionPath(dropletID int) string { + return fmt.Sprintf("v2/droplets/%d/actions", dropletID) +} + +func dropletActionPathByTag(tag string) string { + return fmt.Sprintf("v2/droplets/actions?tag_name=%s", tag) +} diff --git a/vendor/github.com/digitalocean/godo/droplets.go b/vendor/github.com/digitalocean/godo/droplets.go new file mode 100644 index 0000000000..72edf2b438 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/droplets.go @@ -0,0 +1,592 @@ +package godo + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" +) + +const dropletBasePath = "v2/droplets" + +var errNoNetworks = errors.New("no networks have been defined") + +// DropletsService is an interface for interfacing with the Droplet +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#droplets +type DropletsService interface { + List(context.Context, *ListOptions) ([]Droplet, *Response, error) + ListByTag(context.Context, string, *ListOptions) ([]Droplet, *Response, error) + Get(context.Context, int) (*Droplet, *Response, error) + Create(context.Context, *DropletCreateRequest) (*Droplet, *Response, error) + CreateMultiple(context.Context, *DropletMultiCreateRequest) ([]Droplet, *Response, error) + Delete(context.Context, int) (*Response, error) + DeleteByTag(context.Context, string) (*Response, error) + Kernels(context.Context, int, *ListOptions) ([]Kernel, *Response, error) + Snapshots(context.Context, int, *ListOptions) ([]Image, *Response, error) + Backups(context.Context, int, *ListOptions) ([]Image, *Response, error) + Actions(context.Context, int, *ListOptions) ([]Action, *Response, error) + Neighbors(context.Context, int) ([]Droplet, *Response, error) +} + +// DropletsServiceOp handles communication with the Droplet related methods of the +// DigitalOcean API. +type DropletsServiceOp struct { + client *Client +} + +var _ DropletsService = &DropletsServiceOp{} + +// Droplet represents a DigitalOcean Droplet +type Droplet struct { + ID int `json:"id,float64,omitempty"` + Name string `json:"name,omitempty"` + Memory int `json:"memory,omitempty"` + Vcpus int `json:"vcpus,omitempty"` + Disk int `json:"disk,omitempty"` + Region *Region `json:"region,omitempty"` + Image *Image `json:"image,omitempty"` + Size *Size `json:"size,omitempty"` + SizeSlug string `json:"size_slug,omitempty"` + BackupIDs []int `json:"backup_ids,omitempty"` + NextBackupWindow *BackupWindow `json:"next_backup_window,omitempty"` + SnapshotIDs []int `json:"snapshot_ids,omitempty"` + Features []string `json:"features,omitempty"` + Locked bool `json:"locked,bool,omitempty"` + Status string `json:"status,omitempty"` + Networks *Networks `json:"networks,omitempty"` + Created string `json:"created_at,omitempty"` + Kernel *Kernel `json:"kernel,omitempty"` + Tags []string `json:"tags,omitempty"` + VolumeIDs []string `json:"volume_ids"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +// PublicIPv4 returns the public IPv4 address for the Droplet. +func (d *Droplet) PublicIPv4() (string, error) { + if d.Networks == nil { + return "", errNoNetworks + } + + for _, v4 := range d.Networks.V4 { + if v4.Type == "public" { + return v4.IPAddress, nil + } + } + + return "", nil +} + +// PrivateIPv4 returns the private IPv4 address for the Droplet. +func (d *Droplet) PrivateIPv4() (string, error) { + if d.Networks == nil { + return "", errNoNetworks + } + + for _, v4 := range d.Networks.V4 { + if v4.Type == "private" { + return v4.IPAddress, nil + } + } + + return "", nil +} + +// PublicIPv6 returns the public IPv6 address for the Droplet. +func (d *Droplet) PublicIPv6() (string, error) { + if d.Networks == nil { + return "", errNoNetworks + } + + for _, v6 := range d.Networks.V6 { + if v6.Type == "public" { + return v6.IPAddress, nil + } + } + + return "", nil +} + +// Kernel object +type Kernel struct { + ID int `json:"id,float64,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` +} + +// BackupWindow object +type BackupWindow struct { + Start *Timestamp `json:"start,omitempty"` + End *Timestamp `json:"end,omitempty"` +} + +// Convert Droplet to a string +func (d Droplet) String() string { + return Stringify(d) +} + +func (d Droplet) URN() string { + return ToURN("Droplet", d.ID) +} + +// DropletRoot represents a Droplet root +type dropletRoot struct { + Droplet *Droplet `json:"droplet"` + Links *Links `json:"links,omitempty"` +} + +type dropletsRoot struct { + Droplets []Droplet `json:"droplets"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type kernelsRoot struct { + Kernels []Kernel `json:"kernels,omitempty"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type dropletSnapshotsRoot struct { + Snapshots []Image `json:"snapshots,omitempty"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type backupsRoot struct { + Backups []Image `json:"backups,omitempty"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// DropletCreateImage identifies an image for the create request. It prefers slug over ID. +type DropletCreateImage struct { + ID int + Slug string +} + +// MarshalJSON returns either the slug or id of the image. It returns the id +// if the slug is empty. +func (d DropletCreateImage) MarshalJSON() ([]byte, error) { + if d.Slug != "" { + return json.Marshal(d.Slug) + } + + return json.Marshal(d.ID) +} + +// DropletCreateVolume identifies a volume to attach for the create request. It +// prefers Name over ID, +type DropletCreateVolume struct { + ID string + Name string +} + +// MarshalJSON returns an object with either the name or id of the volume. It +// returns the id if the name is empty. +func (d DropletCreateVolume) MarshalJSON() ([]byte, error) { + if d.Name != "" { + return json.Marshal(struct { + Name string `json:"name"` + }{Name: d.Name}) + } + + return json.Marshal(struct { + ID string `json:"id"` + }{ID: d.ID}) +} + +// DropletCreateSSHKey identifies a SSH Key for the create request. It prefers fingerprint over ID. +type DropletCreateSSHKey struct { + ID int + Fingerprint string +} + +// MarshalJSON returns either the fingerprint or id of the ssh key. It returns +// the id if the fingerprint is empty. +func (d DropletCreateSSHKey) MarshalJSON() ([]byte, error) { + if d.Fingerprint != "" { + return json.Marshal(d.Fingerprint) + } + + return json.Marshal(d.ID) +} + +// DropletCreateRequest represents a request to create a Droplet. +type DropletCreateRequest struct { + Name string `json:"name"` + Region string `json:"region"` + Size string `json:"size"` + Image DropletCreateImage `json:"image"` + SSHKeys []DropletCreateSSHKey `json:"ssh_keys"` + Backups bool `json:"backups"` + IPv6 bool `json:"ipv6"` + PrivateNetworking bool `json:"private_networking"` + Monitoring bool `json:"monitoring"` + UserData string `json:"user_data,omitempty"` + Volumes []DropletCreateVolume `json:"volumes,omitempty"` + Tags []string `json:"tags"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +// DropletMultiCreateRequest is a request to create multiple Droplets. +type DropletMultiCreateRequest struct { + Names []string `json:"names"` + Region string `json:"region"` + Size string `json:"size"` + Image DropletCreateImage `json:"image"` + SSHKeys []DropletCreateSSHKey `json:"ssh_keys"` + Backups bool `json:"backups"` + IPv6 bool `json:"ipv6"` + PrivateNetworking bool `json:"private_networking"` + Monitoring bool `json:"monitoring"` + UserData string `json:"user_data,omitempty"` + Tags []string `json:"tags"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +func (d DropletCreateRequest) String() string { + return Stringify(d) +} + +func (d DropletMultiCreateRequest) String() string { + return Stringify(d) +} + +// Networks represents the Droplet's Networks. +type Networks struct { + V4 []NetworkV4 `json:"v4,omitempty"` + V6 []NetworkV6 `json:"v6,omitempty"` +} + +// NetworkV4 represents a DigitalOcean IPv4 Network. +type NetworkV4 struct { + IPAddress string `json:"ip_address,omitempty"` + Netmask string `json:"netmask,omitempty"` + Gateway string `json:"gateway,omitempty"` + Type string `json:"type,omitempty"` +} + +func (n NetworkV4) String() string { + return Stringify(n) +} + +// NetworkV6 represents a DigitalOcean IPv6 network. +type NetworkV6 struct { + IPAddress string `json:"ip_address,omitempty"` + Netmask int `json:"netmask,omitempty"` + Gateway string `json:"gateway,omitempty"` + Type string `json:"type,omitempty"` +} + +func (n NetworkV6) String() string { + return Stringify(n) +} + +// Performs a list request given a path. +func (s *DropletsServiceOp) list(ctx context.Context, path string) ([]Droplet, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(dropletsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Droplets, resp, err +} + +// List all Droplets. +func (s *DropletsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Droplet, *Response, error) { + path := dropletBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + return s.list(ctx, path) +} + +// ListByTag lists all Droplets matched by a Tag. +func (s *DropletsServiceOp) ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Droplet, *Response, error) { + path := fmt.Sprintf("%s?tag_name=%s", dropletBasePath, tag) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + return s.list(ctx, path) +} + +// Get individual Droplet. +func (s *DropletsServiceOp) Get(ctx context.Context, dropletID int) (*Droplet, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", dropletBasePath, dropletID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(dropletRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Droplet, resp, err +} + +// Create Droplet +func (s *DropletsServiceOp) Create(ctx context.Context, createRequest *DropletCreateRequest) (*Droplet, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + path := dropletBasePath + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(dropletRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Droplet, resp, err +} + +// CreateMultiple creates multiple Droplets. +func (s *DropletsServiceOp) CreateMultiple(ctx context.Context, createRequest *DropletMultiCreateRequest) ([]Droplet, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + path := dropletBasePath + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(dropletsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Droplets, resp, err +} + +// Performs a delete request given a path +func (s *DropletsServiceOp) delete(ctx context.Context, path string) (*Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// Delete Droplet. +func (s *DropletsServiceOp) Delete(ctx context.Context, dropletID int) (*Response, error) { + if dropletID < 1 { + return nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", dropletBasePath, dropletID) + + return s.delete(ctx, path) +} + +// DeleteByTag deletes Droplets matched by a Tag. +func (s *DropletsServiceOp) DeleteByTag(ctx context.Context, tag string) (*Response, error) { + if tag == "" { + return nil, NewArgError("tag", "cannot be empty") + } + + path := fmt.Sprintf("%s?tag_name=%s", dropletBasePath, tag) + + return s.delete(ctx, path) +} + +// Kernels lists kernels available for a Droplet. +func (s *DropletsServiceOp) Kernels(ctx context.Context, dropletID int, opt *ListOptions) ([]Kernel, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/kernels", dropletBasePath, dropletID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(kernelsRoot) + resp, err := s.client.Do(ctx, req, root) + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Kernels, resp, err +} + +// Actions lists the actions for a Droplet. +func (s *DropletsServiceOp) Actions(ctx context.Context, dropletID int, opt *ListOptions) ([]Action, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/actions", dropletBasePath, dropletID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Actions, resp, err +} + +// Backups lists the backups for a Droplet. +func (s *DropletsServiceOp) Backups(ctx context.Context, dropletID int, opt *ListOptions) ([]Image, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/backups", dropletBasePath, dropletID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(backupsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Backups, resp, err +} + +// Snapshots lists the snapshots available for a Droplet. +func (s *DropletsServiceOp) Snapshots(ctx context.Context, dropletID int, opt *ListOptions) ([]Image, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/snapshots", dropletBasePath, dropletID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(dropletSnapshotsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Snapshots, resp, err +} + +// Neighbors lists the neighbors for a Droplet. +func (s *DropletsServiceOp) Neighbors(ctx context.Context, dropletID int) ([]Droplet, *Response, error) { + if dropletID < 1 { + return nil, nil, NewArgError("dropletID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d/neighbors", dropletBasePath, dropletID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(dropletsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Droplets, resp, err +} + +func (s *DropletsServiceOp) dropletActionStatus(ctx context.Context, uri string) (string, error) { + action, _, err := s.client.DropletActions.GetByURI(ctx, uri) + + if err != nil { + return "", err + } + + return action.Status, nil +} diff --git a/vendor/github.com/digitalocean/godo/errors.go b/vendor/github.com/digitalocean/godo/errors.go new file mode 100644 index 0000000000..a65ebd76b6 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/errors.go @@ -0,0 +1,24 @@ +package godo + +import "fmt" + +// ArgError is an error that represents an error with an input to godo. It +// identifies the argument and the cause (if possible). +type ArgError struct { + arg string + reason string +} + +var _ error = &ArgError{} + +// NewArgError creates an InputError. +func NewArgError(arg, reason string) *ArgError { + return &ArgError{ + arg: arg, + reason: reason, + } +} + +func (e *ArgError) Error() string { + return fmt.Sprintf("%s is invalid because %s", e.arg, e.reason) +} diff --git a/vendor/github.com/digitalocean/godo/firewalls.go b/vendor/github.com/digitalocean/godo/firewalls.go new file mode 100644 index 0000000000..8453e6645e --- /dev/null +++ b/vendor/github.com/digitalocean/godo/firewalls.go @@ -0,0 +1,271 @@ +package godo + +import ( + "context" + "net/http" + "path" + "strconv" +) + +const firewallsBasePath = "/v2/firewalls" + +// FirewallsService is an interface for managing Firewalls with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2/#firewalls +type FirewallsService interface { + Get(context.Context, string) (*Firewall, *Response, error) + Create(context.Context, *FirewallRequest) (*Firewall, *Response, error) + Update(context.Context, string, *FirewallRequest) (*Firewall, *Response, error) + Delete(context.Context, string) (*Response, error) + List(context.Context, *ListOptions) ([]Firewall, *Response, error) + ListByDroplet(context.Context, int, *ListOptions) ([]Firewall, *Response, error) + AddDroplets(context.Context, string, ...int) (*Response, error) + RemoveDroplets(context.Context, string, ...int) (*Response, error) + AddTags(context.Context, string, ...string) (*Response, error) + RemoveTags(context.Context, string, ...string) (*Response, error) + AddRules(context.Context, string, *FirewallRulesRequest) (*Response, error) + RemoveRules(context.Context, string, *FirewallRulesRequest) (*Response, error) +} + +// FirewallsServiceOp handles communication with Firewalls methods of the DigitalOcean API. +type FirewallsServiceOp struct { + client *Client +} + +// Firewall represents a DigitalOcean Firewall configuration. +type Firewall struct { + ID string `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + InboundRules []InboundRule `json:"inbound_rules"` + OutboundRules []OutboundRule `json:"outbound_rules"` + DropletIDs []int `json:"droplet_ids"` + Tags []string `json:"tags"` + Created string `json:"created_at"` + PendingChanges []PendingChange `json:"pending_changes"` +} + +// String creates a human-readable description of a Firewall. +func (fw Firewall) String() string { + return Stringify(fw) +} + +func (fw Firewall) URN() string { + return ToURN("Firewall", fw.ID) +} + +// FirewallRequest represents the configuration to be applied to an existing or a new Firewall. +type FirewallRequest struct { + Name string `json:"name"` + InboundRules []InboundRule `json:"inbound_rules"` + OutboundRules []OutboundRule `json:"outbound_rules"` + DropletIDs []int `json:"droplet_ids"` + Tags []string `json:"tags"` +} + +// FirewallRulesRequest represents rules configuration to be applied to an existing Firewall. +type FirewallRulesRequest struct { + InboundRules []InboundRule `json:"inbound_rules"` + OutboundRules []OutboundRule `json:"outbound_rules"` +} + +// InboundRule represents a DigitalOcean Firewall inbound rule. +type InboundRule struct { + Protocol string `json:"protocol,omitempty"` + PortRange string `json:"ports,omitempty"` + Sources *Sources `json:"sources"` +} + +// OutboundRule represents a DigitalOcean Firewall outbound rule. +type OutboundRule struct { + Protocol string `json:"protocol,omitempty"` + PortRange string `json:"ports,omitempty"` + Destinations *Destinations `json:"destinations"` +} + +// Sources represents a DigitalOcean Firewall InboundRule sources. +type Sources struct { + Addresses []string `json:"addresses,omitempty"` + Tags []string `json:"tags,omitempty"` + DropletIDs []int `json:"droplet_ids,omitempty"` + LoadBalancerUIDs []string `json:"load_balancer_uids,omitempty"` +} + +// PendingChange represents a DigitalOcean Firewall status details. +type PendingChange struct { + DropletID int `json:"droplet_id,omitempty"` + Removing bool `json:"removing,omitempty"` + Status string `json:"status,omitempty"` +} + +// Destinations represents a DigitalOcean Firewall OutboundRule destinations. +type Destinations struct { + Addresses []string `json:"addresses,omitempty"` + Tags []string `json:"tags,omitempty"` + DropletIDs []int `json:"droplet_ids,omitempty"` + LoadBalancerUIDs []string `json:"load_balancer_uids,omitempty"` +} + +var _ FirewallsService = &FirewallsServiceOp{} + +// Get an existing Firewall by its identifier. +func (fw *FirewallsServiceOp) Get(ctx context.Context, fID string) (*Firewall, *Response, error) { + path := path.Join(firewallsBasePath, fID) + + req, err := fw.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(firewallRoot) + resp, err := fw.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Firewall, resp, err +} + +// Create a new Firewall with a given configuration. +func (fw *FirewallsServiceOp) Create(ctx context.Context, fr *FirewallRequest) (*Firewall, *Response, error) { + req, err := fw.client.NewRequest(ctx, http.MethodPost, firewallsBasePath, fr) + if err != nil { + return nil, nil, err + } + + root := new(firewallRoot) + resp, err := fw.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Firewall, resp, err +} + +// Update an existing Firewall with new configuration. +func (fw *FirewallsServiceOp) Update(ctx context.Context, fID string, fr *FirewallRequest) (*Firewall, *Response, error) { + path := path.Join(firewallsBasePath, fID) + + req, err := fw.client.NewRequest(ctx, "PUT", path, fr) + if err != nil { + return nil, nil, err + } + + root := new(firewallRoot) + resp, err := fw.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Firewall, resp, err +} + +// Delete a Firewall by its identifier. +func (fw *FirewallsServiceOp) Delete(ctx context.Context, fID string) (*Response, error) { + path := path.Join(firewallsBasePath, fID) + return fw.createAndDoReq(ctx, http.MethodDelete, path, nil) +} + +// List Firewalls. +func (fw *FirewallsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Firewall, *Response, error) { + path, err := addOptions(firewallsBasePath, opt) + if err != nil { + return nil, nil, err + } + + return fw.listHelper(ctx, path) +} + +// ListByDroplet Firewalls. +func (fw *FirewallsServiceOp) ListByDroplet(ctx context.Context, dID int, opt *ListOptions) ([]Firewall, *Response, error) { + basePath := path.Join(dropletBasePath, strconv.Itoa(dID), "firewalls") + path, err := addOptions(basePath, opt) + if err != nil { + return nil, nil, err + } + + return fw.listHelper(ctx, path) +} + +// AddDroplets to a Firewall. +func (fw *FirewallsServiceOp) AddDroplets(ctx context.Context, fID string, dropletIDs ...int) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "droplets") + return fw.createAndDoReq(ctx, http.MethodPost, path, &dropletsRequest{IDs: dropletIDs}) +} + +// RemoveDroplets from a Firewall. +func (fw *FirewallsServiceOp) RemoveDroplets(ctx context.Context, fID string, dropletIDs ...int) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "droplets") + return fw.createAndDoReq(ctx, http.MethodDelete, path, &dropletsRequest{IDs: dropletIDs}) +} + +// AddTags to a Firewall. +func (fw *FirewallsServiceOp) AddTags(ctx context.Context, fID string, tags ...string) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "tags") + return fw.createAndDoReq(ctx, http.MethodPost, path, &tagsRequest{Tags: tags}) +} + +// RemoveTags from a Firewall. +func (fw *FirewallsServiceOp) RemoveTags(ctx context.Context, fID string, tags ...string) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "tags") + return fw.createAndDoReq(ctx, http.MethodDelete, path, &tagsRequest{Tags: tags}) +} + +// AddRules to a Firewall. +func (fw *FirewallsServiceOp) AddRules(ctx context.Context, fID string, rr *FirewallRulesRequest) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "rules") + return fw.createAndDoReq(ctx, http.MethodPost, path, rr) +} + +// RemoveRules from a Firewall. +func (fw *FirewallsServiceOp) RemoveRules(ctx context.Context, fID string, rr *FirewallRulesRequest) (*Response, error) { + path := path.Join(firewallsBasePath, fID, "rules") + return fw.createAndDoReq(ctx, http.MethodDelete, path, rr) +} + +type dropletsRequest struct { + IDs []int `json:"droplet_ids"` +} + +type tagsRequest struct { + Tags []string `json:"tags"` +} + +type firewallRoot struct { + Firewall *Firewall `json:"firewall"` +} + +type firewallsRoot struct { + Firewalls []Firewall `json:"firewalls"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +func (fw *FirewallsServiceOp) createAndDoReq(ctx context.Context, method, path string, v interface{}) (*Response, error) { + req, err := fw.client.NewRequest(ctx, method, path, v) + if err != nil { + return nil, err + } + + return fw.client.Do(ctx, req, nil) +} + +func (fw *FirewallsServiceOp) listHelper(ctx context.Context, path string) ([]Firewall, *Response, error) { + req, err := fw.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(firewallsRoot) + resp, err := fw.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Firewalls, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/floating_ips.go b/vendor/github.com/digitalocean/godo/floating_ips.go new file mode 100644 index 0000000000..1720d767b4 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/floating_ips.go @@ -0,0 +1,143 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const floatingBasePath = "v2/floating_ips" + +// FloatingIPsService is an interface for interfacing with the floating IPs +// endpoints of the Digital Ocean API. +// See: https://developers.digitalocean.com/documentation/v2#floating-ips +type FloatingIPsService interface { + List(context.Context, *ListOptions) ([]FloatingIP, *Response, error) + Get(context.Context, string) (*FloatingIP, *Response, error) + Create(context.Context, *FloatingIPCreateRequest) (*FloatingIP, *Response, error) + Delete(context.Context, string) (*Response, error) +} + +// FloatingIPsServiceOp handles communication with the floating IPs related methods of the +// DigitalOcean API. +type FloatingIPsServiceOp struct { + client *Client +} + +var _ FloatingIPsService = &FloatingIPsServiceOp{} + +// FloatingIP represents a Digital Ocean floating IP. +type FloatingIP struct { + Region *Region `json:"region"` + Droplet *Droplet `json:"droplet"` + IP string `json:"ip"` +} + +func (f FloatingIP) String() string { + return Stringify(f) +} + +func (f FloatingIP) URN() string { + return ToURN("FloatingIP", f.IP) +} + +type floatingIPsRoot struct { + FloatingIPs []FloatingIP `json:"floating_ips"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type floatingIPRoot struct { + FloatingIP *FloatingIP `json:"floating_ip"` + Links *Links `json:"links,omitempty"` +} + +// FloatingIPCreateRequest represents a request to create a floating IP. +// If DropletID is not empty, the floating IP will be assigned to the +// droplet. +type FloatingIPCreateRequest struct { + Region string `json:"region"` + DropletID int `json:"droplet_id,omitempty"` +} + +// List all floating IPs. +func (f *FloatingIPsServiceOp) List(ctx context.Context, opt *ListOptions) ([]FloatingIP, *Response, error) { + path := floatingBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := f.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(floatingIPsRoot) + resp, err := f.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.FloatingIPs, resp, err +} + +// Get an individual floating IP. +func (f *FloatingIPsServiceOp) Get(ctx context.Context, ip string) (*FloatingIP, *Response, error) { + path := fmt.Sprintf("%s/%s", floatingBasePath, ip) + + req, err := f.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(floatingIPRoot) + resp, err := f.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.FloatingIP, resp, err +} + +// Create a floating IP. If the DropletID field of the request is not empty, +// the floating IP will also be assigned to the droplet. +func (f *FloatingIPsServiceOp) Create(ctx context.Context, createRequest *FloatingIPCreateRequest) (*FloatingIP, *Response, error) { + path := floatingBasePath + + req, err := f.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(floatingIPRoot) + resp, err := f.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.FloatingIP, resp, err +} + +// Delete a floating IP. +func (f *FloatingIPsServiceOp) Delete(ctx context.Context, ip string) (*Response, error) { + path := fmt.Sprintf("%s/%s", floatingBasePath, ip) + + req, err := f.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := f.client.Do(ctx, req, nil) + + return resp, err +} diff --git a/vendor/github.com/digitalocean/godo/floating_ips_actions.go b/vendor/github.com/digitalocean/godo/floating_ips_actions.go new file mode 100644 index 0000000000..74ad279f95 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/floating_ips_actions.go @@ -0,0 +1,109 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +// FloatingIPActionsService is an interface for interfacing with the +// floating IPs actions endpoints of the Digital Ocean API. +// See: https://developers.digitalocean.com/documentation/v2#floating-ips-action +type FloatingIPActionsService interface { + Assign(ctx context.Context, ip string, dropletID int) (*Action, *Response, error) + Unassign(ctx context.Context, ip string) (*Action, *Response, error) + Get(ctx context.Context, ip string, actionID int) (*Action, *Response, error) + List(ctx context.Context, ip string, opt *ListOptions) ([]Action, *Response, error) +} + +// FloatingIPActionsServiceOp handles communication with the floating IPs +// action related methods of the DigitalOcean API. +type FloatingIPActionsServiceOp struct { + client *Client +} + +// Assign a floating IP to a droplet. +func (s *FloatingIPActionsServiceOp) Assign(ctx context.Context, ip string, dropletID int) (*Action, *Response, error) { + request := &ActionRequest{ + "type": "assign", + "droplet_id": dropletID, + } + return s.doAction(ctx, ip, request) +} + +// Unassign a floating IP from the droplet it is currently assigned to. +func (s *FloatingIPActionsServiceOp) Unassign(ctx context.Context, ip string) (*Action, *Response, error) { + request := &ActionRequest{"type": "unassign"} + return s.doAction(ctx, ip, request) +} + +// Get an action for a particular floating IP by id. +func (s *FloatingIPActionsServiceOp) Get(ctx context.Context, ip string, actionID int) (*Action, *Response, error) { + path := fmt.Sprintf("%s/%d", floatingIPActionPath(ip), actionID) + return s.get(ctx, path) +} + +// List the actions for a particular floating IP. +func (s *FloatingIPActionsServiceOp) List(ctx context.Context, ip string, opt *ListOptions) ([]Action, *Response, error) { + path := floatingIPActionPath(ip) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + return s.list(ctx, path) +} + +func (s *FloatingIPActionsServiceOp) doAction(ctx context.Context, ip string, request *ActionRequest) (*Action, *Response, error) { + path := floatingIPActionPath(ip) + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, request) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *FloatingIPActionsServiceOp) get(ctx context.Context, path string) (*Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *FloatingIPActionsServiceOp) list(ctx context.Context, path string) ([]Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Actions, resp, err +} + +func floatingIPActionPath(ip string) string { + return fmt.Sprintf("%s/%s/actions", floatingBasePath, ip) +} diff --git a/vendor/github.com/digitalocean/godo/go.mod b/vendor/github.com/digitalocean/godo/go.mod new file mode 100644 index 0000000000..36753b19f6 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/go.mod @@ -0,0 +1,16 @@ +module github.com/digitalocean/godo + +go 1.14 + +require ( + github.com/golang/protobuf v1.3.5 // indirect + github.com/google/go-querystring v1.0.0 + github.com/stretchr/testify v1.4.0 + golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect + golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d + google.golang.org/appengine v1.6.5 // indirect +) + +replace github.com/stretchr/objx => github.com/stretchr/objx v0.2.0 + +replace golang.org/x/crypto => golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a diff --git a/vendor/github.com/digitalocean/godo/go.sum b/vendor/github.com/digitalocean/godo/go.sum new file mode 100644 index 0000000000..ccd0f08af3 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/go.sum @@ -0,0 +1,42 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/digitalocean/godo/godo.go b/vendor/github.com/digitalocean/godo/godo.go new file mode 100644 index 0000000000..c6dde8e21b --- /dev/null +++ b/vendor/github.com/digitalocean/godo/godo.go @@ -0,0 +1,432 @@ +package godo + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/url" + "reflect" + "strconv" + "time" + + "github.com/google/go-querystring/query" + "golang.org/x/oauth2" +) + +const ( + libraryVersion = "1.37.0" + defaultBaseURL = "https://api.digitalocean.com/" + userAgent = "godo/" + libraryVersion + mediaType = "application/json" + + headerRateLimit = "RateLimit-Limit" + headerRateRemaining = "RateLimit-Remaining" + headerRateReset = "RateLimit-Reset" +) + +// Client manages communication with DigitalOcean V2 API. +type Client struct { + // HTTP client used to communicate with the DO API. + client *http.Client + + // Base URL for API requests. + BaseURL *url.URL + + // User agent for client + UserAgent string + + // Rate contains the current rate limit for the client as determined by the most recent + // API call. + Rate Rate + + // Services used for communicating with the API + Account AccountService + Actions ActionsService + Balance BalanceService + BillingHistory BillingHistoryService + CDNs CDNService + Domains DomainsService + Droplets DropletsService + DropletActions DropletActionsService + Images ImagesService + ImageActions ImageActionsService + Invoices InvoicesService + Keys KeysService + Regions RegionsService + Sizes SizesService + FloatingIPs FloatingIPsService + FloatingIPActions FloatingIPActionsService + Snapshots SnapshotsService + Storage StorageService + StorageActions StorageActionsService + Tags TagsService + LoadBalancers LoadBalancersService + Certificates CertificatesService + Firewalls FirewallsService + Projects ProjectsService + Kubernetes KubernetesService + Registry RegistryService + Databases DatabasesService + VPCs VPCsService + OneClick OneClickService + + // Optional function called after every successful request made to the DO APIs + onRequestCompleted RequestCompletionCallback +} + +// RequestCompletionCallback defines the type of the request callback function +type RequestCompletionCallback func(*http.Request, *http.Response) + +// ListOptions specifies the optional parameters to various List methods that +// support pagination. +type ListOptions struct { + // For paginated result sets, page of results to retrieve. + Page int `url:"page,omitempty"` + + // For paginated result sets, the number of results to include per page. + PerPage int `url:"per_page,omitempty"` +} + +// Response is a DigitalOcean response. This wraps the standard http.Response returned from DigitalOcean. +type Response struct { + *http.Response + + // Links that were returned with the response. These are parsed from + // request body and not the header. + Links *Links + + // Meta describes generic information about the response. + Meta *Meta + + // Monitoring URI + Monitor string + + Rate +} + +// An ErrorResponse reports the error caused by an API request +type ErrorResponse struct { + // HTTP response that caused this error + Response *http.Response + + // Error message + Message string `json:"message"` + + // RequestID returned from the API, useful to contact support. + RequestID string `json:"request_id"` +} + +// Rate contains the rate limit for the current client. +type Rate struct { + // The number of request per hour the client is currently limited to. + Limit int `json:"limit"` + + // The number of remaining requests the client can make this hour. + Remaining int `json:"remaining"` + + // The time at which the current rate limit will reset. + Reset Timestamp `json:"reset"` +} + +func addOptions(s string, opt interface{}) (string, error) { + v := reflect.ValueOf(opt) + + if v.Kind() == reflect.Ptr && v.IsNil() { + return s, nil + } + + origURL, err := url.Parse(s) + if err != nil { + return s, err + } + + origValues := origURL.Query() + + newValues, err := query.Values(opt) + if err != nil { + return s, err + } + + for k, v := range newValues { + origValues[k] = v + } + + origURL.RawQuery = origValues.Encode() + return origURL.String(), nil +} + +// NewFromToken returns a new DigitalOcean API client with the given API +// token. +func NewFromToken(token string) *Client { + ctx := context.Background() + + config := &oauth2.Config{} + ts := config.TokenSource(ctx, &oauth2.Token{AccessToken: token}) + + return NewClient(oauth2.NewClient(ctx, ts)) +} + +// NewClient returns a new DigitalOcean API client, using the given +// http.Client to perform all requests. +// +// Users who wish to pass their own http.Client should use this method. If +// you're in need of further customization, the godo.New method allows more +// options, such as setting a custom URL or a custom user agent string. +func NewClient(httpClient *http.Client) *Client { + if httpClient == nil { + httpClient = http.DefaultClient + } + + baseURL, _ := url.Parse(defaultBaseURL) + + c := &Client{client: httpClient, BaseURL: baseURL, UserAgent: userAgent} + c.Account = &AccountServiceOp{client: c} + c.Actions = &ActionsServiceOp{client: c} + c.Balance = &BalanceServiceOp{client: c} + c.BillingHistory = &BillingHistoryServiceOp{client: c} + c.CDNs = &CDNServiceOp{client: c} + c.Certificates = &CertificatesServiceOp{client: c} + c.Domains = &DomainsServiceOp{client: c} + c.Droplets = &DropletsServiceOp{client: c} + c.DropletActions = &DropletActionsServiceOp{client: c} + c.Firewalls = &FirewallsServiceOp{client: c} + c.FloatingIPs = &FloatingIPsServiceOp{client: c} + c.FloatingIPActions = &FloatingIPActionsServiceOp{client: c} + c.Images = &ImagesServiceOp{client: c} + c.ImageActions = &ImageActionsServiceOp{client: c} + c.Invoices = &InvoicesServiceOp{client: c} + c.Keys = &KeysServiceOp{client: c} + c.LoadBalancers = &LoadBalancersServiceOp{client: c} + c.Projects = &ProjectsServiceOp{client: c} + c.Regions = &RegionsServiceOp{client: c} + c.Sizes = &SizesServiceOp{client: c} + c.Snapshots = &SnapshotsServiceOp{client: c} + c.Storage = &StorageServiceOp{client: c} + c.StorageActions = &StorageActionsServiceOp{client: c} + c.Tags = &TagsServiceOp{client: c} + c.Kubernetes = &KubernetesServiceOp{client: c} + c.Registry = &RegistryServiceOp{client: c} + c.Databases = &DatabasesServiceOp{client: c} + c.VPCs = &VPCsServiceOp{client: c} + c.OneClick = &OneClickServiceOp{client: c} + + return c +} + +// ClientOpt are options for New. +type ClientOpt func(*Client) error + +// New returns a new DigitalOcean API client instance. +func New(httpClient *http.Client, opts ...ClientOpt) (*Client, error) { + c := NewClient(httpClient) + for _, opt := range opts { + if err := opt(c); err != nil { + return nil, err + } + } + + return c, nil +} + +// SetBaseURL is a client option for setting the base URL. +func SetBaseURL(bu string) ClientOpt { + return func(c *Client) error { + u, err := url.Parse(bu) + if err != nil { + return err + } + + c.BaseURL = u + return nil + } +} + +// SetUserAgent is a client option for setting the user agent. +func SetUserAgent(ua string) ClientOpt { + return func(c *Client) error { + c.UserAgent = fmt.Sprintf("%s %s", ua, c.UserAgent) + return nil + } +} + +// NewRequest creates an API request. A relative URL can be provided in urlStr, which will be resolved to the +// BaseURL of the Client. Relative URLS should always be specified without a preceding slash. If specified, the +// value pointed to by body is JSON encoded and included in as the request body. +func (c *Client) NewRequest(ctx context.Context, method, urlStr string, body interface{}) (*http.Request, error) { + u, err := c.BaseURL.Parse(urlStr) + if err != nil { + return nil, err + } + + buf := new(bytes.Buffer) + if body != nil { + err = json.NewEncoder(buf).Encode(body) + if err != nil { + return nil, err + } + } + + req, err := http.NewRequest(method, u.String(), buf) + if err != nil { + return nil, err + } + + req.Header.Add("Content-Type", mediaType) + req.Header.Add("Accept", mediaType) + req.Header.Add("User-Agent", c.UserAgent) + return req, nil +} + +// OnRequestCompleted sets the DO API request completion callback +func (c *Client) OnRequestCompleted(rc RequestCompletionCallback) { + c.onRequestCompleted = rc +} + +// newResponse creates a new Response for the provided http.Response +func newResponse(r *http.Response) *Response { + response := Response{Response: r} + response.populateRate() + + return &response +} + +// populateRate parses the rate related headers and populates the response Rate. +func (r *Response) populateRate() { + if limit := r.Header.Get(headerRateLimit); limit != "" { + r.Rate.Limit, _ = strconv.Atoi(limit) + } + if remaining := r.Header.Get(headerRateRemaining); remaining != "" { + r.Rate.Remaining, _ = strconv.Atoi(remaining) + } + if reset := r.Header.Get(headerRateReset); reset != "" { + if v, _ := strconv.ParseInt(reset, 10, 64); v != 0 { + r.Rate.Reset = Timestamp{time.Unix(v, 0)} + } + } +} + +// Do sends an API request and returns the API response. The API response is JSON decoded and stored in the value +// pointed to by v, or returned as an error if an API error has occurred. If v implements the io.Writer interface, +// the raw response will be written to v, without attempting to decode it. +func (c *Client) Do(ctx context.Context, req *http.Request, v interface{}) (*Response, error) { + resp, err := DoRequestWithClient(ctx, c.client, req) + if err != nil { + return nil, err + } + if c.onRequestCompleted != nil { + c.onRequestCompleted(req, resp) + } + + defer func() { + if rerr := resp.Body.Close(); err == nil { + err = rerr + } + }() + + response := newResponse(resp) + c.Rate = response.Rate + + err = CheckResponse(resp) + if err != nil { + return response, err + } + + if v != nil { + if w, ok := v.(io.Writer); ok { + _, err = io.Copy(w, resp.Body) + if err != nil { + return nil, err + } + } else { + err = json.NewDecoder(resp.Body).Decode(v) + if err != nil { + return nil, err + } + } + } + + return response, err +} + +// DoRequest submits an HTTP request. +func DoRequest(ctx context.Context, req *http.Request) (*http.Response, error) { + return DoRequestWithClient(ctx, http.DefaultClient, req) +} + +// DoRequestWithClient submits an HTTP request using the specified client. +func DoRequestWithClient( + ctx context.Context, + client *http.Client, + req *http.Request) (*http.Response, error) { + req = req.WithContext(ctx) + return client.Do(req) +} + +func (r *ErrorResponse) Error() string { + if r.RequestID != "" { + return fmt.Sprintf("%v %v: %d (request %q) %v", + r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.RequestID, r.Message) + } + return fmt.Sprintf("%v %v: %d %v", + r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.Message) +} + +// CheckResponse checks the API response for errors, and returns them if present. A response is considered an +// error if it has a status code outside the 200 range. API error responses are expected to have either no response +// body, or a JSON response body that maps to ErrorResponse. Any other response body will be silently ignored. +func CheckResponse(r *http.Response) error { + if c := r.StatusCode; c >= 200 && c <= 299 { + return nil + } + + errorResponse := &ErrorResponse{Response: r} + data, err := ioutil.ReadAll(r.Body) + if err == nil && len(data) > 0 { + err := json.Unmarshal(data, errorResponse) + if err != nil { + errorResponse.Message = string(data) + } + } + + return errorResponse +} + +func (r Rate) String() string { + return Stringify(r) +} + +// String is a helper routine that allocates a new string value +// to store v and returns a pointer to it. +func String(v string) *string { + p := new(string) + *p = v + return p +} + +// Int is a helper routine that allocates a new int32 value +// to store v and returns a pointer to it, but unlike Int32 +// its argument value is an int. +func Int(v int) *int { + p := new(int) + *p = v + return p +} + +// Bool is a helper routine that allocates a new bool value +// to store v and returns a pointer to it. +func Bool(v bool) *bool { + p := new(bool) + *p = v + return p +} + +// StreamToString converts a reader to a string +func StreamToString(stream io.Reader) string { + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(stream) + return buf.String() +} diff --git a/vendor/github.com/digitalocean/godo/image_actions.go b/vendor/github.com/digitalocean/godo/image_actions.go new file mode 100644 index 0000000000..976f7c6872 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/image_actions.go @@ -0,0 +1,102 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +// ImageActionsService is an interface for interfacing with the image actions +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#image-actions +type ImageActionsService interface { + Get(context.Context, int, int) (*Action, *Response, error) + Transfer(context.Context, int, *ActionRequest) (*Action, *Response, error) + Convert(context.Context, int) (*Action, *Response, error) +} + +// ImageActionsServiceOp handles communition with the image action related methods of the +// DigitalOcean API. +type ImageActionsServiceOp struct { + client *Client +} + +var _ ImageActionsService = &ImageActionsServiceOp{} + +// Transfer an image +func (i *ImageActionsServiceOp) Transfer(ctx context.Context, imageID int, transferRequest *ActionRequest) (*Action, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannot be less than 1") + } + + if transferRequest == nil { + return nil, nil, NewArgError("transferRequest", "cannot be nil") + } + + path := fmt.Sprintf("v2/images/%d/actions", imageID) + + req, err := i.client.NewRequest(ctx, http.MethodPost, path, transferRequest) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := i.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +// Convert an image to a snapshot +func (i *ImageActionsServiceOp) Convert(ctx context.Context, imageID int) (*Action, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannont be less than 1") + } + + path := fmt.Sprintf("v2/images/%d/actions", imageID) + + convertRequest := &ActionRequest{ + "type": "convert", + } + + req, err := i.client.NewRequest(ctx, http.MethodPost, path, convertRequest) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := i.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +// Get an action for a particular image by id. +func (i *ImageActionsServiceOp) Get(ctx context.Context, imageID, actionID int) (*Action, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannot be less than 1") + } + + if actionID < 1 { + return nil, nil, NewArgError("actionID", "cannot be less than 1") + } + + path := fmt.Sprintf("v2/images/%d/actions/%d", imageID, actionID) + + req, err := i.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := i.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/images.go b/vendor/github.com/digitalocean/godo/images.go new file mode 100644 index 0000000000..64e72e75eb --- /dev/null +++ b/vendor/github.com/digitalocean/godo/images.go @@ -0,0 +1,245 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const imageBasePath = "v2/images" + +// ImagesService is an interface for interfacing with the images +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#images +type ImagesService interface { + List(context.Context, *ListOptions) ([]Image, *Response, error) + ListDistribution(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) + ListApplication(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) + ListUser(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) + ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Image, *Response, error) + GetByID(context.Context, int) (*Image, *Response, error) + GetBySlug(context.Context, string) (*Image, *Response, error) + Create(context.Context, *CustomImageCreateRequest) (*Image, *Response, error) + Update(context.Context, int, *ImageUpdateRequest) (*Image, *Response, error) + Delete(context.Context, int) (*Response, error) +} + +// ImagesServiceOp handles communication with the image related methods of the +// DigitalOcean API. +type ImagesServiceOp struct { + client *Client +} + +var _ ImagesService = &ImagesServiceOp{} + +// Image represents a DigitalOcean Image +type Image struct { + ID int `json:"id,float64,omitempty"` + Name string `json:"name,omitempty"` + Type string `json:"type,omitempty"` + Distribution string `json:"distribution,omitempty"` + Slug string `json:"slug,omitempty"` + Public bool `json:"public,omitempty"` + Regions []string `json:"regions,omitempty"` + MinDiskSize int `json:"min_disk_size,omitempty"` + SizeGigaBytes float64 `json:"size_gigabytes,omitempty"` + Created string `json:"created_at,omitempty"` + Description string `json:"description,omitempty"` + Tags []string `json:"tags,omitempty"` + Status string `json:"status,omitempty"` + ErrorMessage string `json:"error_message,omitempty"` +} + +// ImageUpdateRequest represents a request to update an image. +type ImageUpdateRequest struct { + Name string `json:"name"` +} + +// CustomImageCreateRequest represents a request to create a custom image. +type CustomImageCreateRequest struct { + Name string `json:"name"` + Url string `json:"url"` + Region string `json:"region"` + Distribution string `json:"distribution,omitempty"` + Description string `json:"description,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +type imageRoot struct { + Image *Image +} + +type imagesRoot struct { + Images []Image + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type listImageOptions struct { + Private bool `url:"private,omitempty"` + Type string `url:"type,omitempty"` + Tag string `url:"tag_name,omitempty"` +} + +func (i Image) String() string { + return Stringify(i) +} + +// List lists all the images available. +func (s *ImagesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) { + return s.list(ctx, opt, nil) +} + +// ListDistribution lists all the distribution images. +func (s *ImagesServiceOp) ListDistribution(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) { + listOpt := listImageOptions{Type: "distribution"} + return s.list(ctx, opt, &listOpt) +} + +// ListApplication lists all the application images. +func (s *ImagesServiceOp) ListApplication(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) { + listOpt := listImageOptions{Type: "application"} + return s.list(ctx, opt, &listOpt) +} + +// ListUser lists all the user images. +func (s *ImagesServiceOp) ListUser(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) { + listOpt := listImageOptions{Private: true} + return s.list(ctx, opt, &listOpt) +} + +// ListByTag lists all images with a specific tag applied. +func (s *ImagesServiceOp) ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Image, *Response, error) { + listOpt := listImageOptions{Tag: tag} + return s.list(ctx, opt, &listOpt) +} + +// GetByID retrieves an image by id. +func (s *ImagesServiceOp) GetByID(ctx context.Context, imageID int) (*Image, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannot be less than 1") + } + + return s.get(ctx, interface{}(imageID)) +} + +// GetBySlug retrieves an image by slug. +func (s *ImagesServiceOp) GetBySlug(ctx context.Context, slug string) (*Image, *Response, error) { + if len(slug) < 1 { + return nil, nil, NewArgError("slug", "cannot be blank") + } + + return s.get(ctx, interface{}(slug)) +} + +func (s *ImagesServiceOp) Create(ctx context.Context, createRequest *CustomImageCreateRequest) (*Image, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + req, err := s.client.NewRequest(ctx, http.MethodPost, imageBasePath, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(imageRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Image, resp, err +} + +// Update an image name. +func (s *ImagesServiceOp) Update(ctx context.Context, imageID int, updateRequest *ImageUpdateRequest) (*Image, *Response, error) { + if imageID < 1 { + return nil, nil, NewArgError("imageID", "cannot be less than 1") + } + + if updateRequest == nil { + return nil, nil, NewArgError("updateRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%d", imageBasePath, imageID) + req, err := s.client.NewRequest(ctx, http.MethodPut, path, updateRequest) + if err != nil { + return nil, nil, err + } + + root := new(imageRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Image, resp, err +} + +// Delete an image. +func (s *ImagesServiceOp) Delete(ctx context.Context, imageID int) (*Response, error) { + if imageID < 1 { + return nil, NewArgError("imageID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", imageBasePath, imageID) + + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// Helper method for getting an individual image +func (s *ImagesServiceOp) get(ctx context.Context, ID interface{}) (*Image, *Response, error) { + path := fmt.Sprintf("%s/%v", imageBasePath, ID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(imageRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Image, resp, err +} + +// Helper method for listing images +func (s *ImagesServiceOp) list(ctx context.Context, opt *ListOptions, listOpt *listImageOptions) ([]Image, *Response, error) { + path := imageBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + path, err = addOptions(path, listOpt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(imagesRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Images, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/invoices.go b/vendor/github.com/digitalocean/godo/invoices.go new file mode 100644 index 0000000000..cc111f871e --- /dev/null +++ b/vendor/github.com/digitalocean/godo/invoices.go @@ -0,0 +1,225 @@ +package godo + +import ( + "bytes" + "context" + "fmt" + "net/http" + "time" +) + +const invoicesBasePath = "v2/customers/my/invoices" + +// InvoicesService is an interface for interfacing with the Invoice +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2/#invoices +type InvoicesService interface { + Get(context.Context, string, *ListOptions) (*Invoice, *Response, error) + GetPDF(context.Context, string) ([]byte, *Response, error) + GetCSV(context.Context, string) ([]byte, *Response, error) + List(context.Context, *ListOptions) (*InvoiceList, *Response, error) + GetSummary(context.Context, string) (*InvoiceSummary, *Response, error) +} + +// InvoicesServiceOp handles communication with the Invoice related methods of +// the DigitalOcean API. +type InvoicesServiceOp struct { + client *Client +} + +var _ InvoicesService = &InvoicesServiceOp{} + +// Invoice represents a DigitalOcean Invoice +type Invoice struct { + InvoiceItems []InvoiceItem `json:"invoice_items"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// InvoiceItem represents a line-item on a DigitalOcean Invoice +type InvoiceItem struct { + Product string `json:"product"` + ResourceID string `json:"resource_id"` + ResourceUUID string `json:"resource_uuid"` + GroupDescription string `json:"group_description"` + Description string `json:"description"` + Amount string `json:"amount"` + Duration string `json:"duration"` + DurationUnit string `json:"duration_unit"` + StartTime time.Time `json:"start_time"` + EndTime time.Time `json:"end_time"` + ProjectName string `json:"project_name"` +} + +// InvoiceList contains a paginated list of all of a customer's invoices. +// The InvoicePreview is the month-to-date usage generated by DigitalOcean. +type InvoiceList struct { + Invoices []InvoiceListItem `json:"invoices"` + InvoicePreview InvoiceListItem `json:"invoice_preview"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// InvoiceListItem contains a small list of information about a customer's invoice. +// More information can be found in the Invoice or InvoiceSummary +type InvoiceListItem struct { + InvoiceUUID string `json:"invoice_uuid"` + Amount string `json:"amount"` + InvoicePeriod string `json:"invoice_period"` + UpdatedAt time.Time `json:"updated_at"` +} + +// InvoiceSummary contains metadata and summarized usage for an invoice generated by DigitalOcean +type InvoiceSummary struct { + InvoiceUUID string `json:"invoice_uuid"` + BillingPeriod string `json:"billing_period"` + Amount string `json:"amount"` + UserName string `json:"user_name"` + UserBillingAddress Address `json:"user_billing_address"` + UserCompany string `json:"user_company"` + UserEmail string `json:"user_email"` + ProductCharges InvoiceSummaryBreakdown `json:"product_charges"` + Overages InvoiceSummaryBreakdown `json:"overages"` + Taxes InvoiceSummaryBreakdown `json:"taxes"` + CreditsAndAdjustments InvoiceSummaryBreakdown `json:"credits_and_adjustments"` +} + +// Address represents the billing address of a customer +type Address struct { + AddressLine1 string `json:"address_line1"` + AddressLine2 string `json:"address_line2"` + City string `json:"city"` + Region string `json:"region"` + PostalCode string `json:"postal_code"` + CountryISO2Code string `json:"country_iso2_code"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +// InvoiceSummaryBreakdown is a grouped set of InvoiceItems from an invoice +type InvoiceSummaryBreakdown struct { + Name string `json:"name"` + Amount string `json:"amount"` + Items []InvoiceSummaryBreakdownItem `json:"items"` +} + +// InvoiceSummaryBreakdownItem further breaks down the InvoiceSummary by product +type InvoiceSummaryBreakdownItem struct { + Name string `json:"name"` + Amount string `json:"amount"` + Count string `json:"count"` +} + +func (i Invoice) String() string { + return Stringify(i) +} + +// Get detailed invoice items for an Invoice +func (s *InvoicesServiceOp) Get(ctx context.Context, invoiceUUID string, opt *ListOptions) (*Invoice, *Response, error) { + path := fmt.Sprintf("%s/%s", invoicesBasePath, invoiceUUID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(Invoice) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root, resp, err +} + +// List invoices for a customer +func (s *InvoicesServiceOp) List(ctx context.Context, opt *ListOptions) (*InvoiceList, *Response, error) { + path := invoicesBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(InvoiceList) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root, resp, err +} + +// Get a summary of metadata and summarized usage for an Invoice +func (s *InvoicesServiceOp) GetSummary(ctx context.Context, invoiceUUID string) (*InvoiceSummary, *Response, error) { + path := fmt.Sprintf("%s/%s/summary", invoicesBasePath, invoiceUUID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(InvoiceSummary) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root, resp, err +} + +// Get the pdf for an Invoice +func (s *InvoicesServiceOp) GetPDF(ctx context.Context, invoiceUUID string) ([]byte, *Response, error) { + path := fmt.Sprintf("%s/%s/pdf", invoicesBasePath, invoiceUUID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + var root bytes.Buffer + resp, err := s.client.Do(ctx, req, &root) + if err != nil { + return nil, resp, err + } + + return root.Bytes(), resp, err +} + +// Get the csv for an Invoice +func (s *InvoicesServiceOp) GetCSV(ctx context.Context, invoiceUUID string) ([]byte, *Response, error) { + path := fmt.Sprintf("%s/%s/csv", invoicesBasePath, invoiceUUID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + var root bytes.Buffer + resp, err := s.client.Do(ctx, req, &root) + if err != nil { + return nil, resp, err + } + + return root.Bytes(), resp, err +} diff --git a/vendor/github.com/digitalocean/godo/keys.go b/vendor/github.com/digitalocean/godo/keys.go new file mode 100644 index 0000000000..b97554d14c --- /dev/null +++ b/vendor/github.com/digitalocean/godo/keys.go @@ -0,0 +1,230 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const keysBasePath = "v2/account/keys" + +// KeysService is an interface for interfacing with the keys +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#keys +type KeysService interface { + List(context.Context, *ListOptions) ([]Key, *Response, error) + GetByID(context.Context, int) (*Key, *Response, error) + GetByFingerprint(context.Context, string) (*Key, *Response, error) + Create(context.Context, *KeyCreateRequest) (*Key, *Response, error) + UpdateByID(context.Context, int, *KeyUpdateRequest) (*Key, *Response, error) + UpdateByFingerprint(context.Context, string, *KeyUpdateRequest) (*Key, *Response, error) + DeleteByID(context.Context, int) (*Response, error) + DeleteByFingerprint(context.Context, string) (*Response, error) +} + +// KeysServiceOp handles communication with key related method of the +// DigitalOcean API. +type KeysServiceOp struct { + client *Client +} + +var _ KeysService = &KeysServiceOp{} + +// Key represents a DigitalOcean Key. +type Key struct { + ID int `json:"id,float64,omitempty"` + Name string `json:"name,omitempty"` + Fingerprint string `json:"fingerprint,omitempty"` + PublicKey string `json:"public_key,omitempty"` +} + +// KeyUpdateRequest represents a request to update a DigitalOcean key. +type KeyUpdateRequest struct { + Name string `json:"name"` +} + +type keysRoot struct { + SSHKeys []Key `json:"ssh_keys"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type keyRoot struct { + SSHKey *Key `json:"ssh_key"` +} + +func (s Key) String() string { + return Stringify(s) +} + +// KeyCreateRequest represents a request to create a new key. +type KeyCreateRequest struct { + Name string `json:"name"` + PublicKey string `json:"public_key"` +} + +// List all keys +func (s *KeysServiceOp) List(ctx context.Context, opt *ListOptions) ([]Key, *Response, error) { + path := keysBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(keysRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.SSHKeys, resp, err +} + +// Performs a get given a path +func (s *KeysServiceOp) get(ctx context.Context, path string) (*Key, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(keyRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.SSHKey, resp, err +} + +// GetByID gets a Key by id +func (s *KeysServiceOp) GetByID(ctx context.Context, keyID int) (*Key, *Response, error) { + if keyID < 1 { + return nil, nil, NewArgError("keyID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", keysBasePath, keyID) + return s.get(ctx, path) +} + +// GetByFingerprint gets a Key by by fingerprint +func (s *KeysServiceOp) GetByFingerprint(ctx context.Context, fingerprint string) (*Key, *Response, error) { + if len(fingerprint) < 1 { + return nil, nil, NewArgError("fingerprint", "cannot not be empty") + } + + path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint) + return s.get(ctx, path) +} + +// Create a key using a KeyCreateRequest +func (s *KeysServiceOp) Create(ctx context.Context, createRequest *KeyCreateRequest) (*Key, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + req, err := s.client.NewRequest(ctx, http.MethodPost, keysBasePath, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(keyRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.SSHKey, resp, err +} + +// UpdateByID updates a key name by ID. +func (s *KeysServiceOp) UpdateByID(ctx context.Context, keyID int, updateRequest *KeyUpdateRequest) (*Key, *Response, error) { + if keyID < 1 { + return nil, nil, NewArgError("keyID", "cannot be less than 1") + } + + if updateRequest == nil { + return nil, nil, NewArgError("updateRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%d", keysBasePath, keyID) + req, err := s.client.NewRequest(ctx, "PUT", path, updateRequest) + if err != nil { + return nil, nil, err + } + + root := new(keyRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.SSHKey, resp, err +} + +// UpdateByFingerprint updates a key name by fingerprint. +func (s *KeysServiceOp) UpdateByFingerprint(ctx context.Context, fingerprint string, updateRequest *KeyUpdateRequest) (*Key, *Response, error) { + if len(fingerprint) < 1 { + return nil, nil, NewArgError("fingerprint", "cannot be empty") + } + + if updateRequest == nil { + return nil, nil, NewArgError("updateRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint) + req, err := s.client.NewRequest(ctx, "PUT", path, updateRequest) + if err != nil { + return nil, nil, err + } + + root := new(keyRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.SSHKey, resp, err +} + +// Delete key using a path +func (s *KeysServiceOp) delete(ctx context.Context, path string) (*Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// DeleteByID deletes a key by its id +func (s *KeysServiceOp) DeleteByID(ctx context.Context, keyID int) (*Response, error) { + if keyID < 1 { + return nil, NewArgError("keyID", "cannot be less than 1") + } + + path := fmt.Sprintf("%s/%d", keysBasePath, keyID) + return s.delete(ctx, path) +} + +// DeleteByFingerprint deletes a key by its fingerprint +func (s *KeysServiceOp) DeleteByFingerprint(ctx context.Context, fingerprint string) (*Response, error) { + if len(fingerprint) < 1 { + return nil, NewArgError("fingerprint", "cannot be empty") + } + + path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint) + return s.delete(ctx, path) +} diff --git a/vendor/github.com/digitalocean/godo/kubernetes.go b/vendor/github.com/digitalocean/godo/kubernetes.go new file mode 100644 index 0000000000..9b80fefb6d --- /dev/null +++ b/vendor/github.com/digitalocean/godo/kubernetes.go @@ -0,0 +1,697 @@ +package godo + +import ( + "bytes" + "context" + "encoding" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" +) + +const ( + kubernetesBasePath = "/v2/kubernetes" + kubernetesClustersPath = kubernetesBasePath + "/clusters" + kubernetesOptionsPath = kubernetesBasePath + "/options" +) + +// KubernetesService is an interface for interfacing with the Kubernetes endpoints +// of the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#kubernetes +type KubernetesService interface { + Create(context.Context, *KubernetesClusterCreateRequest) (*KubernetesCluster, *Response, error) + Get(context.Context, string) (*KubernetesCluster, *Response, error) + GetUser(context.Context, string) (*KubernetesClusterUser, *Response, error) + GetUpgrades(context.Context, string) ([]*KubernetesVersion, *Response, error) + GetKubeConfig(context.Context, string) (*KubernetesClusterConfig, *Response, error) + GetCredentials(context.Context, string, *KubernetesClusterCredentialsGetRequest) (*KubernetesClusterCredentials, *Response, error) + List(context.Context, *ListOptions) ([]*KubernetesCluster, *Response, error) + Update(context.Context, string, *KubernetesClusterUpdateRequest) (*KubernetesCluster, *Response, error) + Upgrade(context.Context, string, *KubernetesClusterUpgradeRequest) (*Response, error) + Delete(context.Context, string) (*Response, error) + + CreateNodePool(ctx context.Context, clusterID string, req *KubernetesNodePoolCreateRequest) (*KubernetesNodePool, *Response, error) + GetNodePool(ctx context.Context, clusterID, poolID string) (*KubernetesNodePool, *Response, error) + ListNodePools(ctx context.Context, clusterID string, opts *ListOptions) ([]*KubernetesNodePool, *Response, error) + UpdateNodePool(ctx context.Context, clusterID, poolID string, req *KubernetesNodePoolUpdateRequest) (*KubernetesNodePool, *Response, error) + // RecycleNodePoolNodes is DEPRECATED please use DeleteNode + // The method will be removed in godo 2.0. + RecycleNodePoolNodes(ctx context.Context, clusterID, poolID string, req *KubernetesNodePoolRecycleNodesRequest) (*Response, error) + DeleteNodePool(ctx context.Context, clusterID, poolID string) (*Response, error) + DeleteNode(ctx context.Context, clusterID, poolID, nodeID string, req *KubernetesNodeDeleteRequest) (*Response, error) + + GetOptions(context.Context) (*KubernetesOptions, *Response, error) +} + +var _ KubernetesService = &KubernetesServiceOp{} + +// KubernetesServiceOp handles communication with Kubernetes methods of the DigitalOcean API. +type KubernetesServiceOp struct { + client *Client +} + +// KubernetesClusterCreateRequest represents a request to create a Kubernetes cluster. +type KubernetesClusterCreateRequest struct { + Name string `json:"name,omitempty"` + RegionSlug string `json:"region,omitempty"` + VersionSlug string `json:"version,omitempty"` + Tags []string `json:"tags,omitempty"` + VPCUUID string `json:"vpc_uuid,omitempty"` + + NodePools []*KubernetesNodePoolCreateRequest `json:"node_pools,omitempty"` + + MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy"` + AutoUpgrade bool `json:"auto_upgrade"` +} + +// KubernetesClusterUpdateRequest represents a request to update a Kubernetes cluster. +type KubernetesClusterUpdateRequest struct { + Name string `json:"name,omitempty"` + Tags []string `json:"tags,omitempty"` + MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy,omitempty"` + AutoUpgrade *bool `json:"auto_upgrade,omitempty"` +} + +// KubernetesClusterUpgradeRequest represents a request to upgrade a Kubernetes cluster. +type KubernetesClusterUpgradeRequest struct { + VersionSlug string `json:"version,omitempty"` +} + +// KubernetesNodePoolCreateRequest represents a request to create a node pool for a +// Kubernetes cluster. +type KubernetesNodePoolCreateRequest struct { + Name string `json:"name,omitempty"` + Size string `json:"size,omitempty"` + Count int `json:"count,omitempty"` + Tags []string `json:"tags,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + AutoScale bool `json:"auto_scale,omitempty"` + MinNodes int `json:"min_nodes,omitempty"` + MaxNodes int `json:"max_nodes,omitempty"` +} + +// KubernetesNodePoolUpdateRequest represents a request to update a node pool in a +// Kubernetes cluster. +type KubernetesNodePoolUpdateRequest struct { + Name string `json:"name,omitempty"` + Count *int `json:"count,omitempty"` + Tags []string `json:"tags,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + AutoScale *bool `json:"auto_scale,omitempty"` + MinNodes *int `json:"min_nodes,omitempty"` + MaxNodes *int `json:"max_nodes,omitempty"` +} + +// KubernetesNodePoolRecycleNodesRequest is DEPRECATED please use DeleteNode +// The type will be removed in godo 2.0. +type KubernetesNodePoolRecycleNodesRequest struct { + Nodes []string `json:"nodes,omitempty"` +} + +// KubernetesNodeDeleteRequest is a request to delete a specific node in a node pool. +type KubernetesNodeDeleteRequest struct { + // Replace will cause a new node to be created to replace the deleted node. + Replace bool `json:"replace,omitempty"` + + // SkipDrain skips draining the node before deleting it. + SkipDrain bool `json:"skip_drain,omitempty"` +} + +// KubernetesClusterCredentialsGetRequest is a request to get cluster credentials. +type KubernetesClusterCredentialsGetRequest struct { + ExpirySeconds *int `json:"expiry_seconds,omitempty"` +} + +// KubernetesCluster represents a Kubernetes cluster. +type KubernetesCluster struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + RegionSlug string `json:"region,omitempty"` + VersionSlug string `json:"version,omitempty"` + ClusterSubnet string `json:"cluster_subnet,omitempty"` + ServiceSubnet string `json:"service_subnet,omitempty"` + IPv4 string `json:"ipv4,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + Tags []string `json:"tags,omitempty"` + VPCUUID string `json:"vpc_uuid,omitempty"` + + NodePools []*KubernetesNodePool `json:"node_pools,omitempty"` + + MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy,omitempty"` + AutoUpgrade bool `json:"auto_upgrade,omitempty"` + + Status *KubernetesClusterStatus `json:"status,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// KubernetesClusterUser represents a Kubernetes cluster user. +type KubernetesClusterUser struct { + Username string `json:"username,omitempty"` + Groups []string `json:"groups,omitempty"` +} + +// KubernetesClusterCredentials represents Kubernetes cluster credentials. +type KubernetesClusterCredentials struct { + Server string `json:"server"` + CertificateAuthorityData []byte `json:"certificate_authority_data"` + ClientCertificateData []byte `json:"client_certificate_data"` + ClientKeyData []byte `json:"client_key_data"` + Token string `json:"token"` + ExpiresAt time.Time `json:"expires_at"` +} + +// KubernetesMaintenancePolicy is a configuration to set the maintenance window +// of a cluster +type KubernetesMaintenancePolicy struct { + StartTime string `json:"start_time"` + Duration string `json:"duration"` + Day KubernetesMaintenancePolicyDay `json:"day"` +} + +// KubernetesMaintenancePolicyDay represents the possible days of a maintenance +// window +type KubernetesMaintenancePolicyDay int + +const ( + KubernetesMaintenanceDayAny KubernetesMaintenancePolicyDay = iota + KubernetesMaintenanceDayMonday + KubernetesMaintenanceDayTuesday + KubernetesMaintenanceDayWednesday + KubernetesMaintenanceDayThursday + KubernetesMaintenanceDayFriday + KubernetesMaintenanceDaySaturday + KubernetesMaintenanceDaySunday +) + +var ( + days = [...]string{ + "any", + "monday", + "tuesday", + "wednesday", + "thursday", + "friday", + "saturday", + "sunday", + } + + toDay = map[string]KubernetesMaintenancePolicyDay{ + "any": KubernetesMaintenanceDayAny, + "monday": KubernetesMaintenanceDayMonday, + "tuesday": KubernetesMaintenanceDayTuesday, + "wednesday": KubernetesMaintenanceDayWednesday, + "thursday": KubernetesMaintenanceDayThursday, + "friday": KubernetesMaintenanceDayFriday, + "saturday": KubernetesMaintenanceDaySaturday, + "sunday": KubernetesMaintenanceDaySunday, + } +) + +// KubernetesMaintenanceToDay returns the appropriate KubernetesMaintenancePolicyDay for the given string. +func KubernetesMaintenanceToDay(day string) (KubernetesMaintenancePolicyDay, error) { + d, ok := toDay[day] + if !ok { + return 0, fmt.Errorf("unknown day: %q", day) + } + + return d, nil +} + +func (k KubernetesMaintenancePolicyDay) String() string { + if KubernetesMaintenanceDayAny <= k && k <= KubernetesMaintenanceDaySunday { + return days[k] + } + return fmt.Sprintf("%d !Weekday", k) + +} + +func (k *KubernetesMaintenancePolicyDay) UnmarshalJSON(data []byte) error { + var val string + if err := json.Unmarshal(data, &val); err != nil { + return err + } + + parsed, err := KubernetesMaintenanceToDay(val) + if err != nil { + return err + } + *k = parsed + return nil +} + +func (k KubernetesMaintenancePolicyDay) MarshalJSON() ([]byte, error) { + if KubernetesMaintenanceDayAny <= k && k <= KubernetesMaintenanceDaySunday { + return json.Marshal(days[k]) + } + + return nil, fmt.Errorf("invalid day: %d", k) +} + +// Possible states for a cluster. +const ( + KubernetesClusterStatusProvisioning = KubernetesClusterStatusState("provisioning") + KubernetesClusterStatusRunning = KubernetesClusterStatusState("running") + KubernetesClusterStatusDegraded = KubernetesClusterStatusState("degraded") + KubernetesClusterStatusError = KubernetesClusterStatusState("error") + KubernetesClusterStatusDeleted = KubernetesClusterStatusState("deleted") + KubernetesClusterStatusUpgrading = KubernetesClusterStatusState("upgrading") + KubernetesClusterStatusInvalid = KubernetesClusterStatusState("invalid") +) + +// KubernetesClusterStatusState represents states for a cluster. +type KubernetesClusterStatusState string + +var _ encoding.TextUnmarshaler = (*KubernetesClusterStatusState)(nil) + +// UnmarshalText unmarshals the state. +func (s *KubernetesClusterStatusState) UnmarshalText(text []byte) error { + switch KubernetesClusterStatusState(strings.ToLower(string(text))) { + case KubernetesClusterStatusProvisioning: + *s = KubernetesClusterStatusProvisioning + case KubernetesClusterStatusRunning: + *s = KubernetesClusterStatusRunning + case KubernetesClusterStatusDegraded: + *s = KubernetesClusterStatusDegraded + case KubernetesClusterStatusError: + *s = KubernetesClusterStatusError + case KubernetesClusterStatusDeleted: + *s = KubernetesClusterStatusDeleted + case KubernetesClusterStatusUpgrading: + *s = KubernetesClusterStatusUpgrading + case "", KubernetesClusterStatusInvalid: + *s = KubernetesClusterStatusInvalid + default: + return fmt.Errorf("unknown cluster state %q", string(text)) + } + return nil +} + +// KubernetesClusterStatus describes the status of a cluster. +type KubernetesClusterStatus struct { + State KubernetesClusterStatusState `json:"state,omitempty"` + Message string `json:"message,omitempty"` +} + +// KubernetesNodePool represents a node pool in a Kubernetes cluster. +type KubernetesNodePool struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Size string `json:"size,omitempty"` + Count int `json:"count,omitempty"` + Tags []string `json:"tags,omitempty"` + Labels map[string]string `json:"labels,omitempty"` + AutoScale bool `json:"auto_scale,omitempty"` + MinNodes int `json:"min_nodes,omitempty"` + MaxNodes int `json:"max_nodes,omitempty"` + + Nodes []*KubernetesNode `json:"nodes,omitempty"` +} + +// KubernetesNode represents a Node in a node pool in a Kubernetes cluster. +type KubernetesNode struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Status *KubernetesNodeStatus `json:"status,omitempty"` + DropletID string `json:"droplet_id,omitempty"` + + CreatedAt time.Time `json:"created_at,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +// KubernetesNodeStatus represents the status of a particular Node in a Kubernetes cluster. +type KubernetesNodeStatus struct { + State string `json:"state,omitempty"` + Message string `json:"message,omitempty"` +} + +// KubernetesOptions represents options available for creating Kubernetes clusters. +type KubernetesOptions struct { + Versions []*KubernetesVersion `json:"versions,omitempty"` + Regions []*KubernetesRegion `json:"regions,omitempty"` + Sizes []*KubernetesNodeSize `json:"sizes,omitempty"` +} + +// KubernetesVersion is a DigitalOcean Kubernetes release. +type KubernetesVersion struct { + Slug string `json:"slug,omitempty"` + KubernetesVersion string `json:"kubernetes_version,omitempty"` +} + +// KubernetesNodeSize is a node sizes supported for Kubernetes clusters. +type KubernetesNodeSize struct { + Name string `json:"name"` + Slug string `json:"slug"` +} + +// KubernetesRegion is a region usable by Kubernetes clusters. +type KubernetesRegion struct { + Name string `json:"name"` + Slug string `json:"slug"` +} + +type kubernetesClustersRoot struct { + Clusters []*KubernetesCluster `json:"kubernetes_clusters,omitempty"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta"` +} + +type kubernetesClusterRoot struct { + Cluster *KubernetesCluster `json:"kubernetes_cluster,omitempty"` +} + +type kubernetesClusterUserRoot struct { + User *KubernetesClusterUser `json:"kubernetes_cluster_user,omitempty"` +} + +type kubernetesNodePoolRoot struct { + NodePool *KubernetesNodePool `json:"node_pool,omitempty"` +} + +type kubernetesNodePoolsRoot struct { + NodePools []*KubernetesNodePool `json:"node_pools,omitempty"` + Links *Links `json:"links,omitempty"` +} + +type kubernetesUpgradesRoot struct { + AvailableUpgradeVersions []*KubernetesVersion `json:"available_upgrade_versions,omitempty"` +} + +// Get retrieves the details of a Kubernetes cluster. +func (svc *KubernetesServiceOp) Get(ctx context.Context, clusterID string) (*KubernetesCluster, *Response, error) { + path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClusterRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Cluster, resp, nil +} + +// GetUser retrieves the details of a Kubernetes cluster user. +func (svc *KubernetesServiceOp) GetUser(ctx context.Context, clusterID string) (*KubernetesClusterUser, *Response, error) { + path := fmt.Sprintf("%s/%s/user", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClusterUserRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.User, resp, nil +} + +// GetUpgrades retrieves versions a Kubernetes cluster can be upgraded to. An +// upgrade can be requested using `Upgrade`. +func (svc *KubernetesServiceOp) GetUpgrades(ctx context.Context, clusterID string) ([]*KubernetesVersion, *Response, error) { + path := fmt.Sprintf("%s/%s/upgrades", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesUpgradesRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, nil, err + } + return root.AvailableUpgradeVersions, resp, nil +} + +// Create creates a Kubernetes cluster. +func (svc *KubernetesServiceOp) Create(ctx context.Context, create *KubernetesClusterCreateRequest) (*KubernetesCluster, *Response, error) { + path := kubernetesClustersPath + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClusterRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Cluster, resp, nil +} + +// Delete deletes a Kubernetes cluster. There is no way to recover a cluster +// once it has been destroyed. +func (svc *KubernetesServiceOp) Delete(ctx context.Context, clusterID string) (*Response, error) { + path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// List returns a list of the Kubernetes clusters visible with the caller's API token. +func (svc *KubernetesServiceOp) List(ctx context.Context, opts *ListOptions) ([]*KubernetesCluster, *Response, error) { + path := kubernetesClustersPath + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClustersRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Clusters, resp, nil +} + +// KubernetesClusterConfig is the content of a Kubernetes config file, which can be +// used to interact with your Kubernetes cluster using `kubectl`. +// See: https://kubernetes.io/docs/tasks/tools/install-kubectl/ +type KubernetesClusterConfig struct { + KubeconfigYAML []byte +} + +// GetKubeConfig returns a Kubernetes config file for the specified cluster. +func (svc *KubernetesServiceOp) GetKubeConfig(ctx context.Context, clusterID string) (*KubernetesClusterConfig, *Response, error) { + path := fmt.Sprintf("%s/%s/kubeconfig", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + configBytes := bytes.NewBuffer(nil) + resp, err := svc.client.Do(ctx, req, configBytes) + if err != nil { + return nil, resp, err + } + res := &KubernetesClusterConfig{ + KubeconfigYAML: configBytes.Bytes(), + } + return res, resp, nil +} + +// GetCredentials returns a Kubernetes API server credentials for the specified cluster. +func (svc *KubernetesServiceOp) GetCredentials(ctx context.Context, clusterID string, get *KubernetesClusterCredentialsGetRequest) (*KubernetesClusterCredentials, *Response, error) { + path := fmt.Sprintf("%s/%s/credentials", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + q := req.URL.Query() + if get.ExpirySeconds != nil { + q.Add("expiry_seconds", strconv.Itoa(*get.ExpirySeconds)) + } + req.URL.RawQuery = q.Encode() + credentials := new(KubernetesClusterCredentials) + resp, err := svc.client.Do(ctx, req, credentials) + if err != nil { + return nil, nil, err + } + return credentials, resp, nil +} + +// Update updates a Kubernetes cluster's properties. +func (svc *KubernetesServiceOp) Update(ctx context.Context, clusterID string, update *KubernetesClusterUpdateRequest) (*KubernetesCluster, *Response, error) { + path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, update) + if err != nil { + return nil, nil, err + } + root := new(kubernetesClusterRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Cluster, resp, nil +} + +// Upgrade upgrades a Kubernetes cluster to a new version. Valid upgrade +// versions for a given cluster can be retrieved with `GetUpgrades`. +func (svc *KubernetesServiceOp) Upgrade(ctx context.Context, clusterID string, upgrade *KubernetesClusterUpgradeRequest) (*Response, error) { + path := fmt.Sprintf("%s/%s/upgrade", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, upgrade) + if err != nil { + return nil, err + } + return svc.client.Do(ctx, req, nil) +} + +// CreateNodePool creates a new node pool in an existing Kubernetes cluster. +func (svc *KubernetesServiceOp) CreateNodePool(ctx context.Context, clusterID string, create *KubernetesNodePoolCreateRequest) (*KubernetesNodePool, *Response, error) { + path := fmt.Sprintf("%s/%s/node_pools", kubernetesClustersPath, clusterID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create) + if err != nil { + return nil, nil, err + } + root := new(kubernetesNodePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.NodePool, resp, nil +} + +// GetNodePool retrieves an existing node pool in a Kubernetes cluster. +func (svc *KubernetesServiceOp) GetNodePool(ctx context.Context, clusterID, poolID string) (*KubernetesNodePool, *Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID) + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesNodePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.NodePool, resp, nil +} + +// ListNodePools lists all the node pools found in a Kubernetes cluster. +func (svc *KubernetesServiceOp) ListNodePools(ctx context.Context, clusterID string, opts *ListOptions) ([]*KubernetesNodePool, *Response, error) { + path := fmt.Sprintf("%s/%s/node_pools", kubernetesClustersPath, clusterID) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesNodePoolsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.NodePools, resp, nil +} + +// UpdateNodePool updates the details of an existing node pool. +func (svc *KubernetesServiceOp) UpdateNodePool(ctx context.Context, clusterID, poolID string, update *KubernetesNodePoolUpdateRequest) (*KubernetesNodePool, *Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID) + req, err := svc.client.NewRequest(ctx, http.MethodPut, path, update) + if err != nil { + return nil, nil, err + } + root := new(kubernetesNodePoolRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.NodePool, resp, nil +} + +// RecycleNodePoolNodes is DEPRECATED please use DeleteNode +// The method will be removed in godo 2.0. +func (svc *KubernetesServiceOp) RecycleNodePoolNodes(ctx context.Context, clusterID, poolID string, recycle *KubernetesNodePoolRecycleNodesRequest) (*Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s/recycle", kubernetesClustersPath, clusterID, poolID) + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, recycle) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// DeleteNodePool deletes a node pool, and subsequently all the nodes in that pool. +func (svc *KubernetesServiceOp) DeleteNodePool(ctx context.Context, clusterID, poolID string) (*Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// DeleteNode deletes a specific node in a node pool. +func (svc *KubernetesServiceOp) DeleteNode(ctx context.Context, clusterID, poolID, nodeID string, deleteReq *KubernetesNodeDeleteRequest) (*Response, error) { + path := fmt.Sprintf("%s/%s/node_pools/%s/nodes/%s", kubernetesClustersPath, clusterID, poolID, nodeID) + if deleteReq != nil { + v := make(url.Values) + if deleteReq.SkipDrain { + v.Set("skip_drain", "1") + } + if deleteReq.Replace { + v.Set("replace", "1") + } + if query := v.Encode(); query != "" { + path = path + "?" + query + } + } + + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +type kubernetesOptionsRoot struct { + Options *KubernetesOptions `json:"options,omitempty"` + Links *Links `json:"links,omitempty"` +} + +// GetOptions returns options about the Kubernetes service, such as the versions available for +// cluster creation. +func (svc *KubernetesServiceOp) GetOptions(ctx context.Context) (*KubernetesOptions, *Response, error) { + path := kubernetesOptionsPath + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(kubernetesOptionsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Options, resp, nil +} diff --git a/vendor/github.com/digitalocean/godo/links.go b/vendor/github.com/digitalocean/godo/links.go new file mode 100644 index 0000000000..6f350bf9ea --- /dev/null +++ b/vendor/github.com/digitalocean/godo/links.go @@ -0,0 +1,83 @@ +package godo + +import ( + "context" + "net/url" + "strconv" +) + +// Links manages links that are returned along with a List +type Links struct { + Pages *Pages `json:"pages,omitempty"` + Actions []LinkAction `json:"actions,omitempty"` +} + +// Pages are pages specified in Links +type Pages struct { + First string `json:"first,omitempty"` + Prev string `json:"prev,omitempty"` + Last string `json:"last,omitempty"` + Next string `json:"next,omitempty"` +} + +// LinkAction is a pointer to an action +type LinkAction struct { + ID int `json:"id,omitempty"` + Rel string `json:"rel,omitempty"` + HREF string `json:"href,omitempty"` +} + +// CurrentPage is current page of the list +func (l *Links) CurrentPage() (int, error) { + return l.Pages.current() +} + +func (p *Pages) current() (int, error) { + switch { + case p == nil: + return 1, nil + case p.Prev == "" && p.Next != "": + return 1, nil + case p.Prev != "": + prevPage, err := pageForURL(p.Prev) + if err != nil { + return 0, err + } + + return prevPage + 1, nil + } + + return 0, nil +} + +// IsLastPage returns true if the current page is the last +func (l *Links) IsLastPage() bool { + if l.Pages == nil { + return true + } + return l.Pages.isLast() +} + +func (p *Pages) isLast() bool { + return p.Next == "" +} + +func pageForURL(urlText string) (int, error) { + u, err := url.ParseRequestURI(urlText) + if err != nil { + return 0, err + } + + pageStr := u.Query().Get("page") + page, err := strconv.Atoi(pageStr) + if err != nil { + return 0, err + } + + return page, nil +} + +// Get a link action by id. +func (la *LinkAction) Get(ctx context.Context, client *Client) (*Action, *Response, error) { + return client.Actions.Get(ctx, la.ID) +} diff --git a/vendor/github.com/digitalocean/godo/load_balancers.go b/vendor/github.com/digitalocean/godo/load_balancers.go new file mode 100644 index 0000000000..86e9776fe4 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/load_balancers.go @@ -0,0 +1,324 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const loadBalancersBasePath = "/v2/load_balancers" +const forwardingRulesPath = "forwarding_rules" + +const dropletsPath = "droplets" + +// LoadBalancersService is an interface for managing load balancers with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#load-balancers +type LoadBalancersService interface { + Get(context.Context, string) (*LoadBalancer, *Response, error) + List(context.Context, *ListOptions) ([]LoadBalancer, *Response, error) + Create(context.Context, *LoadBalancerRequest) (*LoadBalancer, *Response, error) + Update(ctx context.Context, lbID string, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error) + Delete(ctx context.Context, lbID string) (*Response, error) + AddDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) + RemoveDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) + AddForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) + RemoveForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) +} + +// LoadBalancer represents a DigitalOcean load balancer configuration. +// Tags can only be provided upon the creation of a Load Balancer. +type LoadBalancer struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + IP string `json:"ip,omitempty"` + Algorithm string `json:"algorithm,omitempty"` + Status string `json:"status,omitempty"` + Created string `json:"created_at,omitempty"` + ForwardingRules []ForwardingRule `json:"forwarding_rules,omitempty"` + HealthCheck *HealthCheck `json:"health_check,omitempty"` + StickySessions *StickySessions `json:"sticky_sessions,omitempty"` + Region *Region `json:"region,omitempty"` + DropletIDs []int `json:"droplet_ids,omitempty"` + Tag string `json:"tag,omitempty"` + Tags []string `json:"tags,omitempty"` + RedirectHttpToHttps bool `json:"redirect_http_to_https,omitempty"` + EnableProxyProtocol bool `json:"enable_proxy_protocol,omitempty"` + EnableBackendKeepalive bool `json:"enable_backend_keepalive,omitempty"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +// String creates a human-readable description of a LoadBalancer. +func (l LoadBalancer) String() string { + return Stringify(l) +} + +func (l LoadBalancer) URN() string { + return ToURN("LoadBalancer", l.ID) +} + +// AsRequest creates a LoadBalancerRequest that can be submitted to Update with the current values of the LoadBalancer. +// Modifying the returned LoadBalancerRequest will not modify the original LoadBalancer. +func (l LoadBalancer) AsRequest() *LoadBalancerRequest { + r := LoadBalancerRequest{ + Name: l.Name, + Algorithm: l.Algorithm, + ForwardingRules: append([]ForwardingRule(nil), l.ForwardingRules...), + DropletIDs: append([]int(nil), l.DropletIDs...), + Tag: l.Tag, + RedirectHttpToHttps: l.RedirectHttpToHttps, + EnableProxyProtocol: l.EnableProxyProtocol, + EnableBackendKeepalive: l.EnableBackendKeepalive, + HealthCheck: l.HealthCheck, + VPCUUID: l.VPCUUID, + } + + if l.HealthCheck != nil { + r.HealthCheck = &HealthCheck{} + *r.HealthCheck = *l.HealthCheck + } + if l.StickySessions != nil { + r.StickySessions = &StickySessions{} + *r.StickySessions = *l.StickySessions + } + if l.Region != nil { + r.Region = l.Region.Slug + } + return &r +} + +// ForwardingRule represents load balancer forwarding rules. +type ForwardingRule struct { + EntryProtocol string `json:"entry_protocol,omitempty"` + EntryPort int `json:"entry_port,omitempty"` + TargetProtocol string `json:"target_protocol,omitempty"` + TargetPort int `json:"target_port,omitempty"` + CertificateID string `json:"certificate_id,omitempty"` + TlsPassthrough bool `json:"tls_passthrough,omitempty"` +} + +// String creates a human-readable description of a ForwardingRule. +func (f ForwardingRule) String() string { + return Stringify(f) +} + +// HealthCheck represents optional load balancer health check rules. +type HealthCheck struct { + Protocol string `json:"protocol,omitempty"` + Port int `json:"port,omitempty"` + Path string `json:"path,omitempty"` + CheckIntervalSeconds int `json:"check_interval_seconds,omitempty"` + ResponseTimeoutSeconds int `json:"response_timeout_seconds,omitempty"` + HealthyThreshold int `json:"healthy_threshold,omitempty"` + UnhealthyThreshold int `json:"unhealthy_threshold,omitempty"` +} + +// String creates a human-readable description of a HealthCheck. +func (h HealthCheck) String() string { + return Stringify(h) +} + +// StickySessions represents optional load balancer session affinity rules. +type StickySessions struct { + Type string `json:"type,omitempty"` + CookieName string `json:"cookie_name,omitempty"` + CookieTtlSeconds int `json:"cookie_ttl_seconds,omitempty"` +} + +// String creates a human-readable description of a StickySessions instance. +func (s StickySessions) String() string { + return Stringify(s) +} + +// LoadBalancerRequest represents the configuration to be applied to an existing or a new load balancer. +type LoadBalancerRequest struct { + Name string `json:"name,omitempty"` + Algorithm string `json:"algorithm,omitempty"` + Region string `json:"region,omitempty"` + ForwardingRules []ForwardingRule `json:"forwarding_rules,omitempty"` + HealthCheck *HealthCheck `json:"health_check,omitempty"` + StickySessions *StickySessions `json:"sticky_sessions,omitempty"` + DropletIDs []int `json:"droplet_ids,omitempty"` + Tag string `json:"tag,omitempty"` + Tags []string `json:"tags,omitempty"` + RedirectHttpToHttps bool `json:"redirect_http_to_https,omitempty"` + EnableProxyProtocol bool `json:"enable_proxy_protocol,omitempty"` + EnableBackendKeepalive bool `json:"enable_backend_keepalive,omitempty"` + VPCUUID string `json:"vpc_uuid,omitempty"` +} + +// String creates a human-readable description of a LoadBalancerRequest. +func (l LoadBalancerRequest) String() string { + return Stringify(l) +} + +type forwardingRulesRequest struct { + Rules []ForwardingRule `json:"forwarding_rules,omitempty"` +} + +func (l forwardingRulesRequest) String() string { + return Stringify(l) +} + +type dropletIDsRequest struct { + IDs []int `json:"droplet_ids,omitempty"` +} + +func (l dropletIDsRequest) String() string { + return Stringify(l) +} + +type loadBalancersRoot struct { + LoadBalancers []LoadBalancer `json:"load_balancers"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type loadBalancerRoot struct { + LoadBalancer *LoadBalancer `json:"load_balancer"` +} + +// LoadBalancersServiceOp handles communication with load balancer-related methods of the DigitalOcean API. +type LoadBalancersServiceOp struct { + client *Client +} + +var _ LoadBalancersService = &LoadBalancersServiceOp{} + +// Get an existing load balancer by its identifier. +func (l *LoadBalancersServiceOp) Get(ctx context.Context, lbID string) (*LoadBalancer, *Response, error) { + path := fmt.Sprintf("%s/%s", loadBalancersBasePath, lbID) + + req, err := l.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(loadBalancerRoot) + resp, err := l.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.LoadBalancer, resp, err +} + +// List load balancers, with optional pagination. +func (l *LoadBalancersServiceOp) List(ctx context.Context, opt *ListOptions) ([]LoadBalancer, *Response, error) { + path, err := addOptions(loadBalancersBasePath, opt) + if err != nil { + return nil, nil, err + } + + req, err := l.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(loadBalancersRoot) + resp, err := l.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.LoadBalancers, resp, err +} + +// Create a new load balancer with a given configuration. +func (l *LoadBalancersServiceOp) Create(ctx context.Context, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error) { + req, err := l.client.NewRequest(ctx, http.MethodPost, loadBalancersBasePath, lbr) + if err != nil { + return nil, nil, err + } + + root := new(loadBalancerRoot) + resp, err := l.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.LoadBalancer, resp, err +} + +// Update an existing load balancer with new configuration. +func (l *LoadBalancersServiceOp) Update(ctx context.Context, lbID string, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error) { + path := fmt.Sprintf("%s/%s", loadBalancersBasePath, lbID) + + req, err := l.client.NewRequest(ctx, "PUT", path, lbr) + if err != nil { + return nil, nil, err + } + + root := new(loadBalancerRoot) + resp, err := l.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.LoadBalancer, resp, err +} + +// Delete a load balancer by its identifier. +func (l *LoadBalancersServiceOp) Delete(ctx context.Context, ldID string) (*Response, error) { + path := fmt.Sprintf("%s/%s", loadBalancersBasePath, ldID) + + req, err := l.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} + +// AddDroplets adds droplets to a load balancer. +func (l *LoadBalancersServiceOp) AddDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) { + path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, dropletsPath) + + req, err := l.client.NewRequest(ctx, http.MethodPost, path, &dropletIDsRequest{IDs: dropletIDs}) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} + +// RemoveDroplets removes droplets from a load balancer. +func (l *LoadBalancersServiceOp) RemoveDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) { + path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, dropletsPath) + + req, err := l.client.NewRequest(ctx, http.MethodDelete, path, &dropletIDsRequest{IDs: dropletIDs}) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} + +// AddForwardingRules adds forwarding rules to a load balancer. +func (l *LoadBalancersServiceOp) AddForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) { + path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, forwardingRulesPath) + + req, err := l.client.NewRequest(ctx, http.MethodPost, path, &forwardingRulesRequest{Rules: rules}) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} + +// RemoveForwardingRules removes forwarding rules from a load balancer. +func (l *LoadBalancersServiceOp) RemoveForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) { + path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, forwardingRulesPath) + + req, err := l.client.NewRequest(ctx, http.MethodDelete, path, &forwardingRulesRequest{Rules: rules}) + if err != nil { + return nil, err + } + + return l.client.Do(ctx, req, nil) +} diff --git a/vendor/github.com/digitalocean/godo/meta.go b/vendor/github.com/digitalocean/godo/meta.go new file mode 100644 index 0000000000..d0b7017850 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/meta.go @@ -0,0 +1,6 @@ +package godo + +// Meta describes generic information about a response. +type Meta struct { + Total int `json:"total"` +} diff --git a/vendor/github.com/digitalocean/godo/projects.go b/vendor/github.com/digitalocean/godo/projects.go new file mode 100644 index 0000000000..172c2c9e2e --- /dev/null +++ b/vendor/github.com/digitalocean/godo/projects.go @@ -0,0 +1,310 @@ +package godo + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "path" +) + +const ( + // DefaultProject is the ID you should use if you are working with your + // default project. + DefaultProject = "default" + + projectsBasePath = "/v2/projects" +) + +// ProjectsService is an interface for creating and managing Projects with the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2/#projects +type ProjectsService interface { + List(context.Context, *ListOptions) ([]Project, *Response, error) + GetDefault(context.Context) (*Project, *Response, error) + Get(context.Context, string) (*Project, *Response, error) + Create(context.Context, *CreateProjectRequest) (*Project, *Response, error) + Update(context.Context, string, *UpdateProjectRequest) (*Project, *Response, error) + Delete(context.Context, string) (*Response, error) + + ListResources(context.Context, string, *ListOptions) ([]ProjectResource, *Response, error) + AssignResources(context.Context, string, ...interface{}) ([]ProjectResource, *Response, error) +} + +// ProjectsServiceOp handles communication with Projects methods of the DigitalOcean API. +type ProjectsServiceOp struct { + client *Client +} + +// Project represents a DigitalOcean Project configuration. +type Project struct { + ID string `json:"id"` + OwnerUUID string `json:"owner_uuid"` + OwnerID uint64 `json:"owner_id"` + Name string `json:"name"` + Description string `json:"description"` + Purpose string `json:"purpose"` + Environment string `json:"environment"` + IsDefault bool `json:"is_default"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` +} + +// String creates a human-readable description of a Project. +func (p Project) String() string { + return Stringify(p) +} + +// CreateProjectRequest represents the request to create a new project. +type CreateProjectRequest struct { + Name string `json:"name"` + Description string `json:"description"` + Purpose string `json:"purpose"` + Environment string `json:"environment"` +} + +// UpdateProjectRequest represents the request to update project information. +// This type expects certain attribute types, but is built this way to allow +// nil values as well. See `updateProjectRequest` for the "real" types. +type UpdateProjectRequest struct { + Name interface{} + Description interface{} + Purpose interface{} + Environment interface{} + IsDefault interface{} +} + +type updateProjectRequest struct { + Name *string `json:"name"` + Description *string `json:"description"` + Purpose *string `json:"purpose"` + Environment *string `json:"environment"` + IsDefault *bool `json:"is_default"` +} + +// MarshalJSON takes an UpdateRequest and converts it to the "typed" request +// which is sent to the projects API. This is a PATCH request, which allows +// partial attributes, so `null` values are OK. +func (upr *UpdateProjectRequest) MarshalJSON() ([]byte, error) { + d := &updateProjectRequest{} + if str, ok := upr.Name.(string); ok { + d.Name = &str + } + if str, ok := upr.Description.(string); ok { + d.Description = &str + } + if str, ok := upr.Purpose.(string); ok { + d.Purpose = &str + } + if str, ok := upr.Environment.(string); ok { + d.Environment = &str + } + if val, ok := upr.IsDefault.(bool); ok { + d.IsDefault = &val + } + + return json.Marshal(d) +} + +type assignResourcesRequest struct { + Resources []string `json:"resources"` +} + +// ProjectResource is the projects API's representation of a resource. +type ProjectResource struct { + URN string `json:"urn"` + AssignedAt string `json:"assigned_at"` + Links *ProjectResourceLinks `json:"links"` + Status string `json:"status,omitempty"` +} + +// ProjetResourceLinks specify the link for more information about the resource. +type ProjectResourceLinks struct { + Self string `json:"self"` +} + +type projectsRoot struct { + Projects []Project `json:"projects"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type projectRoot struct { + Project *Project `json:"project"` +} + +type projectResourcesRoot struct { + Resources []ProjectResource `json:"resources"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta"` +} + +var _ ProjectsService = &ProjectsServiceOp{} + +// List Projects. +func (p *ProjectsServiceOp) List(ctx context.Context, opts *ListOptions) ([]Project, *Response, error) { + path, err := addOptions(projectsBasePath, opts) + if err != nil { + return nil, nil, err + } + + req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(projectsRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Projects, resp, err +} + +// GetDefault project. +func (p *ProjectsServiceOp) GetDefault(ctx context.Context) (*Project, *Response, error) { + return p.getHelper(ctx, "default") +} + +// Get retrieves a single project by its ID. +func (p *ProjectsServiceOp) Get(ctx context.Context, projectID string) (*Project, *Response, error) { + return p.getHelper(ctx, projectID) +} + +// Create a new project. +func (p *ProjectsServiceOp) Create(ctx context.Context, cr *CreateProjectRequest) (*Project, *Response, error) { + req, err := p.client.NewRequest(ctx, http.MethodPost, projectsBasePath, cr) + if err != nil { + return nil, nil, err + } + + root := new(projectRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Project, resp, err +} + +// Update an existing project. +func (p *ProjectsServiceOp) Update(ctx context.Context, projectID string, ur *UpdateProjectRequest) (*Project, *Response, error) { + path := path.Join(projectsBasePath, projectID) + req, err := p.client.NewRequest(ctx, http.MethodPatch, path, ur) + if err != nil { + return nil, nil, err + } + + root := new(projectRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Project, resp, err +} + +// Delete an existing project. You cannot have any resources in a project +// before deleting it. See the API documentation for more details. +func (p *ProjectsServiceOp) Delete(ctx context.Context, projectID string) (*Response, error) { + path := path.Join(projectsBasePath, projectID) + req, err := p.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + return p.client.Do(ctx, req, nil) +} + +// ListResources lists all resources in a project. +func (p *ProjectsServiceOp) ListResources(ctx context.Context, projectID string, opts *ListOptions) ([]ProjectResource, *Response, error) { + basePath := path.Join(projectsBasePath, projectID, "resources") + path, err := addOptions(basePath, opts) + if err != nil { + return nil, nil, err + } + + req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(projectResourcesRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Resources, resp, err +} + +// AssignResources assigns one or more resources to a project. AssignResources +// accepts resources in two possible formats: + +// 1. The resource type, like `&Droplet{ID: 1}` or `&FloatingIP{IP: "1.2.3.4"}` +// 2. A valid DO URN as a string, like "do:droplet:1234" +// +// There is no unassign. To move a resource to another project, just assign +// it to that other project. +func (p *ProjectsServiceOp) AssignResources(ctx context.Context, projectID string, resources ...interface{}) ([]ProjectResource, *Response, error) { + path := path.Join(projectsBasePath, projectID, "resources") + + ar := &assignResourcesRequest{ + Resources: make([]string, len(resources)), + } + + for i, resource := range resources { + switch resource := resource.(type) { + case ResourceWithURN: + ar.Resources[i] = resource.URN() + case string: + ar.Resources[i] = resource + default: + return nil, nil, fmt.Errorf("%T must either be a string or have a valid URN method", resource) + } + } + req, err := p.client.NewRequest(ctx, http.MethodPost, path, ar) + if err != nil { + return nil, nil, err + } + + root := new(projectResourcesRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Resources, resp, err +} + +func (p *ProjectsServiceOp) getHelper(ctx context.Context, projectID string) (*Project, *Response, error) { + path := path.Join(projectsBasePath, projectID) + + req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(projectRoot) + resp, err := p.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Project, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/regions.go b/vendor/github.com/digitalocean/godo/regions.go new file mode 100644 index 0000000000..b07175e8a9 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/regions.go @@ -0,0 +1,68 @@ +package godo + +import ( + "context" + "net/http" +) + +// RegionsService is an interface for interfacing with the regions +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#regions +type RegionsService interface { + List(context.Context, *ListOptions) ([]Region, *Response, error) +} + +// RegionsServiceOp handles communication with the region related methods of the +// DigitalOcean API. +type RegionsServiceOp struct { + client *Client +} + +var _ RegionsService = &RegionsServiceOp{} + +// Region represents a DigitalOcean Region +type Region struct { + Slug string `json:"slug,omitempty"` + Name string `json:"name,omitempty"` + Sizes []string `json:"sizes,omitempty"` + Available bool `json:"available,omitempty"` + Features []string `json:"features,omitempty"` +} + +type regionsRoot struct { + Regions []Region + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +func (r Region) String() string { + return Stringify(r) +} + +// List all regions +func (s *RegionsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Region, *Response, error) { + path := "v2/regions" + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(regionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Regions, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/registry.go b/vendor/github.com/digitalocean/godo/registry.go new file mode 100644 index 0000000000..1b5c40bd3d --- /dev/null +++ b/vendor/github.com/digitalocean/godo/registry.go @@ -0,0 +1,253 @@ +package godo + +import ( + "bytes" + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "time" +) + +const ( + registryPath = "/v2/registry" + // RegistryServer is the hostname of the DigitalOcean registry service + RegistryServer = "registry.digitalocean.com" +) + +// RegistryService is an interface for interfacing with the Registry endpoints +// of the DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#registry +type RegistryService interface { + Create(context.Context, *RegistryCreateRequest) (*Registry, *Response, error) + Get(context.Context) (*Registry, *Response, error) + Delete(context.Context) (*Response, error) + DockerCredentials(context.Context, *RegistryDockerCredentialsRequest) (*DockerCredentials, *Response, error) + ListRepositories(context.Context, string, *ListOptions) ([]*Repository, *Response, error) + ListRepositoryTags(context.Context, string, string, *ListOptions) ([]*RepositoryTag, *Response, error) + DeleteTag(context.Context, string, string, string) (*Response, error) + DeleteManifest(context.Context, string, string, string) (*Response, error) +} + +var _ RegistryService = &RegistryServiceOp{} + +// RegistryServiceOp handles communication with Registry methods of the DigitalOcean API. +type RegistryServiceOp struct { + client *Client +} + +// RegistryCreateRequest represents a request to create a registry. +type RegistryCreateRequest struct { + Name string `json:"name,omitempty"` +} + +// RegistryDockerCredentialsRequest represents a request to retrieve docker +// credentials for a registry. +type RegistryDockerCredentialsRequest struct { + ReadWrite bool `json:"read_write"` + ExpirySeconds *int `json:"expiry_seconds,omitempty"` +} + +// Registry represents a registry. +type Registry struct { + Name string `json:"name,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` +} + +// Repository represents a repository +type Repository struct { + RegistryName string `json:"registry_name,omitempty"` + Name string `json:"name,omitempty"` + LatestTag *RepositoryTag `json:"latest_tag,omitempty"` + TagCount uint64 `json:"tag_count,omitempty"` +} + +// RepositoryTag represents a repository tag +type RepositoryTag struct { + RegistryName string `json:"registry_name,omitempty"` + Repository string `json:"repository,omitempty"` + Tag string `json:"tag,omitempty"` + ManifestDigest string `json:"manifest_digest,omitempty"` + CompressedSizeBytes uint64 `json:"compressed_size_bytes,omitempty"` + SizeBytes uint64 `json:"size_bytes,omitempty"` + UpdatedAt time.Time `json:"updated_at,omitempty"` +} + +type registryRoot struct { + Registry *Registry `json:"registry,omitempty"` +} + +type repositoriesRoot struct { + Repositories []*Repository `json:"repositories,omitempty"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta"` +} + +type repositoryTagsRoot struct { + Tags []*RepositoryTag `json:"tags,omitempty"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta"` +} + +// Get retrieves the details of a Registry. +func (svc *RegistryServiceOp) Get(ctx context.Context) (*Registry, *Response, error) { + req, err := svc.client.NewRequest(ctx, http.MethodGet, registryPath, nil) + if err != nil { + return nil, nil, err + } + root := new(registryRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Registry, resp, nil +} + +// Create creates a registry. +func (svc *RegistryServiceOp) Create(ctx context.Context, create *RegistryCreateRequest) (*Registry, *Response, error) { + req, err := svc.client.NewRequest(ctx, http.MethodPost, registryPath, create) + if err != nil { + return nil, nil, err + } + root := new(registryRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Registry, resp, nil +} + +// Delete deletes a registry. There is no way to recover a registry once it has +// been destroyed. +func (svc *RegistryServiceOp) Delete(ctx context.Context) (*Response, error) { + req, err := svc.client.NewRequest(ctx, http.MethodDelete, registryPath, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + return resp, nil +} + +// DockerCredentials is the content of a Docker config file +// that is used by the docker CLI +// See: https://docs.docker.com/engine/reference/commandline/cli/#configjson-properties +type DockerCredentials struct { + DockerConfigJSON []byte +} + +// DockerCredentials retrieves a Docker config file containing the registry's credentials. +func (svc *RegistryServiceOp) DockerCredentials(ctx context.Context, request *RegistryDockerCredentialsRequest) (*DockerCredentials, *Response, error) { + path := fmt.Sprintf("%s/%s", registryPath, "docker-credentials") + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + q := req.URL.Query() + q.Add("read_write", strconv.FormatBool(request.ReadWrite)) + if request.ExpirySeconds != nil { + q.Add("expiry_seconds", strconv.Itoa(*request.ExpirySeconds)) + } + req.URL.RawQuery = q.Encode() + + var buf bytes.Buffer + resp, err := svc.client.Do(ctx, req, &buf) + if err != nil { + return nil, resp, err + } + + dc := &DockerCredentials{ + DockerConfigJSON: buf.Bytes(), + } + return dc, resp, nil +} + +// ListRepositories returns a list of the Repositories visible with the registry's credentials. +func (svc *RegistryServiceOp) ListRepositories(ctx context.Context, registry string, opts *ListOptions) ([]*Repository, *Response, error) { + path := fmt.Sprintf("%s/%s/repositories", registryPath, registry) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(repositoriesRoot) + + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Repositories, resp, nil +} + +// ListRepositoryTags returns a list of the RepositoryTags available within the given repository. +func (svc *RegistryServiceOp) ListRepositoryTags(ctx context.Context, registry, repository string, opts *ListOptions) ([]*RepositoryTag, *Response, error) { + path := fmt.Sprintf("%s/%s/repositories/%s/tags", registryPath, registry, url.PathEscape(repository)) + path, err := addOptions(path, opts) + if err != nil { + return nil, nil, err + } + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + root := new(repositoryTagsRoot) + + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Tags, resp, nil +} + +// DeleteTag deletes a tag within a given repository. +func (svc *RegistryServiceOp) DeleteTag(ctx context.Context, registry, repository, tag string) (*Response, error) { + path := fmt.Sprintf("%s/%s/repositories/%s/tags/%s", registryPath, registry, url.PathEscape(repository), tag) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + + return resp, nil +} + +// DeleteManifest deletes a manifest by its digest within a given repository. +func (svc *RegistryServiceOp) DeleteManifest(ctx context.Context, registry, repository, digest string) (*Response, error) { + path := fmt.Sprintf("%s/%s/repositories/%s/digests/%s", registryPath, registry, url.PathEscape(repository), digest) + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + resp, err := svc.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + + return resp, nil +} diff --git a/vendor/github.com/digitalocean/godo/sizes.go b/vendor/github.com/digitalocean/godo/sizes.go new file mode 100644 index 0000000000..d2b93ea7fb --- /dev/null +++ b/vendor/github.com/digitalocean/godo/sizes.go @@ -0,0 +1,72 @@ +package godo + +import ( + "context" + "net/http" +) + +// SizesService is an interface for interfacing with the size +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#sizes +type SizesService interface { + List(context.Context, *ListOptions) ([]Size, *Response, error) +} + +// SizesServiceOp handles communication with the size related methods of the +// DigitalOcean API. +type SizesServiceOp struct { + client *Client +} + +var _ SizesService = &SizesServiceOp{} + +// Size represents a DigitalOcean Size +type Size struct { + Slug string `json:"slug,omitempty"` + Memory int `json:"memory,omitempty"` + Vcpus int `json:"vcpus,omitempty"` + Disk int `json:"disk,omitempty"` + PriceMonthly float64 `json:"price_monthly,omitempty"` + PriceHourly float64 `json:"price_hourly,omitempty"` + Regions []string `json:"regions,omitempty"` + Available bool `json:"available,omitempty"` + Transfer float64 `json:"transfer,omitempty"` +} + +func (s Size) String() string { + return Stringify(s) +} + +type sizesRoot struct { + Sizes []Size + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// List all images +func (s *SizesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Size, *Response, error) { + path := "v2/sizes" + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(sizesRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Sizes, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/snapshots.go b/vendor/github.com/digitalocean/godo/snapshots.go new file mode 100644 index 0000000000..cf95ccc007 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/snapshots.go @@ -0,0 +1,142 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const snapshotBasePath = "v2/snapshots" + +// SnapshotsService is an interface for interfacing with the snapshots +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#snapshots +type SnapshotsService interface { + List(context.Context, *ListOptions) ([]Snapshot, *Response, error) + ListVolume(context.Context, *ListOptions) ([]Snapshot, *Response, error) + ListDroplet(context.Context, *ListOptions) ([]Snapshot, *Response, error) + Get(context.Context, string) (*Snapshot, *Response, error) + Delete(context.Context, string) (*Response, error) +} + +// SnapshotsServiceOp handles communication with the snapshot related methods of the +// DigitalOcean API. +type SnapshotsServiceOp struct { + client *Client +} + +var _ SnapshotsService = &SnapshotsServiceOp{} + +// Snapshot represents a DigitalOcean Snapshot +type Snapshot struct { + ID string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + ResourceID string `json:"resource_id,omitempty"` + ResourceType string `json:"resource_type,omitempty"` + Regions []string `json:"regions,omitempty"` + MinDiskSize int `json:"min_disk_size,omitempty"` + SizeGigaBytes float64 `json:"size_gigabytes,omitempty"` + Created string `json:"created_at,omitempty"` + Tags []string `json:"tags,omitempty"` +} + +type snapshotRoot struct { + Snapshot *Snapshot `json:"snapshot"` +} + +type snapshotsRoot struct { + Snapshots []Snapshot `json:"snapshots"` + Links *Links `json:"links,omitempty"` + Meta *Meta `json:"meta,omitempty"` +} + +type listSnapshotOptions struct { + ResourceType string `url:"resource_type,omitempty"` +} + +func (s Snapshot) String() string { + return Stringify(s) +} + +// List lists all the snapshots available. +func (s *SnapshotsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) { + return s.list(ctx, opt, nil) +} + +// ListDroplet lists all the Droplet snapshots. +func (s *SnapshotsServiceOp) ListDroplet(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) { + listOpt := listSnapshotOptions{ResourceType: "droplet"} + return s.list(ctx, opt, &listOpt) +} + +// ListVolume lists all the volume snapshots. +func (s *SnapshotsServiceOp) ListVolume(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) { + listOpt := listSnapshotOptions{ResourceType: "volume"} + return s.list(ctx, opt, &listOpt) +} + +// Get retrieves an snapshot by id. +func (s *SnapshotsServiceOp) Get(ctx context.Context, snapshotID string) (*Snapshot, *Response, error) { + return s.get(ctx, snapshotID) +} + +// Delete an snapshot. +func (s *SnapshotsServiceOp) Delete(ctx context.Context, snapshotID string) (*Response, error) { + path := fmt.Sprintf("%s/%s", snapshotBasePath, snapshotID) + + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// Helper method for getting an individual snapshot +func (s *SnapshotsServiceOp) get(ctx context.Context, ID string) (*Snapshot, *Response, error) { + path := fmt.Sprintf("%s/%s", snapshotBasePath, ID) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(snapshotRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Snapshot, resp, err +} + +// Helper method for listing snapshots +func (s *SnapshotsServiceOp) list(ctx context.Context, opt *ListOptions, listOpt *listSnapshotOptions) ([]Snapshot, *Response, error) { + path := snapshotBasePath + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + path, err = addOptions(path, listOpt) + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(snapshotsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + + return root.Snapshots, resp, err +} diff --git a/vendor/github.com/digitalocean/godo/storage.go b/vendor/github.com/digitalocean/godo/storage.go new file mode 100644 index 0000000000..e1dda598a5 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/storage.go @@ -0,0 +1,261 @@ +package godo + +import ( + "context" + "fmt" + "net/http" + "time" +) + +const ( + storageBasePath = "v2" + storageAllocPath = storageBasePath + "/volumes" + storageSnapPath = storageBasePath + "/snapshots" +) + +// StorageService is an interface for interfacing with the storage +// endpoints of the Digital Ocean API. +// See: https://developers.digitalocean.com/documentation/v2/#block-storage +type StorageService interface { + ListVolumes(context.Context, *ListVolumeParams) ([]Volume, *Response, error) + GetVolume(context.Context, string) (*Volume, *Response, error) + CreateVolume(context.Context, *VolumeCreateRequest) (*Volume, *Response, error) + DeleteVolume(context.Context, string) (*Response, error) + ListSnapshots(ctx context.Context, volumeID string, opts *ListOptions) ([]Snapshot, *Response, error) + GetSnapshot(context.Context, string) (*Snapshot, *Response, error) + CreateSnapshot(context.Context, *SnapshotCreateRequest) (*Snapshot, *Response, error) + DeleteSnapshot(context.Context, string) (*Response, error) +} + +// StorageServiceOp handles communication with the storage volumes related methods of the +// DigitalOcean API. +type StorageServiceOp struct { + client *Client +} + +// ListVolumeParams stores the options you can set for a ListVolumeCall +type ListVolumeParams struct { + Region string `json:"region"` + Name string `json:"name"` + ListOptions *ListOptions `json:"list_options,omitempty"` +} + +var _ StorageService = &StorageServiceOp{} + +// Volume represents a Digital Ocean block store volume. +type Volume struct { + ID string `json:"id"` + Region *Region `json:"region"` + Name string `json:"name"` + SizeGigaBytes int64 `json:"size_gigabytes"` + Description string `json:"description"` + DropletIDs []int `json:"droplet_ids"` + CreatedAt time.Time `json:"created_at"` + FilesystemType string `json:"filesystem_type"` + FilesystemLabel string `json:"filesystem_label"` + Tags []string `json:"tags"` +} + +func (f Volume) String() string { + return Stringify(f) +} + +func (f Volume) URN() string { + return ToURN("Volume", f.ID) +} + +type storageVolumesRoot struct { + Volumes []Volume `json:"volumes"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type storageVolumeRoot struct { + Volume *Volume `json:"volume"` + Links *Links `json:"links,omitempty"` +} + +// VolumeCreateRequest represents a request to create a block store +// volume. +type VolumeCreateRequest struct { + Region string `json:"region"` + Name string `json:"name"` + Description string `json:"description"` + SizeGigaBytes int64 `json:"size_gigabytes"` + SnapshotID string `json:"snapshot_id"` + FilesystemType string `json:"filesystem_type"` + FilesystemLabel string `json:"filesystem_label"` + Tags []string `json:"tags"` +} + +// ListVolumes lists all storage volumes. +func (svc *StorageServiceOp) ListVolumes(ctx context.Context, params *ListVolumeParams) ([]Volume, *Response, error) { + path := storageAllocPath + if params != nil { + if params.Region != "" && params.Name != "" { + path = fmt.Sprintf("%s?name=%s®ion=%s", path, params.Name, params.Region) + } else if params.Region != "" { + path = fmt.Sprintf("%s?region=%s", path, params.Region) + } else if params.Name != "" { + path = fmt.Sprintf("%s?name=%s", path, params.Name) + } + + if params.ListOptions != nil { + var err error + path, err = addOptions(path, params.ListOptions) + if err != nil { + return nil, nil, err + } + } + } + + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(storageVolumesRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Volumes, resp, nil +} + +// CreateVolume creates a storage volume. The name must be unique. +func (svc *StorageServiceOp) CreateVolume(ctx context.Context, createRequest *VolumeCreateRequest) (*Volume, *Response, error) { + path := storageAllocPath + + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(storageVolumeRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Volume, resp, nil +} + +// GetVolume retrieves an individual storage volume. +func (svc *StorageServiceOp) GetVolume(ctx context.Context, id string) (*Volume, *Response, error) { + path := fmt.Sprintf("%s/%s", storageAllocPath, id) + + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(storageVolumeRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Volume, resp, nil +} + +// DeleteVolume deletes a storage volume. +func (svc *StorageServiceOp) DeleteVolume(ctx context.Context, id string) (*Response, error) { + path := fmt.Sprintf("%s/%s", storageAllocPath, id) + + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + return svc.client.Do(ctx, req, nil) +} + +// SnapshotCreateRequest represents a request to create a block store +// volume. +type SnapshotCreateRequest struct { + VolumeID string `json:"volume_id"` + Name string `json:"name"` + Description string `json:"description"` + Tags []string `json:"tags"` +} + +// ListSnapshots lists all snapshots related to a storage volume. +func (svc *StorageServiceOp) ListSnapshots(ctx context.Context, volumeID string, opt *ListOptions) ([]Snapshot, *Response, error) { + path := fmt.Sprintf("%s/%s/snapshots", storageAllocPath, volumeID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(snapshotsRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Snapshots, resp, nil +} + +// CreateSnapshot creates a snapshot of a storage volume. +func (svc *StorageServiceOp) CreateSnapshot(ctx context.Context, createRequest *SnapshotCreateRequest) (*Snapshot, *Response, error) { + path := fmt.Sprintf("%s/%s/snapshots", storageAllocPath, createRequest.VolumeID) + + req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(snapshotRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + return root.Snapshot, resp, nil +} + +// GetSnapshot retrieves an individual snapshot. +func (svc *StorageServiceOp) GetSnapshot(ctx context.Context, id string) (*Snapshot, *Response, error) { + path := fmt.Sprintf("%s/%s", storageSnapPath, id) + + req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(snapshotRoot) + resp, err := svc.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Snapshot, resp, nil +} + +// DeleteSnapshot deletes a snapshot. +func (svc *StorageServiceOp) DeleteSnapshot(ctx context.Context, id string) (*Response, error) { + path := fmt.Sprintf("%s/%s", storageSnapPath, id) + + req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + return svc.client.Do(ctx, req, nil) +} diff --git a/vendor/github.com/digitalocean/godo/storage_actions.go b/vendor/github.com/digitalocean/godo/storage_actions.go new file mode 100644 index 0000000000..234aba9064 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/storage_actions.go @@ -0,0 +1,132 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +// StorageActionsService is an interface for interfacing with the +// storage actions endpoints of the Digital Ocean API. +// See: https://developers.digitalocean.com/documentation/v2#storage-actions +type StorageActionsService interface { + Attach(ctx context.Context, volumeID string, dropletID int) (*Action, *Response, error) + DetachByDropletID(ctx context.Context, volumeID string, dropletID int) (*Action, *Response, error) + Get(ctx context.Context, volumeID string, actionID int) (*Action, *Response, error) + List(ctx context.Context, volumeID string, opt *ListOptions) ([]Action, *Response, error) + Resize(ctx context.Context, volumeID string, sizeGigabytes int, regionSlug string) (*Action, *Response, error) +} + +// StorageActionsServiceOp handles communication with the storage volumes +// action related methods of the DigitalOcean API. +type StorageActionsServiceOp struct { + client *Client +} + +// StorageAttachment represents the attachement of a block storage +// volume to a specific Droplet under the device name. +type StorageAttachment struct { + DropletID int `json:"droplet_id"` +} + +// Attach a storage volume to a Droplet. +func (s *StorageActionsServiceOp) Attach(ctx context.Context, volumeID string, dropletID int) (*Action, *Response, error) { + request := &ActionRequest{ + "type": "attach", + "droplet_id": dropletID, + } + return s.doAction(ctx, volumeID, request) +} + +// DetachByDropletID a storage volume from a Droplet by Droplet ID. +func (s *StorageActionsServiceOp) DetachByDropletID(ctx context.Context, volumeID string, dropletID int) (*Action, *Response, error) { + request := &ActionRequest{ + "type": "detach", + "droplet_id": dropletID, + } + return s.doAction(ctx, volumeID, request) +} + +// Get an action for a particular storage volume by id. +func (s *StorageActionsServiceOp) Get(ctx context.Context, volumeID string, actionID int) (*Action, *Response, error) { + path := fmt.Sprintf("%s/%d", storageAllocationActionPath(volumeID), actionID) + return s.get(ctx, path) +} + +// List the actions for a particular storage volume. +func (s *StorageActionsServiceOp) List(ctx context.Context, volumeID string, opt *ListOptions) ([]Action, *Response, error) { + path := storageAllocationActionPath(volumeID) + path, err := addOptions(path, opt) + if err != nil { + return nil, nil, err + } + + return s.list(ctx, path) +} + +// Resize a storage volume. +func (s *StorageActionsServiceOp) Resize(ctx context.Context, volumeID string, sizeGigabytes int, regionSlug string) (*Action, *Response, error) { + request := &ActionRequest{ + "type": "resize", + "size_gigabytes": sizeGigabytes, + "region": regionSlug, + } + return s.doAction(ctx, volumeID, request) +} + +func (s *StorageActionsServiceOp) doAction(ctx context.Context, volumeID string, request *ActionRequest) (*Action, *Response, error) { + path := storageAllocationActionPath(volumeID) + + req, err := s.client.NewRequest(ctx, http.MethodPost, path, request) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *StorageActionsServiceOp) get(ctx context.Context, path string) (*Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Event, resp, err +} + +func (s *StorageActionsServiceOp) list(ctx context.Context, path string) ([]Action, *Response, error) { + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(actionsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Actions, resp, err +} + +func storageAllocationActionPath(volumeID string) string { + return fmt.Sprintf("%s/%s/actions", storageAllocPath, volumeID) +} diff --git a/vendor/github.com/digitalocean/godo/strings.go b/vendor/github.com/digitalocean/godo/strings.go new file mode 100644 index 0000000000..4d5c0ad220 --- /dev/null +++ b/vendor/github.com/digitalocean/godo/strings.go @@ -0,0 +1,102 @@ +package godo + +import ( + "bytes" + "fmt" + "io" + "reflect" + "strings" +) + +var timestampType = reflect.TypeOf(Timestamp{}) + +type ResourceWithURN interface { + URN() string +} + +// ToURN converts the resource type and ID to a valid DO API URN. +func ToURN(resourceType string, id interface{}) string { + return fmt.Sprintf("%s:%s:%v", "do", strings.ToLower(resourceType), id) +} + +// Stringify attempts to create a string representation of DigitalOcean types +func Stringify(message interface{}) string { + var buf bytes.Buffer + v := reflect.ValueOf(message) + stringifyValue(&buf, v) + return buf.String() +} + +// stringifyValue was graciously cargoculted from the goprotubuf library +func stringifyValue(w io.Writer, val reflect.Value) { + if val.Kind() == reflect.Ptr && val.IsNil() { + _, _ = w.Write([]byte("")) + return + } + + v := reflect.Indirect(val) + + switch v.Kind() { + case reflect.String: + fmt.Fprintf(w, `"%s"`, v) + case reflect.Slice: + stringifySlice(w, v) + return + case reflect.Struct: + stringifyStruct(w, v) + default: + if v.CanInterface() { + fmt.Fprint(w, v.Interface()) + } + } +} + +func stringifySlice(w io.Writer, v reflect.Value) { + _, _ = w.Write([]byte{'['}) + for i := 0; i < v.Len(); i++ { + if i > 0 { + _, _ = w.Write([]byte{' '}) + } + + stringifyValue(w, v.Index(i)) + } + + _, _ = w.Write([]byte{']'}) +} + +func stringifyStruct(w io.Writer, v reflect.Value) { + if v.Type().Name() != "" { + _, _ = w.Write([]byte(v.Type().String())) + } + + // special handling of Timestamp values + if v.Type() == timestampType { + fmt.Fprintf(w, "{%s}", v.Interface()) + return + } + + _, _ = w.Write([]byte{'{'}) + + var sep bool + for i := 0; i < v.NumField(); i++ { + fv := v.Field(i) + if fv.Kind() == reflect.Ptr && fv.IsNil() { + continue + } + if fv.Kind() == reflect.Slice && fv.IsNil() { + continue + } + + if sep { + _, _ = w.Write([]byte(", ")) + } else { + sep = true + } + + _, _ = w.Write([]byte(v.Type().Field(i).Name)) + _, _ = w.Write([]byte{':'}) + stringifyValue(w, fv) + } + + _, _ = w.Write([]byte{'}'}) +} diff --git a/vendor/github.com/digitalocean/godo/tags.go b/vendor/github.com/digitalocean/godo/tags.go new file mode 100644 index 0000000000..6301e15f1c --- /dev/null +++ b/vendor/github.com/digitalocean/godo/tags.go @@ -0,0 +1,247 @@ +package godo + +import ( + "context" + "fmt" + "net/http" +) + +const tagsBasePath = "v2/tags" + +// TagsService is an interface for interfacing with the tags +// endpoints of the DigitalOcean API +// See: https://developers.digitalocean.com/documentation/v2#tags +type TagsService interface { + List(context.Context, *ListOptions) ([]Tag, *Response, error) + Get(context.Context, string) (*Tag, *Response, error) + Create(context.Context, *TagCreateRequest) (*Tag, *Response, error) + Delete(context.Context, string) (*Response, error) + + TagResources(context.Context, string, *TagResourcesRequest) (*Response, error) + UntagResources(context.Context, string, *UntagResourcesRequest) (*Response, error) +} + +// TagsServiceOp handles communication with tag related method of the +// DigitalOcean API. +type TagsServiceOp struct { + client *Client +} + +var _ TagsService = &TagsServiceOp{} + +// ResourceType represents a class of resource, currently only droplet are supported +type ResourceType string + +const ( + // DropletResourceType holds the string representing our ResourceType of Droplet. + DropletResourceType ResourceType = "droplet" + // ImageResourceType holds the string representing our ResourceType of Image. + ImageResourceType ResourceType = "image" + // VolumeResourceType holds the string representing our ResourceType of Volume. + VolumeResourceType ResourceType = "volume" + // LoadBalancerResourceType holds the string representing our ResourceType of LoadBalancer. + LoadBalancerResourceType ResourceType = "load_balancer" + // VolumeSnapshotResourceType holds the string representing our ResourceType for storage Snapshots. + VolumeSnapshotResourceType ResourceType = "volume_snapshot" + // DatabaseResourceType holds the string representing our ResourceType of Database. + DatabaseResourceType ResourceType = "database" +) + +// Resource represent a single resource for associating/disassociating with tags +type Resource struct { + ID string `json:"resource_id,omitempty"` + Type ResourceType `json:"resource_type,omitempty"` +} + +// TaggedResources represent the set of resources a tag is attached to +type TaggedResources struct { + Count int `json:"count"` + LastTaggedURI string `json:"last_tagged_uri,omitempty"` + Droplets *TaggedDropletsResources `json:"droplets,omitempty"` + Images *TaggedImagesResources `json:"images"` + Volumes *TaggedVolumesResources `json:"volumes"` + VolumeSnapshots *TaggedVolumeSnapshotsResources `json:"volume_snapshots"` + Databases *TaggedDatabasesResources `json:"databases"` +} + +// TaggedDropletsResources represent the droplet resources a tag is attached to +type TaggedDropletsResources struct { + Count int `json:"count,float64,omitempty"` + LastTagged *Droplet `json:"last_tagged,omitempty"` + LastTaggedURI string `json:"last_tagged_uri,omitempty"` +} + +// TaggedResourcesData represent the generic resources a tag is attached to +type TaggedResourcesData struct { + Count int `json:"count,float64,omitempty"` + LastTaggedURI string `json:"last_tagged_uri,omitempty"` +} + +// TaggedImagesResources represent the image resources a tag is attached to +type TaggedImagesResources TaggedResourcesData + +// TaggedVolumesResources represent the volume resources a tag is attached to +type TaggedVolumesResources TaggedResourcesData + +// TaggedVolumeSnapshotsResources represent the volume snapshot resources a tag is attached to +type TaggedVolumeSnapshotsResources TaggedResourcesData + +// TaggedDatabasesResources represent the database resources a tag is attached to +type TaggedDatabasesResources TaggedResourcesData + +// Tag represent DigitalOcean tag +type Tag struct { + Name string `json:"name,omitempty"` + Resources *TaggedResources `json:"resources,omitempty"` +} + +//TagCreateRequest represents the JSON structure of a request of that type. +type TagCreateRequest struct { + Name string `json:"name"` +} + +// TagResourcesRequest represents the JSON structure of a request of that type. +type TagResourcesRequest struct { + Resources []Resource `json:"resources"` +} + +// UntagResourcesRequest represents the JSON structure of a request of that type. +type UntagResourcesRequest struct { + Resources []Resource `json:"resources"` +} + +type tagsRoot struct { + Tags []Tag `json:"tags"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +type tagRoot struct { + Tag *Tag `json:"tag"` +} + +// List all tags +func (s *TagsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Tag, *Response, error) { + path := tagsBasePath + path, err := addOptions(path, opt) + + if err != nil { + return nil, nil, err + } + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(tagsRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.Tags, resp, err +} + +// Get a single tag +func (s *TagsServiceOp) Get(ctx context.Context, name string) (*Tag, *Response, error) { + path := fmt.Sprintf("%s/%s", tagsBasePath, name) + + req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(tagRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Tag, resp, err +} + +// Create a new tag +func (s *TagsServiceOp) Create(ctx context.Context, createRequest *TagCreateRequest) (*Tag, *Response, error) { + if createRequest == nil { + return nil, nil, NewArgError("createRequest", "cannot be nil") + } + + req, err := s.client.NewRequest(ctx, http.MethodPost, tagsBasePath, createRequest) + if err != nil { + return nil, nil, err + } + + root := new(tagRoot) + resp, err := s.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.Tag, resp, err +} + +// Delete an existing tag +func (s *TagsServiceOp) Delete(ctx context.Context, name string) (*Response, error) { + if name == "" { + return nil, NewArgError("name", "cannot be empty") + } + + path := fmt.Sprintf("%s/%s", tagsBasePath, name) + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// TagResources associates resources with a given Tag. +func (s *TagsServiceOp) TagResources(ctx context.Context, name string, tagRequest *TagResourcesRequest) (*Response, error) { + if name == "" { + return nil, NewArgError("name", "cannot be empty") + } + + if tagRequest == nil { + return nil, NewArgError("tagRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s/resources", tagsBasePath, name) + req, err := s.client.NewRequest(ctx, http.MethodPost, path, tagRequest) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} + +// UntagResources dissociates resources with a given Tag. +func (s *TagsServiceOp) UntagResources(ctx context.Context, name string, untagRequest *UntagResourcesRequest) (*Response, error) { + if name == "" { + return nil, NewArgError("name", "cannot be empty") + } + + if untagRequest == nil { + return nil, NewArgError("tagRequest", "cannot be nil") + } + + path := fmt.Sprintf("%s/%s/resources", tagsBasePath, name) + req, err := s.client.NewRequest(ctx, http.MethodDelete, path, untagRequest) + if err != nil { + return nil, err + } + + resp, err := s.client.Do(ctx, req, nil) + + return resp, err +} diff --git a/vendor/github.com/digitalocean/godo/timestamp.go b/vendor/github.com/digitalocean/godo/timestamp.go new file mode 100644 index 0000000000..37a28e5f2f --- /dev/null +++ b/vendor/github.com/digitalocean/godo/timestamp.go @@ -0,0 +1,35 @@ +package godo + +import ( + "strconv" + "time" +) + +// Timestamp represents a time that can be unmarshalled from a JSON string +// formatted as either an RFC3339 or Unix timestamp. All +// exported methods of time.Time can be called on Timestamp. +type Timestamp struct { + time.Time +} + +func (t Timestamp) String() string { + return t.Time.String() +} + +// UnmarshalJSON implements the json.Unmarshaler interface. +// Time is expected in RFC3339 or Unix format. +func (t *Timestamp) UnmarshalJSON(data []byte) error { + str := string(data) + i, err := strconv.ParseInt(str, 10, 64) + if err == nil { + t.Time = time.Unix(i, 0) + } else { + t.Time, err = time.Parse(`"`+time.RFC3339+`"`, str) + } + return err +} + +// Equal reports whether t and u are equal based on time.Equal +func (t Timestamp) Equal(u Timestamp) bool { + return t.Time.Equal(u.Time) +} diff --git a/vendor/github.com/digitalocean/godo/vpcs.go b/vendor/github.com/digitalocean/godo/vpcs.go new file mode 100644 index 0000000000..7fbeaf88be --- /dev/null +++ b/vendor/github.com/digitalocean/godo/vpcs.go @@ -0,0 +1,201 @@ +package godo + +import ( + "context" + "net/http" + "time" +) + +const vpcsBasePath = "/v2/vpcs" + +// VPCsService is an interface for managing Virtual Private Cloud configurations with the +// DigitalOcean API. +// See: https://developers.digitalocean.com/documentation/v2#vpcs +type VPCsService interface { + Create(context.Context, *VPCCreateRequest) (*VPC, *Response, error) + Get(context.Context, string) (*VPC, *Response, error) + List(context.Context, *ListOptions) ([]*VPC, *Response, error) + Update(context.Context, string, *VPCUpdateRequest) (*VPC, *Response, error) + Set(context.Context, string, ...VPCSetField) (*VPC, *Response, error) + Delete(context.Context, string) (*Response, error) +} + +var _ VPCsService = &VPCsServiceOp{} + +// VPCsServiceOp interfaces with VPC endpoints in the DigitalOcean API. +type VPCsServiceOp struct { + client *Client +} + +// VPCCreateRequest represents a request to create a Virtual Private Cloud. +type VPCCreateRequest struct { + Name string `json:"name,omitempty"` + RegionSlug string `json:"region,omitempty"` + Description string `json:"description,omitempty"` + IPRange string `json:"ip_range,omitempty"` +} + +// VPCUpdateRequest represents a request to update a Virtual Private Cloud. +type VPCUpdateRequest struct { + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` +} + +// VPCSetField allows one to set individual fields within a VPC configuration. +type VPCSetField interface { + vpcSetField(map[string]interface{}) +} + +// VPCSetName is used when one want to set the `name` field of a VPC. +// Ex.: VPCs.Set(..., VPCSetName("new-name")) +type VPCSetName string + +// VPCSetDescription is used when one want to set the `description` field of a VPC. +// Ex.: VPCs.Set(..., VPCSetDescription("vpc description")) +type VPCSetDescription string + +// VPC represents a DigitalOcean Virtual Private Cloud configuration. +type VPC struct { + ID string `json:"id,omitempty"` + URN string `json:"urn"` + Name string `json:"name,omitempty"` + Description string `json:"description,omitempty"` + IPRange string `json:"ip_range,omitempty"` + RegionSlug string `json:"region,omitempty"` + CreatedAt time.Time `json:"created_at,omitempty"` + Default bool `json:"default,omitempty"` +} + +type vpcRoot struct { + VPC *VPC `json:"vpc"` +} + +type vpcsRoot struct { + VPCs []*VPC `json:"vpcs"` + Links *Links `json:"links"` + Meta *Meta `json:"meta"` +} + +// Get returns the details of a Virtual Private Cloud. +func (v *VPCsServiceOp) Get(ctx context.Context, id string) (*VPC, *Response, error) { + path := vpcsBasePath + "/" + id + req, err := v.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(vpcRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.VPC, resp, nil +} + +// Create creates a new Virtual Private Cloud. +func (v *VPCsServiceOp) Create(ctx context.Context, create *VPCCreateRequest) (*VPC, *Response, error) { + path := vpcsBasePath + req, err := v.client.NewRequest(ctx, http.MethodPost, path, create) + if err != nil { + return nil, nil, err + } + + root := new(vpcRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.VPC, resp, nil +} + +// List returns a list of the caller's VPCs, with optional pagination. +func (v *VPCsServiceOp) List(ctx context.Context, opt *ListOptions) ([]*VPC, *Response, error) { + path, err := addOptions(vpcsBasePath, opt) + if err != nil { + return nil, nil, err + } + req, err := v.client.NewRequest(ctx, http.MethodGet, path, nil) + if err != nil { + return nil, nil, err + } + + root := new(vpcsRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + if l := root.Links; l != nil { + resp.Links = l + } + if m := root.Meta; m != nil { + resp.Meta = m + } + + return root.VPCs, resp, nil +} + +// Update updates a Virtual Private Cloud's properties. +func (v *VPCsServiceOp) Update(ctx context.Context, id string, update *VPCUpdateRequest) (*VPC, *Response, error) { + path := vpcsBasePath + "/" + id + req, err := v.client.NewRequest(ctx, http.MethodPut, path, update) + if err != nil { + return nil, nil, err + } + + root := new(vpcRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.VPC, resp, nil +} + +func (n VPCSetName) vpcSetField(in map[string]interface{}) { + in["name"] = n +} + +func (n VPCSetDescription) vpcSetField(in map[string]interface{}) { + in["description"] = n +} + +// Set updates specific properties of a Virtual Private Cloud. +func (v *VPCsServiceOp) Set(ctx context.Context, id string, fields ...VPCSetField) (*VPC, *Response, error) { + path := vpcsBasePath + "/" + id + update := make(map[string]interface{}, len(fields)) + for _, field := range fields { + field.vpcSetField(update) + } + + req, err := v.client.NewRequest(ctx, http.MethodPatch, path, update) + if err != nil { + return nil, nil, err + } + + root := new(vpcRoot) + resp, err := v.client.Do(ctx, req, root) + if err != nil { + return nil, resp, err + } + + return root.VPC, resp, nil +} + +// Delete deletes a Virtual Private Cloud. There is no way to recover a VPC once it has been +// destroyed. +func (v *VPCsServiceOp) Delete(ctx context.Context, id string) (*Response, error) { + path := vpcsBasePath + "/" + id + req, err := v.client.NewRequest(ctx, http.MethodDelete, path, nil) + if err != nil { + return nil, err + } + + resp, err := v.client.Do(ctx, req, nil) + if err != nil { + return resp, err + } + + return resp, nil +} diff --git a/vendor/github.com/google/go-querystring/LICENSE b/vendor/github.com/google/go-querystring/LICENSE new file mode 100644 index 0000000000..ae121a1e46 --- /dev/null +++ b/vendor/github.com/google/go-querystring/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2013 Google. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/google/go-querystring/query/encode.go b/vendor/github.com/google/go-querystring/query/encode.go new file mode 100644 index 0000000000..37080b19b5 --- /dev/null +++ b/vendor/github.com/google/go-querystring/query/encode.go @@ -0,0 +1,320 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package query implements encoding of structs into URL query parameters. +// +// As a simple example: +// +// type Options struct { +// Query string `url:"q"` +// ShowAll bool `url:"all"` +// Page int `url:"page"` +// } +// +// opt := Options{ "foo", true, 2 } +// v, _ := query.Values(opt) +// fmt.Print(v.Encode()) // will output: "q=foo&all=true&page=2" +// +// The exact mapping between Go values and url.Values is described in the +// documentation for the Values() function. +package query + +import ( + "bytes" + "fmt" + "net/url" + "reflect" + "strconv" + "strings" + "time" +) + +var timeType = reflect.TypeOf(time.Time{}) + +var encoderType = reflect.TypeOf(new(Encoder)).Elem() + +// Encoder is an interface implemented by any type that wishes to encode +// itself into URL values in a non-standard way. +type Encoder interface { + EncodeValues(key string, v *url.Values) error +} + +// Values returns the url.Values encoding of v. +// +// Values expects to be passed a struct, and traverses it recursively using the +// following encoding rules. +// +// Each exported struct field is encoded as a URL parameter unless +// +// - the field's tag is "-", or +// - the field is empty and its tag specifies the "omitempty" option +// +// The empty values are false, 0, any nil pointer or interface value, any array +// slice, map, or string of length zero, and any time.Time that returns true +// for IsZero(). +// +// The URL parameter name defaults to the struct field name but can be +// specified in the struct field's tag value. The "url" key in the struct +// field's tag value is the key name, followed by an optional comma and +// options. For example: +// +// // Field is ignored by this package. +// Field int `url:"-"` +// +// // Field appears as URL parameter "myName". +// Field int `url:"myName"` +// +// // Field appears as URL parameter "myName" and the field is omitted if +// // its value is empty +// Field int `url:"myName,omitempty"` +// +// // Field appears as URL parameter "Field" (the default), but the field +// // is skipped if empty. Note the leading comma. +// Field int `url:",omitempty"` +// +// For encoding individual field values, the following type-dependent rules +// apply: +// +// Boolean values default to encoding as the strings "true" or "false". +// Including the "int" option signals that the field should be encoded as the +// strings "1" or "0". +// +// time.Time values default to encoding as RFC3339 timestamps. Including the +// "unix" option signals that the field should be encoded as a Unix time (see +// time.Unix()) +// +// Slice and Array values default to encoding as multiple URL values of the +// same name. Including the "comma" option signals that the field should be +// encoded as a single comma-delimited value. Including the "space" option +// similarly encodes the value as a single space-delimited string. Including +// the "semicolon" option will encode the value as a semicolon-delimited string. +// Including the "brackets" option signals that the multiple URL values should +// have "[]" appended to the value name. "numbered" will append a number to +// the end of each incidence of the value name, example: +// name0=value0&name1=value1, etc. +// +// Anonymous struct fields are usually encoded as if their inner exported +// fields were fields in the outer struct, subject to the standard Go +// visibility rules. An anonymous struct field with a name given in its URL +// tag is treated as having that name, rather than being anonymous. +// +// Non-nil pointer values are encoded as the value pointed to. +// +// Nested structs are encoded including parent fields in value names for +// scoping. e.g: +// +// "user[name]=acme&user[addr][postcode]=1234&user[addr][city]=SFO" +// +// All other values are encoded using their default string representation. +// +// Multiple fields that encode to the same URL parameter name will be included +// as multiple URL values of the same name. +func Values(v interface{}) (url.Values, error) { + values := make(url.Values) + val := reflect.ValueOf(v) + for val.Kind() == reflect.Ptr { + if val.IsNil() { + return values, nil + } + val = val.Elem() + } + + if v == nil { + return values, nil + } + + if val.Kind() != reflect.Struct { + return nil, fmt.Errorf("query: Values() expects struct input. Got %v", val.Kind()) + } + + err := reflectValue(values, val, "") + return values, err +} + +// reflectValue populates the values parameter from the struct fields in val. +// Embedded structs are followed recursively (using the rules defined in the +// Values function documentation) breadth-first. +func reflectValue(values url.Values, val reflect.Value, scope string) error { + var embedded []reflect.Value + + typ := val.Type() + for i := 0; i < typ.NumField(); i++ { + sf := typ.Field(i) + if sf.PkgPath != "" && !sf.Anonymous { // unexported + continue + } + + sv := val.Field(i) + tag := sf.Tag.Get("url") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if name == "" { + if sf.Anonymous && sv.Kind() == reflect.Struct { + // save embedded struct for later processing + embedded = append(embedded, sv) + continue + } + + name = sf.Name + } + + if scope != "" { + name = scope + "[" + name + "]" + } + + if opts.Contains("omitempty") && isEmptyValue(sv) { + continue + } + + if sv.Type().Implements(encoderType) { + if !reflect.Indirect(sv).IsValid() { + sv = reflect.New(sv.Type().Elem()) + } + + m := sv.Interface().(Encoder) + if err := m.EncodeValues(name, &values); err != nil { + return err + } + continue + } + + if sv.Kind() == reflect.Slice || sv.Kind() == reflect.Array { + var del byte + if opts.Contains("comma") { + del = ',' + } else if opts.Contains("space") { + del = ' ' + } else if opts.Contains("semicolon") { + del = ';' + } else if opts.Contains("brackets") { + name = name + "[]" + } + + if del != 0 { + s := new(bytes.Buffer) + first := true + for i := 0; i < sv.Len(); i++ { + if first { + first = false + } else { + s.WriteByte(del) + } + s.WriteString(valueString(sv.Index(i), opts)) + } + values.Add(name, s.String()) + } else { + for i := 0; i < sv.Len(); i++ { + k := name + if opts.Contains("numbered") { + k = fmt.Sprintf("%s%d", name, i) + } + values.Add(k, valueString(sv.Index(i), opts)) + } + } + continue + } + + for sv.Kind() == reflect.Ptr { + if sv.IsNil() { + break + } + sv = sv.Elem() + } + + if sv.Type() == timeType { + values.Add(name, valueString(sv, opts)) + continue + } + + if sv.Kind() == reflect.Struct { + reflectValue(values, sv, name) + continue + } + + values.Add(name, valueString(sv, opts)) + } + + for _, f := range embedded { + if err := reflectValue(values, f, scope); err != nil { + return err + } + } + + return nil +} + +// valueString returns the string representation of a value. +func valueString(v reflect.Value, opts tagOptions) string { + for v.Kind() == reflect.Ptr { + if v.IsNil() { + return "" + } + v = v.Elem() + } + + if v.Kind() == reflect.Bool && opts.Contains("int") { + if v.Bool() { + return "1" + } + return "0" + } + + if v.Type() == timeType { + t := v.Interface().(time.Time) + if opts.Contains("unix") { + return strconv.FormatInt(t.Unix(), 10) + } + return t.Format(time.RFC3339) + } + + return fmt.Sprint(v.Interface()) +} + +// isEmptyValue checks if a value should be considered empty for the purposes +// of omitting fields with the "omitempty" option. +func isEmptyValue(v reflect.Value) bool { + switch v.Kind() { + case reflect.Array, reflect.Map, reflect.Slice, reflect.String: + return v.Len() == 0 + case reflect.Bool: + return !v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() == 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + return v.Uint() == 0 + case reflect.Float32, reflect.Float64: + return v.Float() == 0 + case reflect.Interface, reflect.Ptr: + return v.IsNil() + } + + if v.Type() == timeType { + return v.Interface().(time.Time).IsZero() + } + + return false +} + +// tagOptions is the string following a comma in a struct field's "url" tag, or +// the empty string. It does not include the leading comma. +type tagOptions []string + +// parseTag splits a struct field's url tag into its name and comma-separated +// options. +func parseTag(tag string) (string, tagOptions) { + s := strings.Split(tag, ",") + return s[0], s[1:] +} + +// Contains checks whether the tagOptions contains the specified option. +func (o tagOptions) Contains(option string) bool { + for _, s := range o { + if s == option { + return true + } + } + return false +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 3e96e8df5c..92c7fc100e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -82,6 +82,8 @@ github.com/davecgh/go-spew/spew github.com/dgrijalva/jwt-go # github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b github.com/dgryski/go-sip13 +# github.com/digitalocean/godo v1.37.0 +github.com/digitalocean/godo # github.com/edsrzf/mmap-go v1.0.0 github.com/edsrzf/mmap-go # github.com/evanphx/json-patch v4.2.0+incompatible @@ -173,6 +175,8 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value +# github.com/google/go-querystring v1.0.0 +github.com/google/go-querystring/query # github.com/google/gofuzz v1.1.0 github.com/google/gofuzz # github.com/google/pprof v0.0.0-20200507031123-427632fa3b1c From 8d3c2f6829d73be15a6684f9324917e72fbf1a31 Mon Sep 17 00:00:00 2001 From: Ben Kochie Date: Thu, 18 Jun 2020 18:59:40 +0200 Subject: [PATCH 31/31] Enable WAL compression by default (#7410) Enable the `--storage.tsdb.wal-compression` flag by defualt. Signed-off-by: Ben Kochie --- CHANGELOG.md | 4 ++++ cmd/prometheus/main.go | 2 +- docs/storage.md | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 604d98d8c6..92547201b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +## master / unreleased + +* [ENHANCEMENT] TSDB: WAL compression is enabled by default. + ## 2.19.0 / 2020-06-09 * [FEATURE] TSDB: Memory-map full chunks of Head (in-memory) block from disk. This reduces memory footprint and makes restarts faster. #6679 diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index d3386c3b55..c6066b72f4 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -214,7 +214,7 @@ func main() { Default("false").BoolVar(&cfg.tsdb.AllowOverlappingBlocks) a.Flag("storage.tsdb.wal-compression", "Compress the tsdb WAL."). - Default("false").BoolVar(&cfg.tsdb.WALCompression) + Default("true").BoolVar(&cfg.tsdb.WALCompression) a.Flag("storage.remote.flush-deadline", "How long to wait flushing sample on shutdown or config reload."). Default("1m").PlaceHolder("").SetValue(&cfg.RemoteFlushDeadline) diff --git a/docs/storage.md b/docs/storage.md index 808547101d..f599dc8496 100644 --- a/docs/storage.md +++ b/docs/storage.md @@ -63,7 +63,7 @@ Prometheus has several flags that allow configuring the local storage. The most * `--storage.tsdb.retention.time`: This determines when to remove old data. Defaults to `15d`. Overrides `storage.tsdb.retention` if this flag is set to anything other than default. * `--storage.tsdb.retention.size`: [EXPERIMENTAL] This determines the maximum number of bytes that storage blocks can use (note that this does not include the WAL size, which can be substantial). The oldest data will be removed first. Defaults to `0` or disabled. This flag is experimental and can be changed in future releases. Units supported: B, KB, MB, GB, TB, PB, EB. Ex: "512MB" * `--storage.tsdb.retention`: This flag has been deprecated in favour of `storage.tsdb.retention.time`. -* `--storage.tsdb.wal-compression`: This flag enables compression of the write-ahead log (WAL). Depending on your data, you can expect the WAL size to be halved with little extra cpu load. Note that if you enable this flag and subsequently downgrade Prometheus to a version below 2.11.0 you will need to delete your WAL as it will be unreadable. +* `--storage.tsdb.wal-compression`: This flag enables compression of the write-ahead log (WAL). Depending on your data, you can expect the WAL size to be halved with little extra cpu load. This flag was introduced in 2.11.0 and enabled by default in 2.20.0. Note that once enabled, downgrading Prometheus to a version below 2.11.0 will require deleting the WAL. On average, Prometheus uses only around 1-2 bytes per sample. Thus, to plan the capacity of a Prometheus server, you can use the rough formula: