Merge remote-tracking branch 'upstream/master' into merge-release-2.19

Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in>

.circleci/config.yml

@@ -87,11 +87,11 @@ jobs:
       - setup_remote_docker
       - run: ./fuzzit.sh fuzzing
 
-  makefile_sync:
+  repo_sync:
     executor: golang
     steps:
     - checkout
-    - run: ./scripts/sync_makefiles.sh
+    - run: ./scripts/sync_repo_files.sh
 
 workflows:
   version: 2
@@ -143,7 +143,7 @@ workflows:
               only:
                 - master
     jobs:
-    - makefile_sync:
+    - repo_sync:
         context: org-context
     - fuzzit_fuzzing:
         context: org-context

.github/workflows/funcbench.yml

@@ -9,7 +9,6 @@ jobs:
       AUTH_FILE: ${{ secrets.TEST_INFRA_GKE_AUTH }}
       BRANCH: ${{ github.event.client_payload.BRANCH }}
       BENCH_FUNC_REGEX: ${{ github.event.client_payload.BENCH_FUNC_REGEX }}
-      CLUSTER_NAME: test-infra
       GITHUB_TOKEN: ${{ secrets.PROMBOT_TOKEN }}
       GITHUB_ORG: prometheus
       GITHUB_REPO: prometheus

CHANGELOG.md

@@ -1,3 +1,7 @@
+## master / unreleased
+
+* [ENHANCEMENT] TSDB: WAL compression is enabled by default.
+
 ## 2.19.1 / 2020-06-18
 
 * [BUGFIX] TSDB: Fix m-map file truncation leading to unsequential files. #7414
@@ -14,6 +18,9 @@
 * [BUGFIX] React UI: Don't null out data when clicking on the current tab. #7243
 * [BUGFIX] PromQL: Correctly track number of samples for a query. #7307
 * [BUGFIX] PromQL: Return NaN when histogram buckets have 0 observations. #7318
+
+## 2.18.2 / 2020-06-09
+
 * [BUGFIX] TSDB: Fix incorrect query results when using Prometheus with remote reads configured #7361
 
 ## 2.18.1 / 2020-05-07

NOTICE

@@ -86,6 +86,11 @@ https://github.com/samuel/go-zookeeper
 Copyright (c) 2013, Samuel Stauffer <samuel@descolada.com>
 See https://github.com/samuel/go-zookeeper/blob/master/LICENSE for license details.
 
+Time series compression algorithm from Facebook's Gorilla paper
+https://github.com/dgryski/go-tsz
+Copyright (c) 2015,2016 Damian Gryski <damian@gryski.com>
+See https://github.com/dgryski/go-tsz/blob/master/LICENSE for license details.
+
 We also use code from a large number of npm packages. For details, see:
 - https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/package.json
 - https://github.com/prometheus/prometheus/blob/master/web/ui/react-app/package-lock.json

cmd/prometheus/main.go

@@ -214,7 +214,7 @@ func main() {
 		Default("false").BoolVar(&cfg.tsdb.AllowOverlappingBlocks)
 
 	a.Flag("storage.tsdb.wal-compression", "Compress the tsdb WAL.").
-		Default("false").BoolVar(&cfg.tsdb.WALCompression)
+		Default("true").BoolVar(&cfg.tsdb.WALCompression)
 
 	a.Flag("storage.remote.flush-deadline", "How long to wait flushing sample on shutdown or config reload.").
 		Default("1m").PlaceHolder("<duration>").SetValue(&cfg.RemoteFlushDeadline)

cmd/promtool/testdata/alerts.yml

@@ -0,0 +1,13 @@
+# This is the rules file.
+
+groups:
+  - name: example
+    rules:
+      - alert: InstanceDown
+        expr: up == 0
+        for: 5m
+        labels:
+          severity: page
+        annotations:
+          summary: "Instance {{ $labels.instance }} down"
+          description: "{{ $labels.instance }} of job {{ $labels.job }} has been down for more than 5 minutes."

cmd/promtool/testdata/unittest.yml

@@ -0,0 +1,21 @@
+rule_files:
+  - alerts.yml
+
+evaluation_interval: 1m
+
+tests:
+  - interval: 1m
+    input_series:
+      - series: 'up{job="prometheus", instance="localhost:9090"}'
+        values: "0+0x1440"
+    alert_rule_test:
+      - eval_time: 1d
+        alertname: InstanceDown
+        exp_alerts:
+          - exp_labels:
+              severity: page
+              instance: localhost:9090
+              job: prometheus
+            exp_annotations:
+              summary: "Instance localhost:9090 down"
+              description: "localhost:9090 of job prometheus has been down for more than 5 minutes."

cmd/promtool/unittest.go

@@ -29,6 +29,7 @@ import (
 	"github.com/pkg/errors"
 	yaml "gopkg.in/yaml.v2"
 
+	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -76,15 +77,16 @@ func ruleUnitTest(filename string) []error {
 	}
 
 	if unitTestInp.EvaluationInterval == 0 {
-		unitTestInp.EvaluationInterval = 1 * time.Minute
+		unitTestInp.EvaluationInterval = model.Duration(1 * time.Minute)
 	}
 
 	// Bounds for evaluating the rules.
 	mint := time.Unix(0, 0).UTC()
 	maxd := unitTestInp.maxEvalTime()
 	maxt := mint.Add(maxd)
+	evalInterval := time.Duration(unitTestInp.EvaluationInterval)
 	// Rounding off to nearest Eval time (> maxt).
-	maxt = maxt.Add(unitTestInp.EvaluationInterval / 2).Round(unitTestInp.EvaluationInterval)
+	maxt = maxt.Add(evalInterval / 2).Round(evalInterval)
 
 	// Giving number for groups mentioned in the file for ordering.
 	// Lower number group should be evaluated before higher number group.
@@ -99,7 +101,7 @@ func ruleUnitTest(filename string) []error {
 	// Testing.
 	var errs []error
 	for _, t := range unitTestInp.Tests {
-		ers := t.test(mint, maxt, unitTestInp.EvaluationInterval, groupOrderMap,
+		ers := t.test(mint, maxt, evalInterval, groupOrderMap,
 			unitTestInp.RuleFiles...)
 		if ers != nil {
 			errs = append(errs, ers...)
@@ -114,10 +116,10 @@ func ruleUnitTest(filename string) []error {
 
 // unitTestFile holds the contents of a single unit test file.
 type unitTestFile struct {
-	RuleFiles          []string      `yaml:"rule_files"`
+	RuleFiles          []string       `yaml:"rule_files"`
-	EvaluationInterval time.Duration `yaml:"evaluation_interval,omitempty"`
+	EvaluationInterval model.Duration `yaml:"evaluation_interval,omitempty"`
-	GroupEvalOrder     []string      `yaml:"group_eval_order"`
+	GroupEvalOrder     []string       `yaml:"group_eval_order"`
-	Tests              []testGroup   `yaml:"tests"`
+	Tests              []testGroup    `yaml:"tests"`
 }
 
 func (utf *unitTestFile) maxEvalTime() time.Duration {
@@ -157,7 +159,7 @@ func resolveAndGlobFilepaths(baseDir string, utf *unitTestFile) error {
 
 // testGroup is a group of input series and tests associated with it.
 type testGroup struct {
-	Interval        time.Duration    `yaml:"interval"`
+	Interval        model.Duration   `yaml:"interval"`
 	InputSeries     []series         `yaml:"input_series"`
 	AlertRuleTests  []alertTestCase  `yaml:"alert_rule_test,omitempty"`
 	PromqlExprTests []promqlTestCase `yaml:"promql_expr_test,omitempty"`
@@ -182,7 +184,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 		Logger:     log.NewNopLogger(),
 	}
 	m := rules.NewManager(opts)
-	groupsMap, ers := m.LoadGroups(tg.Interval, tg.ExternalLabels, ruleFiles...)
+	groupsMap, ers := m.LoadGroups(time.Duration(tg.Interval), tg.ExternalLabels, ruleFiles...)
 	if ers != nil {
 		return ers
 	}
@@ -193,11 +195,11 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 	// This avoids storing them in memory, as the number of evals might be high.
 
 	// All the `eval_time` for which we have unit tests for alerts.
-	alertEvalTimesMap := map[time.Duration]struct{}{}
+	alertEvalTimesMap := map[model.Duration]struct{}{}
 	// Map of all the eval_time+alertname combination present in the unit tests.
-	alertsInTest := make(map[time.Duration]map[string]struct{})
+	alertsInTest := make(map[model.Duration]map[string]struct{})
 	// Map of all the unit tests for given eval_time.
-	alertTests := make(map[time.Duration][]alertTestCase)
+	alertTests := make(map[model.Duration][]alertTestCase)
 	for _, alert := range tg.AlertRuleTests {
 		alertEvalTimesMap[alert.EvalTime] = struct{}{}
 
@@ -208,7 +210,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 
 		alertTests[alert.EvalTime] = append(alertTests[alert.EvalTime], alert)
 	}
-	alertEvalTimes := make([]time.Duration, 0, len(alertEvalTimesMap))
+	alertEvalTimes := make([]model.Duration, 0, len(alertEvalTimesMap))
 	for k := range alertEvalTimesMap {
 		alertEvalTimes = append(alertEvalTimes, k)
 	}
@@ -242,8 +244,8 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 		}
 
 		for {
-			if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= alertEvalTimes[curr] &&
+			if !(curr < len(alertEvalTimes) && ts.Sub(mint) <= time.Duration(alertEvalTimes[curr]) &&
-				alertEvalTimes[curr] < ts.Add(evalInterval).Sub(mint)) {
+				time.Duration(alertEvalTimes[curr]) < ts.Add(time.Duration(evalInterval)).Sub(mint)) {
 				break
 			}
 
@@ -322,7 +324,7 @@ func (tg *testGroup) test(mint, maxt time.Time, evalInterval time.Duration, grou
 	// Checking promql expressions.
 Outer:
 	for _, testCase := range tg.PromqlExprTests {
-		got, err := query(suite.Context(), testCase.Expr, mint.Add(testCase.EvalTime),
+		got, err := query(suite.Context(), testCase.Expr, mint.Add(time.Duration(testCase.EvalTime)),
 			suite.QueryEngine(), suite.Queryable())
 		if err != nil {
 			errs = append(errs, errors.Errorf("    expr: %q, time: %s, err: %s", testCase.Expr,
@@ -373,15 +375,15 @@ Outer:
 
 // seriesLoadingString returns the input series in PromQL notation.
 func (tg *testGroup) seriesLoadingString() string {
-	result := ""
+
-	result += "load " + shortDuration(tg.Interval) + "\n"
+	result := fmt.Sprintf("load %v\n", shortDuration(tg.Interval))
 	for _, is := range tg.InputSeries {
-		result += "  " + is.Series + " " + is.Values + "\n"
+		result += fmt.Sprintf("  %v %v\n", is.Series, is.Values)
 	}
 	return result
 }
 
-func shortDuration(d time.Duration) string {
+func shortDuration(d model.Duration) string {
 	s := d.String()
 	if strings.HasSuffix(s, "m0s") {
 		s = s[:len(s)-2]
@@ -407,7 +409,7 @@ func orderedGroups(groupsMap map[string]*rules.Group, groupOrderMap map[string]i
 
 // maxEvalTime returns the max eval time among all alert and promql unit tests.
 func (tg *testGroup) maxEvalTime() time.Duration {
-	var maxd time.Duration
+	var maxd model.Duration
 	for _, alert := range tg.AlertRuleTests {
 		if alert.EvalTime > maxd {
 			maxd = alert.EvalTime
@@ -418,7 +420,7 @@ func (tg *testGroup) maxEvalTime() time.Duration {
 			maxd = pet.EvalTime
 		}
 	}
-	return maxd
+	return time.Duration(maxd)
 }
 
 func query(ctx context.Context, qs string, t time.Time, engine *promql.Engine, qu storage.Queryable) (promql.Vector, error) {
@@ -483,9 +485,9 @@ type series struct {
 }
 
 type alertTestCase struct {
-	EvalTime  time.Duration `yaml:"eval_time"`
+	EvalTime  model.Duration `yaml:"eval_time"`
-	Alertname string        `yaml:"alertname"`
+	Alertname string         `yaml:"alertname"`
-	ExpAlerts []alert       `yaml:"exp_alerts"`
+	ExpAlerts []alert        `yaml:"exp_alerts"`
 }
 
 type alert struct {
@@ -494,9 +496,9 @@ type alert struct {
 }
 
 type promqlTestCase struct {
-	Expr       string        `yaml:"expr"`
+	Expr       string         `yaml:"expr"`
-	EvalTime   time.Duration `yaml:"eval_time"`
+	EvalTime   model.Duration `yaml:"eval_time"`
-	ExpSamples []sample      `yaml:"exp_samples"`
+	ExpSamples []sample       `yaml:"exp_samples"`
 }
 
 type sample struct {

cmd/promtool/unittest_test.go

@@ -0,0 +1,42 @@
+// Copyright 2018 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import "testing"
+
+func TestRulesUnitTest(t *testing.T) {
+	type args struct {
+		files []string
+	}
+	tests := []struct {
+		name string
+		args args
+		want int
+	}{
+		{
+			name: "Passing Unit Tests",
+			args: args{
+				files: []string{"./testdata/unittest.yml"},
+			},
+			want: 0,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := RulesUnitTest(tt.args.files...); got != tt.want {
+				t.Errorf("RulesUnitTest() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

discovery/config/config.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/prometheus/prometheus/discovery/azure"
 	"github.com/prometheus/prometheus/discovery/consul"
+	"github.com/prometheus/prometheus/discovery/digitalocean"
 	"github.com/prometheus/prometheus/discovery/dns"
 	"github.com/prometheus/prometheus/discovery/ec2"
 	"github.com/prometheus/prometheus/discovery/file"
@@ -40,6 +41,8 @@ type ServiceDiscoveryConfig struct {
 	FileSDConfigs []*file.SDConfig `yaml:"file_sd_configs,omitempty"`
 	// List of Consul service discovery configurations.
 	ConsulSDConfigs []*consul.SDConfig `yaml:"consul_sd_configs,omitempty"`
+	// List of DigitalOcean service discovery configurations.
+	DigitalOceanSDConfigs []*digitalocean.SDConfig `yaml:"digitalocean_sd_configs,omitempty"`
 	// List of Serverset service discovery configurations.
 	ServersetSDConfigs []*zookeeper.ServersetSDConfig `yaml:"serverset_sd_configs,omitempty"`
 	// NerveSDConfigs is a list of Nerve service discovery configurations.
@@ -72,6 +75,11 @@ func (c *ServiceDiscoveryConfig) Validate() error {
 			return errors.New("empty or null section in consul_sd_configs")
 		}
 	}
+	for _, cfg := range c.DigitalOceanSDConfigs {
+		if cfg == nil {
+			return errors.New("empty or null section in digitalocean_sd_configs")
+		}
+	}
 	for _, cfg := range c.DNSSDConfigs {
 		if cfg == nil {
 			return errors.New("empty or null section in dns_sd_configs")

discovery/digitalocean/digitalocean.go

@@ -0,0 +1,193 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package digitalocean
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/digitalocean/godo"
+	"github.com/go-kit/kit/log"
+	config_util "github.com/prometheus/common/config"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/common/version"
+
+	"github.com/prometheus/prometheus/discovery/refresh"
+	"github.com/prometheus/prometheus/discovery/targetgroup"
+)
+
+const (
+	doLabel            = model.MetaLabelPrefix + "digitalocean_"
+	doLabelID          = doLabel + "droplet_id"
+	doLabelName        = doLabel + "droplet_name"
+	doLabelImage       = doLabel + "image"
+	doLabelPrivateIPv4 = doLabel + "private_ipv4"
+	doLabelPublicIPv4  = doLabel + "public_ipv4"
+	doLabelPublicIPv6  = doLabel + "public_ipv6"
+	doLabelRegion      = doLabel + "region"
+	doLabelSize        = doLabel + "size"
+	doLabelStatus      = doLabel + "status"
+	doLabelFeatures    = doLabel + "features"
+	doLabelTags        = doLabel + "tags"
+	separator          = ","
+)
+
+// DefaultSDConfig is the default DigitalOcean SD configuration.
+var DefaultSDConfig = SDConfig{
+	Port:            80,
+	RefreshInterval: model.Duration(60 * time.Second),
+}
+
+// SDConfig is the configuration for DigitalOcean based service discovery.
+type SDConfig struct {
+	HTTPClientConfig config_util.HTTPClientConfig `yaml:",inline"`
+
+	RefreshInterval model.Duration `yaml:"refresh_interval,omitempty"`
+	Port            int            `yaml:"port"`
+}
+
+// UnmarshalYAML implements the yaml.Unmarshaler interface.
+func (c *SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	*c = DefaultSDConfig
+	type plain SDConfig
+	err := unmarshal((*plain)(c))
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Discovery periodically performs DigitalOcean requests. It implements
+// the Discoverer interface.
+type Discovery struct {
+	*refresh.Discovery
+	client *godo.Client
+	port   int
+}
+
+// NewDiscovery returns a new Discovery which periodically refreshes its targets.
+func NewDiscovery(conf *SDConfig, logger log.Logger) (*Discovery, error) {
+	d := &Discovery{
+		port: conf.Port,
+	}
+
+	rt, err := config_util.NewRoundTripperFromConfig(conf.HTTPClientConfig, "digitalocean_sd", false)
+	if err != nil {
+		return nil, err
+	}
+
+	d.client, err = godo.New(
+		&http.Client{
+			Transport: rt,
+			Timeout:   5 * time.Duration(conf.RefreshInterval),
+		},
+		godo.SetUserAgent(fmt.Sprintf("Prometheus/%s", version.Version)),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("error setting up digital ocean agent: %w", err)
+	}
+
+	d.Discovery = refresh.NewDiscovery(
+		logger,
+		"digitalocean",
+		time.Duration(conf.RefreshInterval),
+		d.refresh,
+	)
+	return d, nil
+}
+
+func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
+	tg := &targetgroup.Group{
+		Source: "DigitalOcean",
+	}
+
+	droplets, err := d.listDroplets()
+	if err != nil {
+		return nil, err
+	}
+	for _, droplet := range droplets {
+		if droplet.Networks == nil || len(droplet.Networks.V4) == 0 {
+			continue
+		}
+
+		privateIPv4, err := droplet.PrivateIPv4()
+		if err != nil {
+			return nil, fmt.Errorf("error while reading private IPv4 of droplet %d: %w", droplet.ID, err)
+		}
+		publicIPv4, err := droplet.PublicIPv4()
+		if err != nil {
+			return nil, fmt.Errorf("error while reading public IPv4 of droplet %d: %w", droplet.ID, err)
+		}
+		publicIPv6, err := droplet.PublicIPv6()
+		if err != nil {
+			return nil, fmt.Errorf("error while reading public IPv6 of droplet %d: %w", droplet.ID, err)
+		}
+
+		labels := model.LabelSet{
+			doLabelID:          model.LabelValue(fmt.Sprintf("%d", droplet.ID)),
+			doLabelName:        model.LabelValue(droplet.Name),
+			doLabelImage:       model.LabelValue(droplet.Image.Slug),
+			doLabelPrivateIPv4: model.LabelValue(privateIPv4),
+			doLabelPublicIPv4:  model.LabelValue(publicIPv4),
+			doLabelPublicIPv6:  model.LabelValue(publicIPv6),
+			doLabelRegion:      model.LabelValue(droplet.Region.Slug),
+			doLabelSize:        model.LabelValue(droplet.SizeSlug),
+			doLabelStatus:      model.LabelValue(droplet.Status),
+		}
+
+		addr := net.JoinHostPort(publicIPv4, strconv.FormatUint(uint64(d.port), 10))
+		labels[model.AddressLabel] = model.LabelValue(addr)
+
+		if len(droplet.Features) > 0 {
+			// We surround the separated list with the separator as well. This way regular expressions
+			// in relabeling rules don't have to consider feature positions.
+			features := separator + strings.Join(droplet.Features, separator) + separator
+			labels[doLabelFeatures] = model.LabelValue(features)
+		}
+
+		if len(droplet.Tags) > 0 {
+			// We surround the separated list with the separator as well. This way regular expressions
+			// in relabeling rules don't have to consider tag positions.
+			tags := separator + strings.Join(droplet.Tags, separator) + separator
+			labels[doLabelTags] = model.LabelValue(tags)
+		}
+
+		tg.Targets = append(tg.Targets, labels)
+	}
+	return []*targetgroup.Group{tg}, nil
+}
+
+func (d *Discovery) listDroplets() ([]godo.Droplet, error) {
+	var (
+		droplets []godo.Droplet
+		opts     = &godo.ListOptions{Page: 1}
+	)
+	for {
+		paginatedDroplets, resp, err := d.client.Droplets.List(context.Background(), opts)
+		if err != nil {
+			return nil, fmt.Errorf("error while listing droplets page %d: %w", opts.Page, err)
+		}
+		droplets = append(droplets, paginatedDroplets...)
+		if resp.Links == nil || resp.Links.IsLastPage() {
+			break
+		}
+		opts.Page++
+	}
+	return droplets, nil
+}

discovery/digitalocean/digitalocean_test.go

@@ -0,0 +1,125 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package digitalocean
+
+import (
+	"context"
+	"fmt"
+	"net/url"
+	"testing"
+
+	"github.com/go-kit/kit/log"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/util/testutil"
+)
+
+type DigitalOceanSDTestSuite struct {
+	Mock *SDMock
+}
+
+func (s *DigitalOceanSDTestSuite) TearDownSuite() {
+	s.Mock.ShutdownServer()
+}
+
+func (s *DigitalOceanSDTestSuite) SetupTest(t *testing.T) {
+	s.Mock = NewSDMock(t)
+	s.Mock.Setup()
+
+	s.Mock.HandleDropletsList()
+}
+
+func TestDigitalOceanSDRefresh(t *testing.T) {
+	sdmock := &DigitalOceanSDTestSuite{}
+	sdmock.SetupTest(t)
+	t.Cleanup(sdmock.TearDownSuite)
+
+	cfg := DefaultSDConfig
+	cfg.HTTPClientConfig.BearerToken = tokenID
+	d, err := NewDiscovery(&cfg, log.NewNopLogger())
+	testutil.Ok(t, err)
+	endpoint, err := url.Parse(sdmock.Mock.Endpoint())
+	testutil.Ok(t, err)
+	d.client.BaseURL = endpoint
+
+	ctx := context.Background()
+	tgs, err := d.refresh(ctx)
+	testutil.Ok(t, err)
+
+	testutil.Equals(t, 1, len(tgs))
+
+	tg := tgs[0]
+	testutil.Assert(t, tg != nil, "tg should not be nil")
+	testutil.Assert(t, tg.Targets != nil, "tg.targets should not be nil")
+	testutil.Equals(t, 4, len(tg.Targets))
+
+	for i, lbls := range []model.LabelSet{
+		{
+			"__address__":                      model.LabelValue("104.236.32.182:80"),
+			"__meta_digitalocean_droplet_id":   model.LabelValue("3164444"),
+			"__meta_digitalocean_droplet_name": model.LabelValue("example.com"),
+			"__meta_digitalocean_image":        model.LabelValue("ubuntu-16-04-x64"),
+			"__meta_digitalocean_private_ipv4": model.LabelValue(""),
+			"__meta_digitalocean_public_ipv4":  model.LabelValue("104.236.32.182"),
+			"__meta_digitalocean_public_ipv6":  model.LabelValue("2604:A880:0800:0010:0000:0000:02DD:4001"),
+			"__meta_digitalocean_region":       model.LabelValue("nyc3"),
+			"__meta_digitalocean_size":         model.LabelValue("s-1vcpu-1gb"),
+			"__meta_digitalocean_status":       model.LabelValue("active"),
+			"__meta_digitalocean_features":     model.LabelValue(",backups,ipv6,virtio,"),
+		},
+		{
+			"__address__":                      model.LabelValue("104.131.186.241:80"),
+			"__meta_digitalocean_droplet_id":   model.LabelValue("3164494"),
+			"__meta_digitalocean_droplet_name": model.LabelValue("prometheus"),
+			"__meta_digitalocean_image":        model.LabelValue("ubuntu-16-04-x64"),
+			"__meta_digitalocean_private_ipv4": model.LabelValue(""),
+			"__meta_digitalocean_public_ipv4":  model.LabelValue("104.131.186.241"),
+			"__meta_digitalocean_public_ipv6":  model.LabelValue(""),
+			"__meta_digitalocean_region":       model.LabelValue("nyc3"),
+			"__meta_digitalocean_size":         model.LabelValue("s-1vcpu-1gb"),
+			"__meta_digitalocean_status":       model.LabelValue("active"),
+			"__meta_digitalocean_tags":         model.LabelValue(",monitor,"),
+			"__meta_digitalocean_features":     model.LabelValue(",virtio,"),
+		},
+		{
+			"__address__":                      model.LabelValue("167.172.111.118:80"),
+			"__meta_digitalocean_droplet_id":   model.LabelValue("175072239"),
+			"__meta_digitalocean_droplet_name": model.LabelValue("prometheus-demo-old"),
+			"__meta_digitalocean_image":        model.LabelValue("ubuntu-18-04-x64"),
+			"__meta_digitalocean_private_ipv4": model.LabelValue("10.135.64.211"),
+			"__meta_digitalocean_public_ipv4":  model.LabelValue("167.172.111.118"),
+			"__meta_digitalocean_public_ipv6":  model.LabelValue(""),
+			"__meta_digitalocean_region":       model.LabelValue("fra1"),
+			"__meta_digitalocean_size":         model.LabelValue("s-1vcpu-1gb"),
+			"__meta_digitalocean_status":       model.LabelValue("off"),
+			"__meta_digitalocean_features":     model.LabelValue(",ipv6,private_networking,"),
+		},
+		{
+			"__address__":                      model.LabelValue("138.65.56.69:80"),
+			"__meta_digitalocean_droplet_id":   model.LabelValue("176011507"),
+			"__meta_digitalocean_droplet_name": model.LabelValue("prometheus-demo"),
+			"__meta_digitalocean_image":        model.LabelValue("ubuntu-18-04-x64"),
+			"__meta_digitalocean_private_ipv4": model.LabelValue("10.135.64.212"),
+			"__meta_digitalocean_public_ipv4":  model.LabelValue("138.65.56.69"),
+			"__meta_digitalocean_public_ipv6":  model.LabelValue("2a03:b0c0:3:f0::cf2:4"),
+			"__meta_digitalocean_region":       model.LabelValue("fra1"),
+			"__meta_digitalocean_size":         model.LabelValue("s-1vcpu-1gb"),
+			"__meta_digitalocean_status":       model.LabelValue("active"),
+			"__meta_digitalocean_features":     model.LabelValue(",ipv6,private_networking,"),
+		},
+	} {
+		t.Run(fmt.Sprintf("item %d", i), func(t *testing.T) {
+			testutil.Equals(t, lbls, tg.Targets[i])
+		})
+	}
+}

discovery/digitalocean/mock_test.go

@@ -0,0 +1,644 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package digitalocean
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"testing"
+)
+
+// SDMock is the interface for the DigitalOcean mock
+type SDMock struct {
+	t      *testing.T
+	Server *httptest.Server
+	Mux    *http.ServeMux
+}
+
+// NewSDMock returns a new SDMock.
+func NewSDMock(t *testing.T) *SDMock {
+	return &SDMock{
+		t: t,
+	}
+}
+
+// Endpoint returns the URI to the mock server
+func (m *SDMock) Endpoint() string {
+	return m.Server.URL + "/"
+}
+
+// Setup creates the mock server
+func (m *SDMock) Setup() {
+	m.Mux = http.NewServeMux()
+	m.Server = httptest.NewServer(m.Mux)
+}
+
+// ShutdownServer creates the mock server
+func (m *SDMock) ShutdownServer() {
+	m.Server.Close()
+}
+
+const tokenID = "3c9a75a2-24fd-4508-b4f2-11f18aa97411"
+
+// HandleDropletsList mocks droplet list.
+func (m *SDMock) HandleDropletsList() {
+	m.Mux.HandleFunc("/v2/droplets", func(w http.ResponseWriter, r *http.Request) {
+		if r.Header.Get("Authorization") != fmt.Sprintf("Bearer %s", tokenID) {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+
+		w.Header().Add("content-type", "application/json; charset=utf-8")
+		w.Header().Add("ratelimit-limit", "1200")
+		w.Header().Add("ratelimit-remaining", "965")
+		w.Header().Add("ratelimit-reset", "1415984218")
+		w.WriteHeader(http.StatusAccepted)
+
+		page := 1
+		if pageQuery, ok := r.URL.Query()["page"]; ok {
+			var err error
+			page, err = strconv.Atoi(pageQuery[0])
+			if err != nil {
+				panic(err)
+			}
+		}
+		fmt.Fprint(w, []string{`
+{
+  "droplets": [
+    {
+      "id": 3164444,
+      "name": "example.com",
+      "memory": 1024,
+      "vcpus": 1,
+      "disk": 25,
+      "locked": false,
+      "status": "active",
+      "kernel": {
+        "id": 2233,
+        "name": "Ubuntu 14.04 x64 vmlinuz-3.13.0-37-generic",
+        "version": "3.13.0-37-generic"
+      },
+      "created_at": "2014-11-14T16:29:21Z",
+      "features": [
+        "backups",
+        "ipv6",
+        "virtio"
+      ],
+      "backup_ids": [
+        7938002
+      ],
+      "snapshot_ids": [
+
+      ],
+      "image": {
+        "id": 6918990,
+        "name": "14.04 x64",
+        "distribution": "Ubuntu",
+        "slug": "ubuntu-16-04-x64",
+        "public": true,
+        "regions": [
+          "nyc1",
+          "ams1",
+          "sfo1",
+          "nyc2",
+          "ams2",
+          "sgp1",
+          "lon1",
+          "nyc3",
+          "ams3",
+          "nyc3"
+        ],
+        "created_at": "2014-10-17T20:24:33Z",
+        "type": "snapshot",
+        "min_disk_size": 20,
+        "size_gigabytes": 2.34
+      },
+      "volume_ids": [
+
+      ],
+      "size": {
+      },
+      "size_slug": "s-1vcpu-1gb",
+      "networks": {
+        "v4": [
+          {
+            "ip_address": "104.236.32.182",
+            "netmask": "255.255.192.0",
+            "gateway": "104.236.0.1",
+            "type": "public"
+          }
+        ],
+        "v6": [
+          {
+            "ip_address": "2604:A880:0800:0010:0000:0000:02DD:4001",
+            "netmask": 64,
+            "gateway": "2604:A880:0800:0010:0000:0000:0000:0001",
+            "type": "public"
+          }
+        ]
+      },
+      "region": {
+        "name": "New York 3",
+        "slug": "nyc3",
+        "sizes": [
+
+        ],
+        "features": [
+          "virtio",
+          "private_networking",
+          "backups",
+          "ipv6",
+          "metadata"
+        ],
+        "available": null
+      },
+      "tags": [
+
+      ],
+      "vpc_uuid": "f9b0769c-e118-42fb-a0c4-fed15ef69662"
+    },
+    {
+      "id": 3164494,
+      "name": "prometheus",
+      "memory": 1024,
+      "vcpus": 1,
+      "disk": 25,
+      "locked": false,
+      "status": "active",
+      "kernel": {
+        "id": 2233,
+        "name": "Ubuntu 14.04 x64 vmlinuz-3.13.0-37-generic",
+        "version": "3.13.0-37-generic"
+      },
+      "created_at": "2014-11-14T16:36:31Z",
+      "features": [
+        "virtio"
+      ],
+      "backup_ids": [
+
+      ],
+      "snapshot_ids": [
+        7938206
+      ],
+      "image": {
+        "id": 6918990,
+        "name": "14.04 x64",
+        "distribution": "Ubuntu",
+        "slug": "ubuntu-16-04-x64",
+        "public": true,
+        "regions": [
+          "nyc1",
+          "ams1",
+          "sfo1",
+          "nyc2",
+          "ams2",
+          "sgp1",
+          "lon1",
+          "nyc3",
+          "ams3",
+          "nyc3"
+        ],
+        "created_at": "2014-10-17T20:24:33Z",
+        "type": "snapshot",
+        "min_disk_size": 20,
+        "size_gigabytes": 2.34
+      },
+      "volume_ids": [
+
+      ],
+      "size": {
+      },
+      "size_slug": "s-1vcpu-1gb",
+      "networks": {
+        "v4": [
+          {
+            "ip_address": "104.131.186.241",
+            "netmask": "255.255.240.0",
+            "gateway": "104.131.176.1",
+            "type": "public"
+          }
+        ]
+      },
+      "region": {
+        "name": "New York 3",
+        "slug": "nyc3",
+        "sizes": [
+          "s-1vcpu-1gb",
+          "s-1vcpu-2gb",
+          "s-1vcpu-3gb",
+          "s-2vcpu-2gb",
+          "s-3vcpu-1gb",
+          "s-2vcpu-4gb",
+          "s-4vcpu-8gb",
+          "s-6vcpu-16gb",
+          "s-8vcpu-32gb",
+          "s-12vcpu-48gb",
+          "s-16vcpu-64gb",
+          "s-20vcpu-96gb",
+          "s-24vcpu-128gb",
+          "s-32vcpu-192gb"
+        ],
+        "features": [
+          "virtio",
+          "private_networking",
+          "backups",
+          "ipv6",
+          "metadata"
+        ],
+        "available": true
+      },
+      "tags": [
+        "monitor"
+      ],
+      "vpc_uuid": "f9b0769c-e118-42fb-a0c4-fed15ef69662"
+    }
+  ],
+  "links": {
+    "pages": {
+      "next": "https://api.digitalocean.com/v2/droplets?page=2&per_page=2",
+      "last": "https://api.digitalocean.com/v2/droplets?page=2&per_page=2"
+    }
+  },
+  "meta": {
+    "total": 4
+  }
+}
+	`,
+			`
+{
+  "droplets": [
+    {
+      "id": 175072239,
+      "name": "prometheus-demo-old",
+      "memory": 1024,
+      "vcpus": 1,
+      "disk": 25,
+      "locked": false,
+      "status": "off",
+      "kernel": null,
+      "created_at": "2020-01-10T16:47:39Z",
+      "features": [
+        "ipv6",
+        "private_networking"
+      ],
+      "backup_ids": [],
+      "next_backup_window": null,
+      "snapshot_ids": [],
+      "image": {
+        "id": 53893572,
+        "name": "18.04.3 (LTS) x64",
+        "distribution": "Ubuntu",
+        "slug": "ubuntu-18-04-x64",
+        "public": true,
+        "regions": [
+          "nyc3",
+          "nyc1",
+          "sfo1",
+          "nyc2",
+          "ams2",
+          "sgp1",
+          "lon1",
+          "nyc3",
+          "ams3",
+          "fra1",
+          "tor1",
+          "sfo2",
+          "blr1",
+          "sfo3"
+        ],
+        "created_at": "2019-10-22T01:38:19Z",
+        "min_disk_size": 20,
+        "type": "base",
+        "size_gigabytes": 2.36,
+        "description": "Ubuntu 18.04 x64 20191022",
+        "tags": [],
+        "status": "available"
+      },
+      "volume_ids": [],
+      "size": {
+        "slug": "s-1vcpu-1gb",
+        "memory": 1024,
+        "vcpus": 1,
+        "disk": 25,
+        "transfer": 1,
+        "price_monthly": 5,
+        "price_hourly": 0.00744,
+        "regions": [
+          "ams2",
+          "ams3",
+          "blr1",
+          "fra1",
+          "lon1",
+          "nyc1",
+          "nyc2",
+          "nyc3",
+          "sfo1",
+          "sfo2",
+          "sfo3",
+          "sgp1",
+          "tor1"
+        ],
+        "available": true
+      },
+      "size_slug": "s-1vcpu-1gb",
+      "networks": {
+        "v4": [
+          {
+            "ip_address": "167.172.111.118",
+            "netmask": "255.255.240.0",
+            "gateway": "167.172.176.1",
+            "type": "public"
+          },
+          {
+            "ip_address": "10.135.64.211",
+            "netmask": "255.255.0.0",
+            "gateway": "10.135.0.1",
+            "type": "private"
+          }
+        ],
+        "v6": [
+        ]
+      },
+      "region": {
+        "name": "Frankfurt 1",
+        "slug": "fra1",
+        "features": [
+          "private_networking",
+          "backups",
+          "ipv6",
+          "metadata",
+          "install_agent",
+          "storage",
+          "image_transfer"
+        ],
+        "available": true,
+        "sizes": [
+          "s-1vcpu-1gb",
+          "512mb",
+          "s-1vcpu-2gb",
+          "1gb",
+          "s-3vcpu-1gb",
+          "s-2vcpu-2gb",
+          "s-1vcpu-3gb",
+          "s-2vcpu-4gb",
+          "2gb",
+          "s-4vcpu-8gb",
+          "m-1vcpu-8gb",
+          "c-2",
+          "4gb",
+          "g-2vcpu-8gb",
+          "gd-2vcpu-8gb",
+          "m-16gb",
+          "s-6vcpu-16gb",
+          "c-4",
+          "8gb",
+          "m-2vcpu-16gb",
+          "m3-2vcpu-16gb",
+          "g-4vcpu-16gb",
+          "gd-4vcpu-16gb",
+          "m6-2vcpu-16gb",
+          "m-32gb",
+          "s-8vcpu-32gb",
+          "c-8",
+          "16gb",
+          "m-4vcpu-32gb",
+          "m3-4vcpu-32gb",
+          "g-8vcpu-32gb",
+          "s-12vcpu-48gb",
+          "gd-8vcpu-32gb",
+          "m6-4vcpu-32gb",
+          "m-64gb",
+          "s-16vcpu-64gb",
+          "c-16",
+          "32gb",
+          "m-8vcpu-64gb",
+          "m3-8vcpu-64gb",
+          "g-16vcpu-64gb",
+          "s-20vcpu-96gb",
+          "48gb",
+          "gd-16vcpu-64gb",
+          "m6-8vcpu-64gb",
+          "m-128gb",
+          "s-24vcpu-128gb",
+          "c-32",
+          "64gb",
+          "m-16vcpu-128gb",
+          "m3-16vcpu-128gb",
+          "s-32vcpu-192gb",
+          "m-24vcpu-192gb",
+          "m-224gb",
+          "m6-16vcpu-128gb",
+          "m3-24vcpu-192gb",
+          "m6-24vcpu-192gb"
+        ]
+      },
+      "tags": []
+    },
+    {
+      "id": 176011507,
+      "name": "prometheus-demo",
+      "memory": 1024,
+      "vcpus": 1,
+      "disk": 25,
+      "locked": false,
+      "status": "active",
+      "kernel": null,
+      "created_at": "2020-01-17T12:06:26Z",
+      "features": [
+        "ipv6",
+        "private_networking"
+      ],
+      "backup_ids": [],
+      "next_backup_window": null,
+      "snapshot_ids": [],
+      "image": {
+        "id": 53893572,
+        "name": "18.04.3 (LTS) x64",
+        "distribution": "Ubuntu",
+        "slug": "ubuntu-18-04-x64",
+        "public": true,
+        "regions": [
+          "nyc3",
+          "nyc1",
+          "sfo1",
+          "nyc2",
+          "ams2",
+          "sgp1",
+          "lon1",
+          "nyc3",
+          "ams3",
+          "fra1",
+          "tor1",
+          "sfo2",
+          "blr1",
+          "sfo3"
+        ],
+        "created_at": "2019-10-22T01:38:19Z",
+        "min_disk_size": 20,
+        "type": "base",
+        "size_gigabytes": 2.36,
+        "description": "Ubuntu 18.04 x64 20191022",
+        "tags": [],
+        "status": "available"
+      },
+      "volume_ids": [],
+      "size": {
+        "slug": "s-1vcpu-1gb",
+        "memory": 1024,
+        "vcpus": 1,
+        "disk": 25,
+        "transfer": 1,
+        "price_monthly": 5,
+        "price_hourly": 0.00744,
+        "regions": [
+          "ams2",
+          "ams3",
+          "blr1",
+          "fra1",
+          "lon1",
+          "nyc1",
+          "nyc2",
+          "nyc3",
+          "sfo1",
+          "sfo2",
+          "sfo3",
+          "sgp1",
+          "tor1"
+        ],
+        "available": true
+      },
+      "size_slug": "s-1vcpu-1gb",
+      "networks": {
+        "v4": [
+          {
+            "ip_address": "138.65.56.69",
+            "netmask": "255.255.240.0",
+            "gateway": "138.65.64.1",
+            "type": "public"
+          },
+          {
+            "ip_address": "154.245.26.111",
+            "netmask": "255.255.252.0",
+            "gateway": "154.245.24.1",
+            "type": "public"
+          },
+          {
+            "ip_address": "10.135.64.212",
+            "netmask": "255.255.0.0",
+            "gateway": "10.135.0.1",
+            "type": "private"
+          }
+        ],
+        "v6": [
+          {
+            "ip_address": "2a03:b0c0:3:f0::cf2:4",
+            "netmask": 64,
+            "gateway": "2a03:b0c0:3:f0::1",
+            "type": "public"
+          }
+        ]
+      },
+      "region": {
+        "name": "Frankfurt 1",
+        "slug": "fra1",
+        "features": [
+          "private_networking",
+          "backups",
+          "ipv6",
+          "metadata",
+          "install_agent",
+          "storage",
+          "image_transfer"
+        ],
+        "available": true,
+        "sizes": [
+          "s-1vcpu-1gb",
+          "512mb",
+          "s-1vcpu-2gb",
+          "1gb",
+          "s-3vcpu-1gb",
+          "s-2vcpu-2gb",
+          "s-1vcpu-3gb",
+          "s-2vcpu-4gb",
+          "2gb",
+          "s-4vcpu-8gb",
+          "m-1vcpu-8gb",
+          "c-2",
+          "4gb",
+          "g-2vcpu-8gb",
+          "gd-2vcpu-8gb",
+          "m-16gb",
+          "s-6vcpu-16gb",
+          "c-4",
+          "8gb",
+          "m-2vcpu-16gb",
+          "m3-2vcpu-16gb",
+          "g-4vcpu-16gb",
+          "gd-4vcpu-16gb",
+          "m6-2vcpu-16gb",
+          "m-32gb",
+          "s-8vcpu-32gb",
+          "c-8",
+          "16gb",
+          "m-4vcpu-32gb",
+          "m3-4vcpu-32gb",
+          "g-8vcpu-32gb",
+          "s-12vcpu-48gb",
+          "gd-8vcpu-32gb",
+          "m6-4vcpu-32gb",
+          "m-64gb",
+          "s-16vcpu-64gb",
+          "c-16",
+          "32gb",
+          "m-8vcpu-64gb",
+          "m3-8vcpu-64gb",
+          "g-16vcpu-64gb",
+          "s-20vcpu-96gb",
+          "48gb",
+          "gd-16vcpu-64gb",
+          "m6-8vcpu-64gb",
+          "m-128gb",
+          "s-24vcpu-128gb",
+          "c-32",
+          "64gb",
+          "m-16vcpu-128gb",
+          "m3-16vcpu-128gb",
+          "s-32vcpu-192gb",
+          "m-24vcpu-192gb",
+          "m-224gb",
+          "m6-16vcpu-128gb",
+          "m3-24vcpu-192gb",
+          "m6-24vcpu-192gb"
+        ]
+      },
+      "tags": []
+    }
+  ],
+  "links": {
+    "pages": {
+      "first": "https://api.digitalocean.com/v2/droplets?page=1&per_page=2",
+      "prev": "https://api.digitalocean.com/v2/droplets?page=1&per_page=2"
+    }
+  },
+  "meta": {
+    "total": 4
+  }
+}
+`,
+		}[page-1],
+		)
+	})
+}

discovery/ec2/ec2.go

@@ -38,6 +38,7 @@ import (
 
 const (
 	ec2Label                  = model.MetaLabelPrefix + "ec2_"
+	ec2LabelAMI               = ec2Label + "ami"
 	ec2LabelAZ                = ec2Label + "availability_zone"
 	ec2LabelArch              = ec2Label + "architecture"
 	ec2LabelInstanceID        = ec2Label + "instance_id"
@@ -212,6 +213,7 @@ func (d *Discovery) refresh(ctx context.Context) ([]*targetgroup.Group, error) {
 					labels[ec2LabelPublicDNS] = model.LabelValue(*inst.PublicDnsName)
 				}
 
+				labels[ec2LabelAMI] = model.LabelValue(*inst.ImageId)
 				labels[ec2LabelAZ] = model.LabelValue(*inst.Placement.AvailabilityZone)
 				labels[ec2LabelInstanceState] = model.LabelValue(*inst.State.Name)
 				labels[ec2LabelInstanceType] = model.LabelValue(*inst.InstanceType)

discovery/kubernetes/kubernetes_test.go

@@ -107,14 +107,18 @@ func (d k8sDiscoveryTest) Run(t *testing.T) {
 // readResultWithTimeout reads all targegroups from channel with timeout.
 // It merges targegroups by source and sends the result to result channel.
 func readResultWithTimeout(t *testing.T, ch <-chan []*targetgroup.Group, max int, timeout time.Duration, resChan chan<- map[string]*targetgroup.Group) {
-	allTgs := make([][]*targetgroup.Group, 0)
+	res := make(map[string]*targetgroup.Group)
-
 Loop:
 	for {
 		select {
 		case tgs := <-ch:
-			allTgs = append(allTgs, tgs)
+			for _, tg := range tgs {
-			if len(allTgs) == max {
+				if tg == nil {
+					continue
+				}
+				res[tg.Source] = tg
+			}
+			if len(res) == max {
 				// Reached max target groups we may get, break fast.
 				break Loop
 			}
@@ -122,21 +126,11 @@ Loop:
 			// Because we use queue, an object that is created then
 			// deleted or updated may be processed only once.
 			// So possibly we may skip events, timed out here.
-			t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(allTgs), max)
+			t.Logf("timed out, got %d (max: %d) items, some events are skipped", len(res), max)
 			break Loop
 		}
 	}
 
-	// Merge by source and sent it to channel.
-	res := make(map[string]*targetgroup.Group)
-	for _, tgs := range allTgs {
-		for _, tg := range tgs {
-			if tg == nil {
-				continue
-			}
-			res[tg.Source] = tg
-		}
-	}
 	resChan <- res
 }
 

discovery/manager.go

@@ -29,6 +29,7 @@ import (
 
 	"github.com/prometheus/prometheus/discovery/azure"
 	"github.com/prometheus/prometheus/discovery/consul"
+	"github.com/prometheus/prometheus/discovery/digitalocean"
 	"github.com/prometheus/prometheus/discovery/dns"
 	"github.com/prometheus/prometheus/discovery/ec2"
 	"github.com/prometheus/prometheus/discovery/file"
@@ -369,6 +370,11 @@ func (m *Manager) registerProviders(cfg sd_config.ServiceDiscoveryConfig, setNam
 			return consul.NewDiscovery(c, log.With(m.logger, "discovery", "consul"))
 		})
 	}
+	for _, c := range cfg.DigitalOceanSDConfigs {
+		add(c, func() (Discoverer, error) {
+			return digitalocean.NewDiscovery(c, log.With(m.logger, "discovery", "digitalocean"))
+		})
+	}
 	for _, c := range cfg.MarathonSDConfigs {
 		add(c, func() (Discoverer, error) {
 			return marathon.NewDiscovery(*c, log.With(m.logger, "discovery", "marathon"))

docs/configuration/configuration.md

@@ -173,7 +173,7 @@ basic_auth:
 
 # Sets the `Authorization` header on every scrape request with the bearer token
 # read from the configured file. It is mutually exclusive with `bearer_token`.
-[ bearer_token_file: /path/to/bearer/token/file ]
+[ bearer_token_file: <filename> ]
 
 # Configures the scrape request's TLS settings.
 tls_config:
@@ -190,6 +190,10 @@ azure_sd_configs:
 consul_sd_configs:
   [ - <consul_sd_config> ... ]
 
+# List of DigitalOcean service discovery configurations.
+digitalocean_sd_configs:
+  [ - <digitalocean_sd_config> ... ]
+
 # List of DNS service discovery configurations.
 dns_sd_configs:
   [ - <dns_sd_config> ... ]
@@ -386,6 +390,60 @@ users with thousands of services it can be more efficient to use the Consul API
 directly which has basic support for filtering nodes (currently by node
 metadata and a single tag).
 
+### `<digitalocean_sd_config>`
+
+DigitalOcean SD configurations allow retrieving scrape targets from [DigitalOcean's](https://www.digitalocean.com/)
+Droplets API.
+This service discovery uses the public IPv4 address by default, by that can be
+changed with relabelling, as demonstrated in [the Prometheus digitalocean-sd
+configuration file](/documentation/examples/prometheus-digitalocean.yml).
+
+The following meta labels are available on targets during [relabeling](#relabel_config):
+
+* `__meta_digitalocean_droplet_id`: the id of the droplet
+* `__meta_digitalocean_droplet_name`: the name of the droplet
+* `__meta_digitalocean_image`: the image name of the droplet
+* `__meta_digitalocean_private_ipv4`: the private IPv4 of the droplet
+* `__meta_digitalocean_public_ipv4`: the public IPv4 of the droplet
+* `__meta_digitalocean_public_ipv6`: the public IPv6 of the droplet
+* `__meta_digitalocean_region`: the region of the droplet
+* `__meta_digitalocean_size`: the size of the droplet
+* `__meta_digitalocean_status`: the status of the droplet
+* `__meta_digitalocean_features`: the comma-separated list of features of the droplet
+* `__meta_digitalocean_tags`: the comma-separated list of tags of the droplet
+
+```yaml
+# Authentication information used to authenticate to the API server.
+# Note that `basic_auth`, `bearer_token` and `bearer_token_file` options are
+# mutually exclusive.
+# password and password_file are mutually exclusive.
+
+# Optional HTTP basic authentication information, not currently supported by DigitalOcean.
+basic_auth:
+  [ username: <string> ]
+  [ password: <secret> ]
+  [ password_file: <string> ]
+
+# Optional bearer token authentication information.
+[ bearer_token: <secret> ]
+
+# Optional bearer token file authentication information.
+[ bearer_token_file: <filename> ]
+
+# Optional proxy URL.
+[ proxy_url: <string> ]
+
+# TLS configuration.
+tls_config:
+  [ <tls_config> ]
+
+# The port to scrape metrics from.
+[ port: <int> | default = 80 ]
+
+# The time after which the droplets are refreshed.
+[ refresh_interval: <duration> | default = 60s ]
+```
+
 ### `<dns_sd_config>`
 
 A DNS-based service discovery configuration allows specifying a set of DNS
@@ -426,6 +484,7 @@ the public IP address with relabeling.
 
 The following meta labels are available on targets during [relabeling](#relabel_config):
 
+* `__meta_ec2_ami`: the EC2 Amazon Machine Image
 * `__meta_ec2_architecture`: the architecture of the instance
 * `__meta_ec2_availability_zone`: the availability zone in which the instance is running
 * `__meta_ec2_instance_id`: the EC2 instance ID
@@ -872,7 +931,7 @@ namespaces:
 # if you just want to monitor small subset of pods in large cluster it's recommended to use selectors.
 # Decision, if selectors should be used or not depends on the particular situation.
 [ selectors:
-  [ - role: <role>  
+  [ - role: <role>
     [ label: <string> ]
     [ field: <string> ] ]]
 ```
@@ -941,7 +1000,7 @@ basic_auth:
 # Sets the `Authorization` header on every request with the bearer token
 # read from the configured file. It is mutually exclusive with `bearer_token` and other authentication mechanisms.
 # NOTE: The current version of DC/OS marathon (v1.11.0) does not support standard Bearer token authentication. Use `auth_token_file` instead.
-[ bearer_token_file: /path/to/bearer/token/file ]
+[ bearer_token_file: <filename> ]
 
 # TLS configuration for connecting to marathon servers
 tls_config:
@@ -1236,7 +1295,7 @@ basic_auth:
 
 # Sets the `Authorization` header on every request with the bearer token
 # read from the configured file. It is mutually exclusive with `bearer_token`.
-[ bearer_token_file: /path/to/bearer/token/file ]
+[ bearer_token_file: <filename> ]
 
 # Configures the scrape request's TLS settings.
 tls_config:
@@ -1337,7 +1396,7 @@ basic_auth:
 
 # Sets the `Authorization` header on every remote write request with the bearer token
 # read from the configured file. It is mutually exclusive with `bearer_token`.
-[ bearer_token_file: /path/to/bearer/token/file ]
+[ bearer_token_file: <filename> ]
 
 # Configures the remote write request's TLS settings.
 tls_config:
@@ -1409,7 +1468,7 @@ basic_auth:
 
 # Sets the `Authorization` header on every remote read request with the bearer token
 # read from the configured file. It is mutually exclusive with `bearer_token`.
-[ bearer_token_file: /path/to/bearer/token/file ]
+[ bearer_token_file: <filename> ]
 
 # Configures the remote read request's TLS settings.
 tls_config:

docs/storage.md

@@ -63,7 +63,7 @@ Prometheus has several flags that allow configuring the local storage. The most
 * `--storage.tsdb.retention.time`: This determines when to remove old data. Defaults to `15d`. Overrides `storage.tsdb.retention` if this flag is set to anything other than default.
 * `--storage.tsdb.retention.size`: [EXPERIMENTAL] This determines the maximum number of bytes that storage blocks can use (note that this does not include the WAL size, which can be substantial). The oldest data will be removed first. Defaults to `0` or disabled. This flag is experimental and can be changed in future releases. Units supported: B, KB, MB, GB, TB, PB, EB. Ex: "512MB"
 * `--storage.tsdb.retention`: This flag has been deprecated in favour of `storage.tsdb.retention.time`.
-* `--storage.tsdb.wal-compression`: This flag enables compression of the write-ahead log (WAL). Depending on your data, you can expect the WAL size to be halved with little extra cpu load. Note that if you enable this flag and subsequently downgrade Prometheus to a version below 2.11.0 you will need to delete your WAL as it will be unreadable.
+* `--storage.tsdb.wal-compression`: This flag enables compression of the write-ahead log (WAL). Depending on your data, you can expect the WAL size to be halved with little extra cpu load. This flag was introduced in 2.11.0 and enabled by default in 2.20.0. Note that once enabled, downgrading Prometheus to a version below 2.11.0 will require deleting the WAL.
 
 On average, Prometheus uses only around 1-2 bytes per sample. Thus, to plan the capacity of a Prometheus server, you can use the rough formula:
 

documentation/examples/prometheus-digitalocean.yml

@@ -0,0 +1,25 @@
+# A example scrape configuration for running Prometheus with
+# DigitalOcean.
+
+scrape_configs:
+  
+  # Make Prometheus scrape itself for metrics.
+  - job_name: 'prometheus'
+    static_configs:
+    - targets: ['localhost:9090']
+
+  # Discover Node Exporter instances to scrape.
+  - job_name: 'node'
+
+    digitalocean_sd_configs:
+      - bearer_token: "<replace with a Personal Access Token>"
+    relabel_configs:
+      # Only scrape targets that have a tag 'monitoring'.
+      - source_labels: [__meta_digitalocean_tags]
+        regex: '.*,monitoring,.*'
+        action: keep
+
+      # Use the public IPv6 address and port 9100 to scrape the target.
+      - source_labels: [__meta_digitalocean_public_ipv6]
+        target_label: __address__
+        replacement: '[$1]:9100'

documentation/prometheus-mixin/README.md

@@ -10,12 +10,12 @@ To use them, you need to have `jsonnet` (v0.13+) and `jb` installed. If you
 have a working Go development environment, it's easiest to run the following:
 ```bash
 $ go get github.com/google/go-jsonnet/cmd/jsonnet
+$ go get github.com/google/go-jsonnet/cmd/jsonnetfmt 
 $ go get github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
 ```
 
 _Note: The make targets `lint` and `fmt` need the `jsonnetfmt` binary, which is
-currently not included in the Go implementation of `jsonnet`. For the time
+available from [v.0.16.0](https://github.com/google/jsonnet/releases/tag/v0.16.0) in the Go implementation of `jsonnet`. If your jsonnet version is older than 0.16.0 you have to either upgrade or install the [C++ version of
-being, you have to install the [C++ version of
 jsonnetfmt](https://github.com/google/jsonnet) if you want to use `make lint`
 or `make fmt`._
 

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/cespare/xxhash v1.1.0
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b
+	github.com/digitalocean/godo v1.37.0
 	github.com/edsrzf/mmap-go v1.0.0
 	github.com/go-kit/kit v0.10.0
 	github.com/go-logfmt/logfmt v0.5.0

go.sum

@@ -150,6 +150,8 @@ github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZm
 github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ=
 github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b h1:Yqiad0+sloMPdd/0Fg22actpFx0dekpzt1xJmVNVkU0=
 github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/digitalocean/godo v1.37.0 h1:NEj5ne2cvLBHo1GJY1DNN/iEt9ipa72CMwwAjKEA530=
+github.com/digitalocean/godo v1.37.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2xcz76ulEJRU=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
@@ -362,6 +364,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g=

pkg/labels/labels_test.go

@@ -531,3 +531,84 @@ func TestLabels_Copy(t *testing.T) {
 func TestLabels_Map(t *testing.T) {
 	testutil.Equals(t, map[string]string{"aaa": "111", "bbb": "222"}, Labels{{"aaa", "111"}, {"bbb", "222"}}.Map())
 }
+
+func TestLabels_WithLabels(t *testing.T) {
+	testutil.Equals(t, Labels{{"aaa", "111"}, {"bbb", "222"}}, Labels{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}}.WithLabels("aaa", "bbb"))
+}
+
+func TestLabels_WithoutLabels(t *testing.T) {
+	testutil.Equals(t, Labels{{"aaa", "111"}}, Labels{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}}.WithoutLabels("bbb", "ccc"))
+	testutil.Equals(t, Labels{{"aaa", "111"}}, Labels{{"aaa", "111"}, {"bbb", "222"}, {MetricName, "333"}}.WithoutLabels("bbb"))
+}
+
+func TestLabels_FromStrings(t *testing.T) {
+	testutil.Equals(t, Labels{{"aaa", "111"}, {"bbb", "222"}}, FromStrings("aaa", "111", "bbb", "222"))
+}
+
+func TestBulider_NewBulider(t *testing.T) {
+	testutil.Equals(
+		t,
+		&Builder{
+			base: Labels{{"aaa", "111"}},
+			del:  []string{},
+			add:  []Label{},
+		},
+		NewBuilder(Labels{{"aaa", "111"}}),
+	)
+}
+
+func TestBuilder_Del(t *testing.T) {
+	testutil.Equals(
+		t,
+		&Builder{
+			del: []string{"bbb"},
+			add: []Label{{"aaa", "111"}, {"ccc", "333"}},
+		},
+		(&Builder{
+			del: []string{},
+			add: []Label{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}},
+		}).Del("bbb"),
+	)
+}
+
+func TestBuilder_Set(t *testing.T) {
+	testutil.Equals(
+		t,
+		&Builder{
+			base: Labels{{"aaa", "111"}},
+			del:  []string{},
+			add:  []Label{{"bbb", "222"}},
+		},
+		(&Builder{
+			base: Labels{{"aaa", "111"}},
+			del:  []string{},
+			add:  []Label{},
+		}).Set("bbb", "222"),
+	)
+
+	testutil.Equals(
+		t,
+		&Builder{
+			base: Labels{{"aaa", "111"}},
+			del:  []string{},
+			add:  []Label{{"bbb", "333"}},
+		},
+		(&Builder{
+			base: Labels{{"aaa", "111"}},
+			del:  []string{},
+			add:  []Label{{"bbb", "222"}},
+		}).Set("bbb", "333"),
+	)
+}
+
+func TestBuilder_Labels(t *testing.T) {
+	testutil.Equals(
+		t,
+		Labels{{"aaa", "111"}, {"ccc", "333"}, {"ddd", "444"}},
+		(&Builder{
+			base: Labels{{"aaa", "111"}, {"bbb", "222"}, {"ccc", "333"}},
+			del:  []string{"bbb"},
+			add:  []Label{{"ddd", "444"}},
+		}).Labels(),
+	)
+}

promql/engine.go

@@ -29,7 +29,7 @@ import (
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
-	opentracing "github.com/opentracing/opentracing-go"
+	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
@@ -418,7 +418,7 @@ func (ng *Engine) newTestQuery(f func(context.Context) error) Query {
 //
 // At this point per query only one EvalStmt is evaluated. Alert and record
 // statements are not handled by the Engine.
-func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, w storage.Warnings, err error) {
+func (ng *Engine) exec(ctx context.Context, q *query) (v parser.Value, ws storage.Warnings, err error) {
 	ng.metrics.currentQueries.Inc()
 	defer ng.metrics.currentQueries.Dec()
 
@@ -517,13 +517,9 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval
 	}
 	defer querier.Close()
 
-	warnings, err := ng.populateSeries(ctxPrepare, querier, s)
+	ng.populateSeries(querier, s)
 	prepareSpanTimer.Finish()
 
-	if err != nil {
-		return nil, warnings, err
-	}
-
 	evalSpanTimer, ctxInnerEval := query.stats.GetSpanTimer(ctx, stats.InnerEvalTime, ng.metrics.queryInnerEval)
 	// Instant evaluation. This is executed as a range evaluation with one step.
 	if s.Start == s.End && s.Interval == 0 {
@@ -539,7 +535,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval
 			lookbackDelta:       ng.lookbackDelta,
 		}
 
-		val, err := evaluator.Eval(s.Expr)
+		val, warnings, err := evaluator.Eval(s.Expr)
 		if err != nil {
 			return nil, warnings, err
 		}
@@ -588,7 +584,7 @@ func (ng *Engine) execEvalStmt(ctx context.Context, query *query, s *parser.Eval
 		logger:              ng.logger,
 		lookbackDelta:       ng.lookbackDelta,
 	}
-	val, err := evaluator.Eval(s.Expr)
+	val, warnings, err := evaluator.Eval(s.Expr)
 	if err != nil {
 		return nil, warnings, err
 	}
@@ -649,35 +645,29 @@ func (ng *Engine) findMinTime(s *parser.EvalStmt) time.Time {
 	return s.Start.Add(-maxOffset)
 }
 
-func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s *parser.EvalStmt) (storage.Warnings, error) {
+func (ng *Engine) populateSeries(querier storage.Querier, s *parser.EvalStmt) {
-	var (
+	// Whenever a MatrixSelector is evaluated, evalRange is set to the corresponding range.
-		// Whenever a MatrixSelector is evaluated, evalRange is set to the corresponding range.
+	// The evaluation of the VectorSelector inside then evaluates the given range and unsets
-		// The evaluation of the VectorSelector inside then evaluates the given range and unsets
+	// the variable.
-		// the variable.
+	var evalRange time.Duration
-		evalRange time.Duration
-		warnings  storage.Warnings
-		err       error
-	)
 
 	parser.Inspect(s.Expr, func(node parser.Node, path []parser.Node) error {
-		var set storage.SeriesSet
-		var wrn storage.Warnings
-		hints := &storage.SelectHints{
-			Start: timestamp.FromTime(s.Start),
-			End:   timestamp.FromTime(s.End),
-			Step:  durationToInt64Millis(s.Interval),
-		}
-
-		// We need to make sure we select the timerange selected by the subquery.
-		// TODO(gouthamve): cumulativeSubqueryOffset gives the sum of range and the offset
-		// we can optimise it by separating out the range and offsets, and subtracting the offsets
-		// from end also.
-		subqOffset := ng.cumulativeSubqueryOffset(path)
-		offsetMilliseconds := durationMilliseconds(subqOffset)
-		hints.Start = hints.Start - offsetMilliseconds
-
 		switch n := node.(type) {
 		case *parser.VectorSelector:
+			hints := &storage.SelectHints{
+				Start: timestamp.FromTime(s.Start),
+				End:   timestamp.FromTime(s.End),
+				Step:  durationToInt64Millis(s.Interval),
+			}
+
+			// We need to make sure we select the timerange selected by the subquery.
+			// TODO(gouthamve): cumulativeSubqueryOffset gives the sum of range and the offset
+			// we can optimise it by separating out the range and offsets, and subtracting the offsets
+			// from end also.
+			subqOffset := ng.cumulativeSubqueryOffset(path)
+			offsetMilliseconds := durationMilliseconds(subqOffset)
+			hints.Start = hints.Start - offsetMilliseconds
+
 			if evalRange == 0 {
 				hints.Start = hints.Start - durationMilliseconds(ng.lookbackDelta)
 			} else {
@@ -696,20 +686,12 @@ func (ng *Engine) populateSeries(ctx context.Context, querier storage.Querier, s
 				hints.End = hints.End - offsetMilliseconds
 			}
 
-			set, wrn, err = querier.Select(false, hints, n.LabelMatchers...)
+			n.UnexpandedSeriesSet = querier.Select(false, hints, n.LabelMatchers...)
-			warnings = append(warnings, wrn...)
-			if err != nil {
-				level.Error(ng.logger).Log("msg", "error selecting series set", "err", err)
-				return err
-			}
-			n.UnexpandedSeriesSet = set
-
 		case *parser.MatrixSelector:
 			evalRange = n.Range
 		}
 		return nil
 	})
-	return warnings, err
 }
 
 // extractFuncFromPath walks up the path and searches for the first instance of
@@ -743,34 +725,40 @@ func extractGroupsFromPath(p []parser.Node) (bool, []string) {
 	return false, nil
 }
 
-func checkForSeriesSetExpansion(ctx context.Context, expr parser.Expr) {
+func checkAndExpandSeriesSet(ctx context.Context, expr parser.Expr) (storage.Warnings, error) {
 	switch e := expr.(type) {
 	case *parser.MatrixSelector:
-		checkForSeriesSetExpansion(ctx, e.VectorSelector)
+		return checkAndExpandSeriesSet(ctx, e.VectorSelector)
 	case *parser.VectorSelector:
-		if e.Series == nil {
+		if e.Series != nil {
-			series, err := expandSeriesSet(ctx, e.UnexpandedSeriesSet)
+			return nil, nil
-			if err != nil {
-				panic(err)
-			} else {
-				e.Series = series
-			}
 		}
+		series, ws, err := expandSeriesSet(ctx, e.UnexpandedSeriesSet)
+		e.Series = series
+		return ws, err
 	}
+	return nil, nil
 }
 
-func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.Series, err error) {
+func expandSeriesSet(ctx context.Context, it storage.SeriesSet) (res []storage.Series, ws storage.Warnings, err error) {
 	for it.Next() {
 		select {
 		case <-ctx.Done():
-			return nil, ctx.Err()
+			return nil, nil, ctx.Err()
 		default:
 		}
 		res = append(res, it.At())
 	}
-	return res, it.Err()
+	return res, it.Warnings(), it.Err()
 }
 
+type errWithWarnings struct {
+	err      error
+	warnings storage.Warnings
+}
+
+func (e errWithWarnings) Error() string { return e.err.Error() }
+
 // An evaluator evaluates given expressions over given fixed timestamps. It
 // is attached to an engine through which it connects to a querier and reports
 // errors. On timeout or cancellation of its context it terminates.
@@ -799,26 +787,33 @@ func (ev *evaluator) error(err error) {
 }
 
 // recover is the handler that turns panics into returns from the top level of evaluation.
-func (ev *evaluator) recover(errp *error) {
+func (ev *evaluator) recover(ws *storage.Warnings, errp *error) {
 	e := recover()
 	if e == nil {
 		return
 	}
-	if err, ok := e.(runtime.Error); ok {
+
+	switch err := e.(type) {
+	case runtime.Error:
 		// Print the stack trace but do not inhibit the running application.
 		buf := make([]byte, 64<<10)
 		buf = buf[:runtime.Stack(buf, false)]
 
 		level.Error(ev.logger).Log("msg", "runtime panic in parser", "err", e, "stacktrace", string(buf))
 		*errp = errors.Wrap(err, "unexpected error")
-	} else {
+	case errWithWarnings:
+		*errp = err.err
+		*ws = append(*ws, err.warnings...)
+	default:
 		*errp = e.(error)
 	}
 }
 
-func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, err error) {
+func (ev *evaluator) Eval(expr parser.Expr) (v parser.Value, ws storage.Warnings, err error) {
-	defer ev.recover(&err)
+	defer ev.recover(&ws, &err)
-	return ev.eval(expr), nil
+
+	v, ws = ev.eval(expr)
+	return v, ws, nil
 }
 
 // EvalNodeHelper stores extra information and caches for evaluating a single node across steps.
@@ -884,17 +879,20 @@ func (enh *EvalNodeHelper) signatureFunc(on bool, names ...string) func(labels.L
 // the given function with the values computed for each expression at that
 // step.  The return value is the combination into time series of all the
 // function call results.
-func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, exprs ...parser.Expr) Matrix {
+func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) (Vector, storage.Warnings), exprs ...parser.Expr) (Matrix, storage.Warnings) {
 	numSteps := int((ev.endTimestamp-ev.startTimestamp)/ev.interval) + 1
 	matrixes := make([]Matrix, len(exprs))
 	origMatrixes := make([]Matrix, len(exprs))
 	originalNumSamples := ev.currentSamples
 
+	var warnings storage.Warnings
 	for i, e := range exprs {
 		// Functions will take string arguments from the expressions, not the values.
 		if e != nil && e.Type() != parser.ValueTypeString {
 			// ev.currentSamples will be updated to the correct value within the ev.eval call.
-			matrixes[i] = ev.eval(e).(Matrix)
+			val, ws := ev.eval(e)
+			warnings = append(warnings, ws...)
+			matrixes[i] = val.(Matrix)
 
 			// Keep a copy of the original point slices so that they
 			// can be returned to the pool.
@@ -946,11 +944,12 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e
 		}
 		// Make the function call.
 		enh.ts = ts
-		result := f(args, enh)
+		result, ws := f(args, enh)
 		if result.ContainsSameLabelset() {
 			ev.errorf("vector cannot contain metrics with the same labelset")
 		}
 		enh.out = result[:0] // Reuse result vector.
+		warnings = append(warnings, ws...)
 
 		ev.currentSamples += len(result)
 		// When we reset currentSamples to tempNumSamples during the next iteration of the loop it also
@@ -969,7 +968,7 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e
 				mat[i] = Series{Metric: s.Metric, Points: []Point{s.Point}}
 			}
 			ev.currentSamples = originalNumSamples + mat.TotalSamples()
-			return mat
+			return mat, warnings
 		}
 
 		// Add samples in output vector to output series.
@@ -1001,29 +1000,30 @@ func (ev *evaluator) rangeEval(f func([]parser.Value, *EvalNodeHelper) Vector, e
 		mat = append(mat, ss)
 	}
 	ev.currentSamples = originalNumSamples + mat.TotalSamples()
-	return mat
+	return mat, warnings
 }
 
 // evalSubquery evaluates given SubqueryExpr and returns an equivalent
 // evaluated MatrixSelector in its place. Note that the Name and LabelMatchers are not set.
-func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) *parser.MatrixSelector {
+func (ev *evaluator) evalSubquery(subq *parser.SubqueryExpr) (*parser.MatrixSelector, storage.Warnings) {
-	val := ev.eval(subq).(Matrix)
+	val, ws := ev.eval(subq)
+	mat := val.(Matrix)
 	vs := &parser.VectorSelector{
 		Offset: subq.Offset,
-		Series: make([]storage.Series, 0, len(val)),
+		Series: make([]storage.Series, 0, len(mat)),
 	}
 	ms := &parser.MatrixSelector{
 		Range:          subq.Range,
 		VectorSelector: vs,
 	}
-	for _, s := range val {
+	for _, s := range mat {
 		vs.Series = append(vs.Series, NewStorageSeries(s))
 	}
-	return ms
+	return ms, ws
 }
 
 // eval evaluates the given expression as the given AST expression node requires.
-func (ev *evaluator) eval(expr parser.Expr) parser.Value {
+func (ev *evaluator) eval(expr parser.Expr) (parser.Value, storage.Warnings) {
 	// This is the top-level evaluation method.
 	// Thus, we check for timeout/cancellation here.
 	if err := contextDone(ev.ctx, "expression evaluation"); err != nil {
@@ -1035,16 +1035,16 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 	case *parser.AggregateExpr:
 		unwrapParenExpr(&e.Param)
 		if s, ok := e.Param.(*parser.StringLiteral); ok {
-			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-				return ev.aggregation(e.Op, e.Grouping, e.Without, s.Val, v[0].(Vector), enh)
+				return ev.aggregation(e.Op, e.Grouping, e.Without, s.Val, v[0].(Vector), enh), nil
 			}, e.Expr)
 		}
-		return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+		return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
 			var param float64
 			if e.Param != nil {
 				param = v[0].(Vector)[0].V
 			}
-			return ev.aggregation(e.Op, e.Grouping, e.Without, param, v[1].(Vector), enh)
+			return ev.aggregation(e.Op, e.Grouping, e.Without, param, v[1].(Vector), enh), nil
 		}, e.Param, e.Expr)
 
 	case *parser.Call:
@@ -1056,15 +1056,19 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 			// a vector selector.
 			vs, ok := e.Args[0].(*parser.VectorSelector)
 			if ok {
-				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-					return call([]parser.Value{ev.vectorSelector(vs, enh.ts)}, e.Args, enh)
+					val, ws := ev.vectorSelector(vs, enh.ts)
+					return call([]parser.Value{val}, e.Args, enh), ws
 				})
 			}
 		}
 
 		// Check if the function has a matrix argument.
-		var matrixArgIndex int
+		var (
-		var matrixArg bool
+			matrixArgIndex int
+			matrixArg      bool
+			warnings       storage.Warnings
+		)
 		for i := range e.Args {
 			unwrapParenExpr(&e.Args[i])
 			a := e.Args[i]
@@ -1078,14 +1082,16 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 				matrixArgIndex = i
 				matrixArg = true
 				// Replacing parser.SubqueryExpr with parser.MatrixSelector.
-				e.Args[i] = ev.evalSubquery(subq)
+				val, ws := ev.evalSubquery(subq)
+				e.Args[i] = val
+				warnings = append(warnings, ws...)
 				break
 			}
 		}
 		if !matrixArg {
 			// Does not have a matrix argument.
-			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-				return call(v, e.Args, enh)
+				return call(v, e.Args, enh), warnings
 			}, e.Args...)
 		}
 
@@ -1095,16 +1101,22 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 		otherInArgs := make([]Vector, len(e.Args))
 		for i, e := range e.Args {
 			if i != matrixArgIndex {
-				otherArgs[i] = ev.eval(e).(Matrix)
+				val, ws := ev.eval(e)
+				otherArgs[i] = val.(Matrix)
 				otherInArgs[i] = Vector{Sample{}}
 				inArgs[i] = otherInArgs[i]
+				warnings = append(warnings, ws...)
 			}
 		}
 
 		sel := e.Args[matrixArgIndex].(*parser.MatrixSelector)
 		selVS := sel.VectorSelector.(*parser.VectorSelector)
 
-		checkForSeriesSetExpansion(ev.ctx, sel)
+		ws, err := checkAndExpandSeriesSet(ev.ctx, sel)
+		warnings = append(warnings, ws...)
+		if err != nil {
+			ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), warnings})
+		}
 		mat := make(Matrix, 0, len(selVS.Series)) // Output matrix.
 		offset := durationMilliseconds(selVS.Offset)
 		selRange := durationMilliseconds(sel.Range)
@@ -1182,7 +1194,7 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 			// Iterate once to look for a complete series.
 			for _, s := range mat {
 				if len(s.Points) == steps {
-					return Matrix{}
+					return Matrix{}, warnings
 				}
 			}
 
@@ -1193,7 +1205,7 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 					found[p.T] = struct{}{}
 				}
 				if i > 0 && len(found) == steps {
-					return Matrix{}
+					return Matrix{}, warnings
 				}
 			}
 
@@ -1209,20 +1221,21 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 					Metric: createLabelsForAbsentFunction(e.Args[0]),
 					Points: newp,
 				},
-			}
+			}, warnings
 		}
 
 		if mat.ContainsSameLabelset() {
 			ev.errorf("vector cannot contain metrics with the same labelset")
 		}
 
-		return mat
+		return mat, warnings
 
 	case *parser.ParenExpr:
 		return ev.eval(e.Expr)
 
 	case *parser.UnaryExpr:
-		mat := ev.eval(e.Expr).(Matrix)
+		val, ws := ev.eval(e.Expr)
+		mat := val.(Matrix)
 		if e.Op == parser.SUB {
 			for i := range mat {
 				mat[i].Metric = dropMetricName(mat[i].Metric)
@@ -1234,53 +1247,56 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 				ev.errorf("vector cannot contain metrics with the same labelset")
 			}
 		}
-		return mat
+		return mat, ws
 
 	case *parser.BinaryExpr:
 		switch lt, rt := e.LHS.Type(), e.RHS.Type(); {
 		case lt == parser.ValueTypeScalar && rt == parser.ValueTypeScalar:
-			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
 				val := scalarBinop(e.Op, v[0].(Vector)[0].Point.V, v[1].(Vector)[0].Point.V)
-				return append(enh.out, Sample{Point: Point{V: val}})
+				return append(enh.out, Sample{Point: Point{V: val}}), nil
 			}, e.LHS, e.RHS)
 		case lt == parser.ValueTypeVector && rt == parser.ValueTypeVector:
 			switch e.Op {
 			case parser.LAND:
-				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-					return ev.VectorAnd(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh)
+					return ev.VectorAnd(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil
 				}, e.LHS, e.RHS)
 			case parser.LOR:
-				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-					return ev.VectorOr(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh)
+					return ev.VectorOr(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil
 				}, e.LHS, e.RHS)
 			case parser.LUNLESS:
-				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-					return ev.VectorUnless(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh)
+					return ev.VectorUnless(v[0].(Vector), v[1].(Vector), e.VectorMatching, enh), nil
 				}, e.LHS, e.RHS)
 			default:
-				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+				return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-					return ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, enh)
+					return ev.VectorBinop(e.Op, v[0].(Vector), v[1].(Vector), e.VectorMatching, e.ReturnBool, enh), nil
 				}, e.LHS, e.RHS)
 			}
 
 		case lt == parser.ValueTypeVector && rt == parser.ValueTypeScalar:
-			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-				return ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].Point.V}, false, e.ReturnBool, enh)
+				return ev.VectorscalarBinop(e.Op, v[0].(Vector), Scalar{V: v[1].(Vector)[0].Point.V}, false, e.ReturnBool, enh), nil
 			}, e.LHS, e.RHS)
 
 		case lt == parser.ValueTypeScalar && rt == parser.ValueTypeVector:
-			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+			return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-				return ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].Point.V}, true, e.ReturnBool, enh)
+				return ev.VectorscalarBinop(e.Op, v[1].(Vector), Scalar{V: v[0].(Vector)[0].Point.V}, true, e.ReturnBool, enh), nil
 			}, e.LHS, e.RHS)
 		}
 
 	case *parser.NumberLiteral:
-		return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) Vector {
+		return ev.rangeEval(func(v []parser.Value, enh *EvalNodeHelper) (Vector, storage.Warnings) {
-			return append(enh.out, Sample{Point: Point{V: e.Val}})
+			return append(enh.out, Sample{Point: Point{V: e.Val}}), nil
 		})
 
 	case *parser.VectorSelector:
-		checkForSeriesSetExpansion(ev.ctx, e)
+		ws, err := checkAndExpandSeriesSet(ev.ctx, e)
+		if err != nil {
+			ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws})
+		}
 		mat := make(Matrix, 0, len(e.Series))
 		it := storage.NewBuffer(durationMilliseconds(ev.lookbackDelta))
 		for i, s := range e.Series {
@@ -1307,9 +1323,8 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 			} else {
 				putPointSlice(ss.Points)
 			}
-
 		}
-		return mat
+		return mat, ws
 
 	case *parser.MatrixSelector:
 		if ev.startTimestamp != ev.endTimestamp {
@@ -1342,11 +1357,11 @@ func (ev *evaluator) eval(expr parser.Expr) parser.Value {
 			newEv.startTimestamp += newEv.interval
 		}
 
-		res := newEv.eval(e.Expr)
+		res, ws := newEv.eval(e.Expr)
 		ev.currentSamples = newEv.currentSamples
-		return res
+		return res, ws
 	case *parser.StringLiteral:
-		return String{V: e.Val, T: ev.startTimestamp}
+		return String{V: e.Val, T: ev.startTimestamp}, nil
 	}
 
 	panic(errors.Errorf("unhandled expression of type: %T", expr))
@@ -1357,13 +1372,12 @@ func durationToInt64Millis(d time.Duration) int64 {
 }
 
 // vectorSelector evaluates a *parser.VectorSelector expression.
-func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) Vector {
+func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) (Vector, storage.Warnings) {
-	checkForSeriesSetExpansion(ev.ctx, node)
+	ws, err := checkAndExpandSeriesSet(ev.ctx, node)
-
+	if err != nil {
-	var (
+		ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws})
-		vec = make(Vector, 0, len(node.Series))
+	}
-	)
+	vec := make(Vector, 0, len(node.Series))
-
 	it := storage.NewBuffer(durationMilliseconds(ev.lookbackDelta))
 	for i, s := range node.Series {
 		it.Reset(s.Iterator())
@@ -1381,7 +1395,7 @@ func (ev *evaluator) vectorSelector(node *parser.VectorSelector, ts int64) Vecto
 			ev.error(ErrTooManySamples(env))
 		}
 	}
-	return vec
+	return vec, ws
 }
 
 // vectorSelectorSingle evaluates a instant vector for the iterator of one time series.
@@ -1429,21 +1443,23 @@ func putPointSlice(p []Point) {
 }
 
 // matrixSelector evaluates a *parser.MatrixSelector expression.
-func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) Matrix {
+func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) (Matrix, storage.Warnings) {
-	checkForSeriesSetExpansion(ev.ctx, node)
-
-	vs := node.VectorSelector.(*parser.VectorSelector)
-
 	var (
+		vs = node.VectorSelector.(*parser.VectorSelector)
+
 		offset = durationMilliseconds(vs.Offset)
 		maxt   = ev.startTimestamp - offset
 		mint   = maxt - durationMilliseconds(node.Range)
 		matrix = make(Matrix, 0, len(vs.Series))
+
+		it = storage.NewBuffer(durationMilliseconds(node.Range))
 	)
+	ws, err := checkAndExpandSeriesSet(ev.ctx, node)
+	if err != nil {
+		ev.error(errWithWarnings{errors.Wrap(err, "expanding series"), ws})
+	}
 
-	it := storage.NewBuffer(durationMilliseconds(node.Range))
 	series := vs.Series
-
 	for i, s := range series {
 		if err := contextDone(ev.ctx, "expression evaluation"); err != nil {
 			ev.error(err)
@@ -1461,7 +1477,7 @@ func (ev *evaluator) matrixSelector(node *parser.MatrixSelector) Matrix {
 			putPointSlice(ss.Points)
 		}
 	}
-	return matrix
+	return matrix, ws
 }
 
 // matrixIterSlice populates a matrix vector covering the requested range for a

promql/engine_test.go

@@ -19,7 +19,6 @@ import (
 	"io/ioutil"
 	"os"
 	"sort"
-	"strings"
 	"testing"
 	"time"
 
@@ -117,9 +116,7 @@ func TestQueryTimeout(t *testing.T) {
 	testutil.NotOk(t, res.Err, "expected timeout error but got none")
 
 	var e ErrQueryTimeout
-	// TODO: when circleci-windows moves to go 1.13:
+	testutil.Assert(t, errors.As(res.Err, &e), "expected timeout error but got: %s", res.Err)
-	// testutil.Assert(t, errors.As(res.Err, &e), "expected timeout error but got: %s", res.Err)
-	testutil.Assert(t, strings.HasPrefix(res.Err.Error(), e.Error()), "expected timeout error but got: %s", res.Err)
 }
 
 const errQueryCanceled = ErrQueryCanceled("test statement execution")
@@ -175,8 +172,8 @@ type errQuerier struct {
 	err error
 }
 
-func (q *errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q *errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet {
-	return errSeriesSet{err: q.err}, nil, q.err
+	return errSeriesSet{err: q.err}
 }
 func (*errQuerier) LabelValues(string) ([]string, storage.Warnings, error) { return nil, nil, nil }
 func (*errQuerier) LabelNames() ([]string, storage.Warnings, error)        { return nil, nil, nil }
@@ -187,9 +184,10 @@ type errSeriesSet struct {
 	err error
 }
 
-func (errSeriesSet) Next() bool         { return false }
+func (errSeriesSet) Next() bool                   { return false }
-func (errSeriesSet) At() storage.Series { return nil }
+func (errSeriesSet) At() storage.Series           { return nil }
-func (e errSeriesSet) Err() error       { return e.err }
+func (e errSeriesSet) Err() error                 { return e.err }
+func (e errSeriesSet) Warnings() storage.Warnings { return nil }
 
 func TestQueryError(t *testing.T) {
 	opts := EngineOpts{
@@ -211,14 +209,14 @@ func TestQueryError(t *testing.T) {
 
 	res := vectorQuery.Exec(ctx)
 	testutil.NotOk(t, res.Err, "expected error on failed select but got none")
-	testutil.Equals(t, errStorage, res.Err)
+	testutil.Assert(t, errors.Is(res.Err, errStorage), "expected error doesn't match")
 
 	matrixQuery, err := engine.NewInstantQuery(queryable, "foo[1m]", time.Unix(1, 0))
 	testutil.Ok(t, err)
 
 	res = matrixQuery.Exec(ctx)
 	testutil.NotOk(t, res.Err, "expected error on failed select but got none")
-	testutil.Equals(t, errStorage, res.Err)
+	testutil.Assert(t, errors.Is(res.Err, errStorage), "expected error doesn't match")
 }
 
 // hintCheckerQuerier implements storage.Querier which checks the start and end times
@@ -234,7 +232,7 @@ type hintCheckerQuerier struct {
 	t *testing.T
 }
 
-func (q *hintCheckerQuerier) Select(_ bool, sp *storage.SelectHints, _ ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q *hintCheckerQuerier) Select(_ bool, sp *storage.SelectHints, _ ...*labels.Matcher) storage.SeriesSet {
 	testutil.Equals(q.t, q.start, sp.Start)
 	testutil.Equals(q.t, q.end, sp.End)
 	testutil.Equals(q.t, q.grouping, sp.Grouping)
@@ -242,7 +240,7 @@ func (q *hintCheckerQuerier) Select(_ bool, sp *storage.SelectHints, _ ...*label
 	testutil.Equals(q.t, q.selRange, sp.Range)
 	testutil.Equals(q.t, q.function, sp.Func)
 
-	return errSeriesSet{err: nil}, nil, nil
+	return errSeriesSet{err: nil}
 }
 func (*hintCheckerQuerier) LabelValues(string) ([]string, storage.Warnings, error) {
 	return nil, nil, nil
@@ -499,9 +497,7 @@ func TestEngineShutdown(t *testing.T) {
 	testutil.NotOk(t, res2.Err, "expected error on querying with canceled context but got none")
 
 	var e ErrQueryCanceled
-	// TODO: when circleci-windows moves to go 1.13:
+	testutil.Assert(t, errors.As(res2.Err, &e), "expected cancellation error but got: %s", res2.Err)
-	// testutil.Assert(t, errors.As(res2.Err, &e), "expected cancellation error but got: %s", res2.Err)
-	testutil.Assert(t, strings.HasPrefix(res2.Err.Error(), e.Error()), "expected cancellation error but got: %s", res2.Err)
 }
 
 func TestEngineEvalStmtTimestamps(t *testing.T) {
@@ -599,9 +595,8 @@ load 10s
 		}
 
 		testutil.Ok(t, res.Err)
-		testutil.Equals(t, c.Result, res.Value)
+		testutil.Equals(t, c.Result, res.Value, "query %q failed", c.Query)
 	}
-
 }
 
 func TestMaxQuerySamples(t *testing.T) {
@@ -831,7 +826,7 @@ load 10s
 
 		res := qry.Exec(test.Context())
 		testutil.Equals(t, c.Result.Err, res.Err)
-		testutil.Equals(t, c.Result.Value, res.Value)
+		testutil.Equals(t, c.Result.Value, res.Value, "query %q failed", c.Query)
 	}
 }
 
@@ -839,7 +834,7 @@ func TestRecoverEvaluatorRuntime(t *testing.T) {
 	ev := &evaluator{logger: log.NewNopLogger()}
 
 	var err error
-	defer ev.recover(&err)
+	defer ev.recover(nil, &err)
 
 	// Cause a runtime panic.
 	var a []int
@@ -862,7 +857,31 @@ func TestRecoverEvaluatorError(t *testing.T) {
 			t.Fatalf("wrong error message: %q, expected %q", err, e)
 		}
 	}()
-	defer ev.recover(&err)
+	defer ev.recover(nil, &err)
+
+	panic(e)
+}
+
+func TestRecoverEvaluatorErrorWithWarnings(t *testing.T) {
+	ev := &evaluator{logger: log.NewNopLogger()}
+	var err error
+	var ws storage.Warnings
+
+	warnings := storage.Warnings{errors.New("custom warning")}
+	e := errWithWarnings{
+		err:      errors.New("custom error"),
+		warnings: warnings,
+	}
+
+	defer func() {
+		if err.Error() != e.Error() {
+			t.Fatalf("wrong error message: %q, expected %q", err, e)
+		}
+		if len(ws) != len(warnings) && ws[0] != warnings[0] {
+			t.Fatalf("wrong warning message: %q, expected %q", ws[0], warnings[0])
+		}
+	}()
+	defer ev.recover(&ws, &err)
 
 	panic(e)
 }

promql/quantile.go

@@ -168,7 +168,7 @@ func coalesceBuckets(buckets buckets) buckets {
 
 func ensureMonotonic(buckets buckets) {
 	max := buckets[0].count
-	for i := range buckets[1:] {
+	for i := 1; i < len(buckets); i++ {
 		switch {
 		case buckets[i].count > max:
 			max = buckets[i].count

promql/test_test.go

@@ -133,8 +133,7 @@ func TestLazyLoader_WithSamplesTill(t *testing.T) {
 					}
 
 					// Get the series for the matcher.
-					ss, _, err := querier.Select(false, nil, matchers...)
+					ss := querier.Select(false, nil, matchers...)
-					testutil.Ok(t, err)
 					testutil.Assert(t, ss.Next(), "")
 					storageSeries := ss.At()
 					testutil.Assert(t, !ss.Next(), "Expecting only 1 series")

promql/testdata/histograms.test

@@ -151,21 +151,25 @@ eval instant at 50m histogram_quantile(0.5, rate(request_duration_seconds_bucket
 
 # A histogram with nonmonotonic bucket counts. This may happen when recording
 # rule evaluation or federation races scrape ingestion, causing some buckets
-# counts to be derived from fewer samples. The wrong answer we want to avoid
+# counts to be derived from fewer samples.
-# is for histogram_quantile(0.99, nonmonotonic_bucket) to return ~1000 instead
-# of 1.
 
 load 5m
-    nonmonotonic_bucket{le="0.1"}   0+1x10
+    nonmonotonic_bucket{le="0.1"}   0+2x10
-    nonmonotonic_bucket{le="1"}     0+9x10
+    nonmonotonic_bucket{le="1"}     0+1x10
-    nonmonotonic_bucket{le="10"}    0+8x10
+    nonmonotonic_bucket{le="10"}    0+5x10
-    nonmonotonic_bucket{le="100"}   0+8x10
+    nonmonotonic_bucket{le="100"}   0+4x10
     nonmonotonic_bucket{le="1000"}  0+9x10
-    nonmonotonic_bucket{le="+Inf"}  0+9x10
+    nonmonotonic_bucket{le="+Inf"}  0+8x10
 
 # Nonmonotonic buckets
+eval instant at 50m histogram_quantile(0.01, nonmonotonic_bucket)
+    {} 0.0045
+
+eval instant at 50m histogram_quantile(0.5, nonmonotonic_bucket)
+    {} 8.5
+
 eval instant at 50m histogram_quantile(0.99, nonmonotonic_bucket)
-    {} 0.989875
+    {} 979.75
 
 # Buckets with different representations of the same upper bound.
 eval instant at 50m histogram_quantile(0.5, rate(mixed_bucket[5m]))

rules/manager.go

@@ -707,12 +707,7 @@ func (g *Group) RestoreForState(ts time.Time) {
 				matchers = append(matchers, mt)
 			}
 
-			sset, err, _ := q.Select(false, nil, matchers...)
+			sset := q.Select(false, nil, matchers...)
-			if err != nil {
-				level.Error(g.logger).Log("msg", "Failed to restore 'for' state",
-					labels.AlertName, alertRule.Name(), "stage", "Select", "err", err)
-				return
-			}
 
 			seriesFound := false
 			var s storage.Series
@@ -727,6 +722,17 @@ func (g *Group) RestoreForState(ts time.Time) {
 				}
 			}
 
+			if err := sset.Err(); err != nil {
+				// Querier Warnings are ignored. We do not care unless we have an error.
+				level.Error(g.logger).Log(
+					"msg", "Failed to restore 'for' state",
+					labels.AlertName, alertRule.Name(),
+					"stage", "Select",
+					"err", err,
+				)
+				return
+			}
+
 			if !seriesFound {
 				return
 			}

rules/manager_test.go

@@ -563,9 +563,7 @@ func TestStaleness(t *testing.T) {
 	matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "a_plus_one")
 	testutil.Ok(t, err)
 
-	set, _, err := querier.Select(false, nil, matcher)
+	set := querier.Select(false, nil, matcher)
-	testutil.Ok(t, err)
-
 	samples, err := readSeriesSet(set)
 	testutil.Ok(t, err)
 
@@ -686,9 +684,7 @@ func TestDeletedRuleMarkedStale(t *testing.T) {
 	matcher, err := labels.NewMatcher(labels.MatchEqual, "l1", "v1")
 	testutil.Ok(t, err)
 
-	set, _, err := querier.Select(false, nil, matcher)
+	set := querier.Select(false, nil, matcher)
-	testutil.Ok(t, err)
-
 	samples, err := readSeriesSet(set)
 	testutil.Ok(t, err)
 
@@ -1107,9 +1103,7 @@ func countStaleNaN(t *testing.T, st storage.Storage) int {
 	matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "test_2")
 	testutil.Ok(t, err)
 
-	set, _, err := querier.Select(false, nil, matcher)
+	set := querier.Select(false, nil, matcher)
-	testutil.Ok(t, err)
-
 	samples, err := readSeriesSet(set)
 	testutil.Ok(t, err)
 

scrape/scrape.go

@@ -1140,7 +1140,7 @@ loop:
 
 		if ok {
 			err = app.AddFast(ce.ref, t, v)
-			sampleAdded, err = sl.checkAddError(ce, met, tp, err, &sampleLimitErr, &appErrs)
+			_, err = sl.checkAddError(ce, met, tp, err, &sampleLimitErr, &appErrs)
 			// In theory this should never happen.
 			if err == storage.ErrNotFound {
 				ok = false
@@ -1187,10 +1187,10 @@ loop:
 			}
 		}
 
-		// Increment added even if there's a sampleLimitErr so we correctly report the number of samples scraped.
+		// Increment added even if there's an error so we correctly report the
-		if sampleAdded || sampleLimitErr != nil {
+		// number of samples remaining after relabelling.
-			added++
+		added++
-		}
+
 	}
 	if sampleLimitErr != nil {
 		if err == nil {
@@ -1275,7 +1275,7 @@ const (
 	scrapeSeriesAddedMetricName  = "scrape_series_added" + "\xff"
 )
 
-func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, appended, seriesAdded int, scrapeErr error) (err error) {
+func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, added, seriesAdded int, scrapeErr error) (err error) {
 	sl.scraper.Report(start, duration, scrapeErr)
 
 	ts := timestamp.FromTime(start)
@@ -1302,7 +1302,7 @@ func (sl *scrapeLoop) report(start time.Time, duration time.Duration, scraped, a
 	if err = sl.addReportSample(app, scrapeSamplesMetricName, ts, float64(scraped)); err != nil {
 		return
 	}
-	if err = sl.addReportSample(app, samplesPostRelabelMetricName, ts, float64(appended)); err != nil {
+	if err = sl.addReportSample(app, samplesPostRelabelMetricName, ts, float64(added)); err != nil {
 		return
 	}
 	if err = sl.addReportSample(app, scrapeSeriesAddedMetricName, ts, float64(seriesAdded)); err != nil {

scrape/scrape_test.go

@@ -1340,7 +1340,7 @@ func TestScrapeLoopAppendGracefullyIfAmendOrOutOfOrderOrOutOfBounds(t *testing.T
 	}
 	testutil.Equals(t, want, app.result, "Appended samples not as expected")
 	testutil.Equals(t, 4, total)
-	testutil.Equals(t, 1, added)
+	testutil.Equals(t, 4, added)
 	testutil.Equals(t, 1, seriesAdded)
 }
 
@@ -1365,7 +1365,7 @@ func TestScrapeLoopOutOfBoundsTimeError(t *testing.T) {
 	now := time.Now().Add(20 * time.Minute)
 	total, added, seriesAdded, err := sl.append([]byte("normal 1\n"), "", now)
 	testutil.Equals(t, 1, total)
-	testutil.Equals(t, 0, added)
+	testutil.Equals(t, 1, added)
 	testutil.Equals(t, 0, seriesAdded)
 
 	testutil.Ok(t, err)
@@ -1617,9 +1617,9 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) {
 
 	q, err := s.Querier(ctx, time.Time{}.UnixNano(), 0)
 	testutil.Ok(t, err)
-	series, _, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*"))
+	series := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*"))
-	testutil.Ok(t, err)
 	testutil.Equals(t, false, series.Next(), "series found in tsdb")
+	testutil.Ok(t, series.Err())
 
 	// We add a good metric to check that it is recorded.
 	_, _, _, err = sl.append([]byte("test_metric{le=\"500\"} 1\n"), "", time.Time{})
@@ -1627,9 +1627,9 @@ func TestScrapeLoopDiscardDuplicateLabels(t *testing.T) {
 
 	q, err = s.Querier(ctx, time.Time{}.UnixNano(), 0)
 	testutil.Ok(t, err)
-	series, _, err = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "le", "500"))
+	series = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "le", "500"))
-	testutil.Ok(t, err)
 	testutil.Equals(t, true, series.Next(), "series not found in tsdb")
+	testutil.Ok(t, series.Err())
 	testutil.Equals(t, false, series.Next(), "more than one series found in tsdb")
 }
 
@@ -1663,9 +1663,9 @@ func TestScrapeLoopDiscardUnnamedMetrics(t *testing.T) {
 
 	q, err := s.Querier(ctx, time.Time{}.UnixNano(), 0)
 	testutil.Ok(t, err)
-	series, _, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*"))
+	series := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*"))
-	testutil.Ok(t, err)
 	testutil.Equals(t, false, series.Next(), "series found in tsdb")
+	testutil.Ok(t, series.Err())
 }
 
 func TestReusableConfig(t *testing.T) {

scripts/sync_repo_files.sh

@@ -7,10 +7,10 @@ set -uo pipefail
 
 git_mail="prometheus-team@googlegroups.com"
 git_user="prombot"
-branch="makefile_common"
+branch="repo_sync"
-commit_msg="makefile: update Makefile.common with newer version"
+commit_msg="Update common Prometheus files"
-pr_title="Synchronize Makefile.common from prometheus/prometheus"
+pr_title="Synchronize common files from prometheus/prometheus"
-pr_msg="Propagating changes from master Makefile.common located in prometheus/prometheus."
+pr_msg="Propagating changes from prometheus/prometheus default branch."
 orgs="prometheus prometheus-community"
 
 GITHUB_TOKEN="${GITHUB_TOKEN:-}"
@@ -19,11 +19,13 @@ if [ -z "${GITHUB_TOKEN}" ]; then
   exit 1
 fi
 
+# List of files that should be synced.
+SYNC_FILES="CODE_OF_CONDUCT.md LICENSE Makefile.common"
+
 # Go to the root of the repo
 cd "$(git rev-parse --show-cdup)" || exit 1
 
-source_makefile="$(pwd)/Makefile.common"
+source_dir="$(pwd)"
-source_checksum="$(sha256sum Makefile.common | cut -d' ' -f1)"
 
 tmp_dir="$(mktemp -d)"
 trap 'rm -rf "${tmp_dir}"' EXIT
@@ -55,14 +57,25 @@ process_repo() {
   local org_repo="$1"
   echo -e "\e[32mAnalyzing '${org_repo}'\e[0m"
 
-  target_makefile="$(curl -s --fail "https://raw.githubusercontent.com/${org_repo}/master/Makefile.common")"
+  local needs_update=0
-  if [ -z "${target_makefile}" ]; then
+  for source_file in ${SYNC_FILES}; do
-    echo "Makefile.common doesn't exist in ${org_repo}"
+    source_checksum="$(sha256sum "${souce_dir}/${source_file}" | cut -d' ' -f1)"
-    return
+
-  fi
+    target_file="$(curl -s --fail "https://raw.githubusercontent.com/${org_repo}/master/${source_file}")"
-  target_checksum="$(echo "${target_makefile}" | sha256sum | cut -d' ' -f1)"
+    if [[ -z "${target_makefile}" ]]; then
-  if [ "${source_checksum}" == "${target_checksum}" ]; then
+      echo "${source_file} doesn't exist in ${org_repo}"
-    echo "Makefile.common is already in sync."
+      continue
+    fi
+    target_checksum="$(echo "${target_file}" | sha256sum | cut -d' ' -f1)"
+    if [ "${source_checksum}" == "${target_checksum}" ]; then
+      echo "${source_file} is already in sync."
+      continue
+    fi
+    needs_update=1
+  done
+
+  if [[ "${needs_update}" -eq 0 ]] ; then
+    echo "No files need sync."
     return
   fi
 
@@ -71,8 +84,11 @@ process_repo() {
   cd "${tmp_dir}/${org_repo}" || return 1
   git checkout -b "${branch}" || return 1
 
-  # Replace Makefile.common in target repo by one from prometheus/prometheus
+  # Update the files in target repo by one from prometheus/prometheus.
-  cp -f "${source_makefile}" ./
+  for source_file in ${SYNC_FILES}; do
+    cp -f "${source_dir}/${source_file}" "./${source_file}"
+  done
+
   if [ -n "$(git status --porcelain)" ]; then
     git config user.email "${git_mail}"
     git config user.name "${git_user}"

storage/fanout.go

@@ -16,14 +16,15 @@ package storage
 import (
 	"container/heap"
 	"context"
-	"reflect"
 	"sort"
 	"strings"
+	"sync"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	"github.com/pkg/errors"
 	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/chunks"
@@ -37,8 +38,15 @@ type fanout struct {
 	secondaries []Storage
 }
 
-// NewFanout returns a new fan-out Storage, which proxies reads and writes
+// NewFanout returns a new fanout Storage, which proxies reads and writes
 // through to multiple underlying storages.
+//
+// The difference between primary and secondary Storage is only for read (Querier) path and it goes as follows:
+// * If the primary querier returns an error, then any of the Querier operations will fail.
+// * If any secondary querier returns an error the result from that queries is discarded. The overall operation will succeed,
+// and the error from the secondary querier will be returned as a warning.
+//
+// NOTE: In the case of Prometheus, it treats all remote storages as secondary / best effort.
 func NewFanout(logger log.Logger, primary Storage, secondaries ...Storage) Storage {
 	return &fanout{
 		logger:      logger,
@@ -56,8 +64,8 @@ func (f *fanout) StartTime() (int64, error) {
 		return int64(model.Latest), err
 	}
 
-	for _, storage := range f.secondaries {
+	for _, s := range f.secondaries {
-		t, err := storage.StartTime()
+		t, err := s.StartTime()
 		if err != nil {
 			return int64(model.Latest), err
 		}
@@ -69,29 +77,27 @@ func (f *fanout) StartTime() (int64, error) {
 }
 
 func (f *fanout) Querier(ctx context.Context, mint, maxt int64) (Querier, error) {
-	queriers := make([]Querier, 0, 1+len(f.secondaries))
+	primary, err := f.primary.Querier(ctx, mint, maxt)
-
-	// Add primary querier.
-	primaryQuerier, err := f.primary.Querier(ctx, mint, maxt)
 	if err != nil {
 		return nil, err
 	}
-	queriers = append(queriers, primaryQuerier)
 
-	// Add secondary queriers.
+	secondaries := make([]Querier, 0, len(f.secondaries))
 	for _, storage := range f.secondaries {
 		querier, err := storage.Querier(ctx, mint, maxt)
 		if err != nil {
-			for _, q := range queriers {
+			// Close already open Queriers, append potential errors to returned error.
-				// TODO(bwplotka): Log error.
+			errs := tsdb_errors.MultiError{err}
-				_ = q.Close()
+			errs.Add(primary.Close())
+			for _, q := range secondaries {
+				errs.Add(q.Close())
 			}
-			return nil, err
+			return nil, errs.Err()
 		}
-		queriers = append(queriers, querier)
-	}
 
-	return NewMergeQuerier(primaryQuerier, queriers, ChainedSeriesMerge), nil
+		secondaries = append(secondaries, querier)
+	}
+	return NewMergeQuerier(primary, secondaries, ChainedSeriesMerge), nil
 }
 
 func (f *fanout) Appender() Appender {
@@ -109,18 +115,12 @@ func (f *fanout) Appender() Appender {
 
 // Close closes the storage and all its underlying resources.
 func (f *fanout) Close() error {
-	if err := f.primary.Close(); err != nil {
+	errs := tsdb_errors.MultiError{}
-		return err
+	errs.Add(f.primary.Close())
+	for _, s := range f.secondaries {
+		errs.Add(s.Close())
 	}
-
+	return errs.Err()
-	// TODO return multiple errors?
-	var lastErr error
-	for _, storage := range f.secondaries {
-		if err := storage.Close(); err != nil {
-			lastErr = err
-		}
-	}
-	return lastErr
 }
 
 // fanoutAppender implements Appender.
@@ -188,153 +188,138 @@ func (f *fanoutAppender) Rollback() (err error) {
 }
 
 type mergeGenericQuerier struct {
-	mergeFunc genericSeriesMergeFunc
+	queriers []genericQuerier
 
-	primaryQuerier genericQuerier
+	// mergeFn is used when we see series from different queriers Selects with the same labels.
-	queriers       []genericQuerier
+	mergeFn genericSeriesMergeFunc
-	failedQueriers map[genericQuerier]struct{}
-	setQuerierMap  map[genericSeriesSet]genericQuerier
 }
 
-// NewMergeQuerier returns a new Querier that merges results of chkQuerierSeries queriers.
+// NewMergeQuerier returns a new Querier that merges results of given primary and slice of secondary queriers.
-// NewMergeQuerier will return NoopQuerier if no queriers are passed to it
+// See NewFanout commentary to learn more about primary vs secondary differences.
-// and will filter NoopQueriers from its arguments, in order to reduce overhead
+//
-// when only one querier is passed.
+// In case of overlaps between the data given by primary + secondaries Selects, merge function will be used.
-// The difference between primary and secondary is as follows: f the primaryQuerier returns an error, query fails.
+func NewMergeQuerier(primary Querier, secondaries []Querier, mergeFn VerticalSeriesMergeFunc) Querier {
-// For secondaries it just return warnings.
+	queriers := make([]genericQuerier, 0, len(secondaries)+1)
-func NewMergeQuerier(primaryQuerier Querier, queriers []Querier, mergeFunc VerticalSeriesMergeFunc) Querier {
+	if primary != nil {
-	filtered := make([]genericQuerier, 0, len(queriers))
+		queriers = append(queriers, newGenericQuerierFrom(primary))
-	for _, querier := range queriers {
+	}
+	for _, querier := range secondaries {
 		if _, ok := querier.(noopQuerier); !ok && querier != nil {
-			filtered = append(filtered, newGenericQuerierFrom(querier))
+			queriers = append(queriers, newSecondaryQuerierFrom(querier))
 		}
 	}
 
-	if len(filtered) == 0 {
-		return primaryQuerier
-	}
-
-	if primaryQuerier == nil && len(filtered) == 1 {
-		return &querierAdapter{filtered[0]}
-	}
-
 	return &querierAdapter{&mergeGenericQuerier{
-		mergeFunc:      (&seriesMergerAdapter{VerticalSeriesMergeFunc: mergeFunc}).Merge,
+		mergeFn:  (&seriesMergerAdapter{VerticalSeriesMergeFunc: mergeFn}).Merge,
-		primaryQuerier: newGenericQuerierFrom(primaryQuerier),
+		queriers: queriers,
-		queriers:       filtered,
-		failedQueriers: make(map[genericQuerier]struct{}),
-		setQuerierMap:  make(map[genericSeriesSet]genericQuerier),
 	}}
 }
 
-// NewMergeChunkQuerier returns a new ChunkQuerier that merges results of chkQuerierSeries chunk queriers.
+// NewMergeChunkQuerier returns a new ChunkQuerier that merges results of given primary and slice of secondary chunk queriers.
-// NewMergeChunkQuerier will return NoopChunkQuerier if no chunk queriers are passed to it,
+// See NewFanout commentary to learn more about primary vs secondary differences.
-// and will filter NoopQuerieNoopChunkQuerierrs from its arguments, in order to reduce overhead
+//
-// when only one chunk querier is passed.
+// In case of overlaps between the data given by primary + secondaries Selects, merge function will be used.
-func NewMergeChunkQuerier(primaryQuerier ChunkQuerier, queriers []ChunkQuerier, merger VerticalChunkSeriesMergerFunc) ChunkQuerier {
+// TODO(bwplotka): Currently merge will compact overlapping chunks with bigger chunk, without limit. Split it: https://github.com/prometheus/tsdb/issues/670
-	filtered := make([]genericQuerier, 0, len(queriers))
+func NewMergeChunkQuerier(primary ChunkQuerier, secondaries []ChunkQuerier, mergeFn VerticalChunkSeriesMergerFunc) ChunkQuerier {
-	for _, querier := range queriers {
+	queriers := make([]genericQuerier, 0, len(secondaries)+1)
+	if primary != nil {
+		queriers = append(queriers, newGenericQuerierFromChunk(primary))
+	}
+	for _, querier := range secondaries {
 		if _, ok := querier.(noopChunkQuerier); !ok && querier != nil {
-			filtered = append(filtered, newGenericQuerierFromChunk(querier))
+			queriers = append(queriers, newSecondaryQuerierFromChunk(querier))
 		}
 	}
 
-	if len(filtered) == 0 {
-		return primaryQuerier
-	}
-
-	if primaryQuerier == nil && len(filtered) == 1 {
-		return &chunkQuerierAdapter{filtered[0]}
-	}
-
 	return &chunkQuerierAdapter{&mergeGenericQuerier{
-		mergeFunc:      (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge,
+		mergeFn:  (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: mergeFn}).Merge,
-		primaryQuerier: newGenericQuerierFromChunk(primaryQuerier),
+		queriers: queriers,
-		queriers:       filtered,
-		failedQueriers: make(map[genericQuerier]struct{}),
-		setQuerierMap:  make(map[genericSeriesSet]genericQuerier),
 	}}
 }
 
 // Select returns a set of series that matches the given label matchers.
-func (q *mergeGenericQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (genericSeriesSet, Warnings, error) {
+func (q *mergeGenericQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet {
 	if len(q.queriers) == 1 {
 		return q.queriers[0].Select(sortSeries, hints, matchers...)
 	}
 
 	var (
-		seriesSets = make([]genericSeriesSet, 0, len(q.queriers))
+		seriesSets    = make([]genericSeriesSet, 0, len(q.queriers))
-		warnings   Warnings
+		wg            sync.WaitGroup
-		priErr     error
+		seriesSetChan = make(chan genericSeriesSet)
 	)
-	type queryResult struct {
-		qr          genericQuerier
-		set         genericSeriesSet
-		wrn         Warnings
-		selectError error
-	}
-	queryResultChan := make(chan *queryResult)
 
+	// Schedule all Selects for all queriers we know about.
 	for _, querier := range q.queriers {
+		wg.Add(1)
 		go func(qr genericQuerier) {
+			defer wg.Done()
+
 			// We need to sort for NewMergeSeriesSet to work.
-			set, wrn, err := qr.Select(true, hints, matchers...)
+			seriesSetChan <- qr.Select(true, hints, matchers...)
-			queryResultChan <- &queryResult{qr: qr, set: set, wrn: wrn, selectError: err}
 		}(querier)
 	}
-	for i := 0; i < len(q.queriers); i++ {
+	go func() {
-		qryResult := <-queryResultChan
+		wg.Wait()
-		q.setQuerierMap[qryResult.set] = qryResult.qr
+		close(seriesSetChan)
-		if qryResult.wrn != nil {
+	}()
-			warnings = append(warnings, qryResult.wrn...)
+
+	for r := range seriesSetChan {
+		seriesSets = append(seriesSets, r)
+	}
+	return &lazySeriesSet{create: create(seriesSets, q.mergeFn)}
+}
+
+func create(seriesSets []genericSeriesSet, mergeFn genericSeriesMergeFunc) func() (genericSeriesSet, bool) {
+	// Returned function gets called with the first call to Next().
+	return func() (genericSeriesSet, bool) {
+		if len(seriesSets) == 1 {
+			return seriesSets[0], seriesSets[0].Next()
 		}
-		if qryResult.selectError != nil {
+		var h genericSeriesSetHeap
-			q.failedQueriers[qryResult.qr] = struct{}{}
+		for _, set := range seriesSets {
-			// If the error source isn't the primary querier, return the error as a warning and continue.
+			if set == nil {
-			if !reflect.DeepEqual(qryResult.qr, q.primaryQuerier) {
+				continue
-				warnings = append(warnings, qryResult.selectError)
+			}
-			} else {
+			if set.Next() {
-				priErr = qryResult.selectError
+				heap.Push(&h, set)
+				continue
+			}
+			// When primary fails ignore results from secondaries.
+			// Only the primary querier returns error.
+			if err := set.Err(); err != nil {
+				return errorOnlySeriesSet{err}, false
 			}
-			continue
 		}
-		seriesSets = append(seriesSets, qryResult.set)
+		set := &genericMergeSeriesSet{
+			mergeFn: mergeFn,
+			sets:    seriesSets,
+			heap:    h,
+		}
+		return set, set.Next()
 	}
-	if priErr != nil {
-		return nil, nil, priErr
-	}
-	return newGenericMergeSeriesSet(seriesSets, q, q.mergeFunc), warnings, nil
 }
 
 // LabelValues returns all potential values for a label name.
 func (q *mergeGenericQuerier) LabelValues(name string) ([]string, Warnings, error) {
-	var results [][]string
+	var (
-	var warnings Warnings
+		results  [][]string
+		warnings Warnings
+	)
 	for _, querier := range q.queriers {
 		values, wrn, err := querier.LabelValues(name)
 		if wrn != nil {
+			// TODO(bwplotka): We could potentially wrap warnings.
 			warnings = append(warnings, wrn...)
 		}
 		if err != nil {
-			q.failedQueriers[querier] = struct{}{}
+			return nil, nil, errors.Wrapf(err, "LabelValues() from Querier for label %s", name)
-			// If the error source isn't the primary querier, return the error as a warning and continue.
-			if querier != q.primaryQuerier {
-				warnings = append(warnings, err)
-				continue
-			} else {
-				return nil, nil, err
-			}
 		}
 		results = append(results, values)
 	}
 	return mergeStringSlices(results), warnings, nil
 }
 
-func (q *mergeGenericQuerier) IsFailedSet(set genericSeriesSet) bool {
-	_, isFailedQuerier := q.failedQueriers[q.setQuerierMap[set]]
-	return isFailedQuerier
-}
-
 func mergeStringSlices(ss [][]string) []string {
 	switch len(ss) {
 	case 0:
@@ -381,37 +366,31 @@ func (q *mergeGenericQuerier) LabelNames() ([]string, Warnings, error) {
 	for _, querier := range q.queriers {
 		names, wrn, err := querier.LabelNames()
 		if wrn != nil {
+			// TODO(bwplotka): We could potentially wrap warnings.
 			warnings = append(warnings, wrn...)
 		}
-
 		if err != nil {
-			q.failedQueriers[querier] = struct{}{}
+			return nil, nil, errors.Wrap(err, "LabelNames() from Querier")
-			// If the error source isn't the primaryQuerier querier, return the error as a warning and continue.
-			if querier != q.primaryQuerier {
-				warnings = append(warnings, err)
-				continue
-			} else {
-				return nil, nil, errors.Wrap(err, "LabelNames() from Querier")
-			}
 		}
-
 		for _, name := range names {
 			labelNamesMap[name] = struct{}{}
 		}
 	}
+	if len(labelNamesMap) == 0 {
+		return nil, warnings, nil
+	}
 
 	labelNames := make([]string, 0, len(labelNamesMap))
 	for name := range labelNamesMap {
 		labelNames = append(labelNames, name)
 	}
 	sort.Strings(labelNames)
-
 	return labelNames, warnings, nil
 }
 
 // Close releases the resources of the Querier.
 func (q *mergeGenericQuerier) Close() error {
-	var errs tsdb_errors.MultiError
+	errs := tsdb_errors.MultiError{}
 	for _, querier := range q.queriers {
 		if err := querier.Close(); err != nil {
 			errs.Add(err)
@@ -420,18 +399,6 @@ func (q *mergeGenericQuerier) Close() error {
 	return errs.Err()
 }
 
-// genericMergeSeriesSet implements genericSeriesSet
-type genericMergeSeriesSet struct {
-	currentLabels labels.Labels
-	mergeFunc     genericSeriesMergeFunc
-
-	heap genericSeriesSetHeap
-	sets []genericSeriesSet
-
-	currentSets []genericSeriesSet
-	querier     *mergeGenericQuerier
-}
-
 // VerticalSeriesMergeFunc returns merged series implementation that merges series with same labels together.
 // It has to handle time-overlapped series as well.
 type VerticalSeriesMergeFunc func(...Series) Series
@@ -447,7 +414,7 @@ func NewMergeSeriesSet(sets []SeriesSet, merger VerticalSeriesMergeFunc) SeriesS
 		genericSets = append(genericSets, &genericSeriesSetAdapter{s})
 
 	}
-	return &seriesSetAdapter{newGenericMergeSeriesSet(genericSets, nil, (&seriesMergerAdapter{VerticalSeriesMergeFunc: merger}).Merge)}
+	return &seriesSetAdapter{newGenericMergeSeriesSet(genericSets, (&seriesMergerAdapter{VerticalSeriesMergeFunc: merger}).Merge)}
 }
 
 // NewMergeChunkSeriesSet returns a new ChunkSeriesSet that merges results of chkQuerierSeries ChunkSeriesSets.
@@ -457,21 +424,30 @@ func NewMergeChunkSeriesSet(sets []ChunkSeriesSet, merger VerticalChunkSeriesMer
 		genericSets = append(genericSets, &genericChunkSeriesSetAdapter{s})
 
 	}
-	return &chunkSeriesSetAdapter{newGenericMergeSeriesSet(genericSets, nil, (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge)}
+	return &chunkSeriesSetAdapter{newGenericMergeSeriesSet(genericSets, (&chunkSeriesMergerAdapter{VerticalChunkSeriesMergerFunc: merger}).Merge)}
+}
+
+// genericMergeSeriesSet implements genericSeriesSet.
+type genericMergeSeriesSet struct {
+	currentLabels labels.Labels
+	mergeFn       genericSeriesMergeFunc
+
+	heap        genericSeriesSetHeap
+	sets        []genericSeriesSet
+	currentSets []genericSeriesSet
 }
 
 // newGenericMergeSeriesSet returns a new genericSeriesSet that merges (and deduplicates)
-// series returned by the chkQuerierSeries series sets when iterating.
+// series returned by the series sets when iterating.
-// Each chkQuerierSeries series set must return its series in labels order, otherwise
+// Each series set must return its series in labels order, otherwise
 // merged series set will be incorrect.
-// Argument 'querier' is optional and can be nil. Pass Querier if you want to retry query in case of failing series set.
+// Overlapping cases are merged using provided mergeFn.
-// Overlapped situations are merged using provided mergeFunc.
+func newGenericMergeSeriesSet(sets []genericSeriesSet, mergeFn genericSeriesMergeFunc) genericSeriesSet {
-func newGenericMergeSeriesSet(sets []genericSeriesSet, querier *mergeGenericQuerier, mergeFunc genericSeriesMergeFunc) genericSeriesSet {
 	if len(sets) == 1 {
 		return sets[0]
 	}
 
-	// Sets need to be pre-advanced, so we can introspect the label of the
+	// We are pre-advancing sets, so we can introspect the label of the
 	// series under the cursor.
 	var h genericSeriesSetHeap
 	for _, set := range sets {
@@ -483,26 +459,25 @@ func newGenericMergeSeriesSet(sets []genericSeriesSet, querier *mergeGenericQuer
 		}
 	}
 	return &genericMergeSeriesSet{
-		mergeFunc: mergeFunc,
+		mergeFn: mergeFn,
-		heap:      h,
+		sets:    sets,
-		sets:      sets,
+		heap:    h,
-		querier:   querier,
 	}
 }
 
 func (c *genericMergeSeriesSet) Next() bool {
 	// Run in a loop because the "next" series sets may not be valid anymore.
-	// If a remote querier fails, we discard all series sets from that querier.
 	// If, for the current label set, all the next series sets come from
 	// failed remote storage sources, we want to keep trying with the next label set.
 	for {
-		// Firstly advance all the current series sets.  If any of them have run out
+		// Firstly advance all the current series sets. If any of them have run out
 		// we can drop them, otherwise they should be inserted back into the heap.
 		for _, set := range c.currentSets {
 			if set.Next() {
 				heap.Push(&c.heap, set)
 			}
 		}
+
 		if len(c.heap) == 0 {
 			return false
 		}
@@ -512,9 +487,6 @@ func (c *genericMergeSeriesSet) Next() bool {
 		c.currentLabels = c.heap[0].At().Labels()
 		for len(c.heap) > 0 && labels.Equal(c.currentLabels, c.heap[0].At().Labels()) {
 			set := heap.Pop(&c.heap).(genericSeriesSet)
-			if c.querier != nil && c.querier.IsFailedSet(set) {
-				continue
-			}
 			c.currentSets = append(c.currentSets, set)
 		}
 
@@ -535,7 +507,7 @@ func (c *genericMergeSeriesSet) At() Labels {
 	for _, seriesSet := range c.currentSets {
 		series = append(series, seriesSet.At())
 	}
-	return c.mergeFunc(series...)
+	return c.mergeFn(series...)
 }
 
 func (c *genericMergeSeriesSet) Err() error {
@@ -547,6 +519,14 @@ func (c *genericMergeSeriesSet) Err() error {
 	return nil
 }
 
+func (c *genericMergeSeriesSet) Warnings() Warnings {
+	var ws Warnings
+	for _, set := range c.sets {
+		ws = append(ws, set.Warnings()...)
+	}
+	return ws
+}
+
 type genericSeriesSetHeap []genericSeriesSet
 
 func (h genericSeriesSetHeap) Len() int      { return len(h) }

storage/fanout/fanout_test.go

@@ -16,14 +16,14 @@ package storage
 import (
 	"context"
 	"errors"
+	"testing"
 
 	"github.com/prometheus/common/model"
+
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/storage"
 	"github.com/prometheus/prometheus/util/teststorage"
 	"github.com/prometheus/prometheus/util/testutil"
-
-	"testing"
 )
 
 func TestSelectSorted(t *testing.T) {
@@ -79,8 +79,7 @@ func TestSelectSorted(t *testing.T) {
 	matcher, err := labels.NewMatcher(labels.MatchEqual, model.MetricNameLabel, "a")
 	testutil.Ok(t, err)
 
-	seriesSet, _, err := querier.Select(true, nil, matcher)
+	seriesSet := querier.Select(true, nil, matcher)
-	testutil.Ok(t, err)
 
 	result := make(map[int64]float64)
 	var labelsResult labels.Labels
@@ -95,6 +94,7 @@ func TestSelectSorted(t *testing.T) {
 		}
 	}
 
+	testutil.Ok(t, seriesSet.Err())
 	testutil.Equals(t, labelsResult, outputLabel)
 	testutil.Equals(t, inputTotalSize, len(result))
 }
@@ -131,19 +131,12 @@ func TestFanoutErrors(t *testing.T) {
 		defer querier.Close()
 
 		matcher := labels.MustNewMatcher(labels.MatchEqual, "a", "b")
-		ss, warnings, err := querier.Select(true, nil, matcher)
+		ss := querier.Select(true, nil, matcher)
-		testutil.Equals(t, tc.err, err)
-		testutil.Equals(t, tc.warnings, warnings)
-
-		// Only test series iteration if there are no errors.
-		if err != nil {
-			continue
-		}
-
 		for ss.Next() {
 			ss.At()
 		}
-		testutil.Ok(t, ss.Err())
+		testutil.Equals(t, tc.err, ss.Err())
+		testutil.Equals(t, tc.warnings, ss.Warnings())
 	}
 }
 
@@ -169,8 +162,8 @@ func (errStorage) Close() error {
 
 type errQuerier struct{}
 
-func (errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (errQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet {
-	return nil, nil, errSelect
+	return storage.ErrSeriesSet(errSelect)
 }
 
 func (errQuerier) LabelValues(name string) ([]string, storage.Warnings, error) {

storage/fanout_test.go

@@ -17,8 +17,11 @@ import (
 	"fmt"
 	"math"
 	"sort"
+	"sync"
 	"testing"
 
+	"github.com/pkg/errors"
+
 	"github.com/prometheus/prometheus/pkg/labels"
 	"github.com/prometheus/prometheus/tsdb/chunkenc"
 	"github.com/prometheus/prometheus/tsdb/tsdbutil"
@@ -56,24 +59,37 @@ func TestMergeTwoStringSlices(t *testing.T) {
 
 func TestMergeQuerierWithChainMerger(t *testing.T) {
 	for _, tc := range []struct {
-		name          string
+		name                 string
-		querierSeries [][]Series
+		primaryQuerierSeries []Series
-		extraQueriers []Querier
+		querierSeries        [][]Series
+		extraQueriers        []Querier
 
 		expected SeriesSet
 	}{
 		{
-			name:          "1 querier with no series",
+			name:                 "one primary querier with no series",
+			primaryQuerierSeries: []Series{},
+			expected:             NewMockSeriesSet(),
+		},
+		{
+			name:          "one secondary querier with no series",
 			querierSeries: [][]Series{{}},
 			expected:      NewMockSeriesSet(),
 		},
 		{
-			name:          "many queriers with no series",
+			name:          "many secondary queriers with no series",
 			querierSeries: [][]Series{{}, {}, {}, {}, {}, {}, {}},
 			expected:      NewMockSeriesSet(),
 		},
 		{
-			name: "1 querier, two series",
+			name:                 "mix of queriers with no series",
+			primaryQuerierSeries: []Series{},
+			querierSeries:        [][]Series{{}, {}, {}, {}, {}, {}, {}},
+			expected:             NewMockSeriesSet(),
+		},
+		// Test rest of cases on secondary queriers as the different between primary vs secondary is just error handling.
+		{
+			name: "one querier, two series",
 			querierSeries: [][]Series{{
 				NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{1, 1}, sample{2, 2}, sample{3, 3}}),
 				NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}),
@@ -84,7 +100,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 			),
 		},
 		{
-			name: "2 queriers, 1 different series each",
+			name: "two queriers, one different series each",
 			querierSeries: [][]Series{{
 				NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{1, 1}, sample{2, 2}, sample{3, 3}}),
 			}, {
@@ -96,7 +112,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 			),
 		},
 		{
-			name: "2 time unsorted queriers, 2 series each",
+			name: "two time unsorted queriers, two series each",
 			querierSeries: [][]Series{{
 				NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{5, 5}, sample{6, 6}}),
 				NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}),
@@ -116,7 +132,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 			),
 		},
 		{
-			name: "5 queriers, only 2 queriers have 2 time unsorted series each",
+			name: "five queriers, only two queriers have two time unsorted series each",
 			querierSeries: [][]Series{{}, {}, {
 				NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{5, 5}, sample{6, 6}}),
 				NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}),
@@ -136,7 +152,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 			),
 		},
 		{
-			name: "2 queriers, only 2 queriers have 2 time unsorted series each, with 3 noop and one nil querier together",
+			name: "two queriers, only two queriers have two time unsorted series each, with 3 noop and one nil querier together",
 			querierSeries: [][]Series{{}, {}, {
 				NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{5, 5}, sample{6, 6}}),
 				NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}),
@@ -157,7 +173,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 			),
 		},
 		{
-			name: "2 queriers, with 2 series, one is overlapping",
+			name: "two queriers, with two series, one is overlapping",
 			querierSeries: [][]Series{{}, {}, {
 				NewListSeries(labels.FromStrings("bar", "baz"), []tsdbutil.Sample{sample{2, 21}, sample{3, 31}, sample{5, 5}, sample{6, 6}}),
 				NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, 0}, sample{1, 1}, sample{2, 2}}),
@@ -177,7 +193,7 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 			),
 		},
 		{
-			name: "2 queries, one with NaN samples series",
+			name: "two queries, one with NaN samples series",
 			querierSeries: [][]Series{{
 				NewListSeries(labels.FromStrings("foo", "bar"), []tsdbutil.Sample{sample{0, math.NaN()}}),
 			}, {
@@ -189,13 +205,17 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
+			var p Querier
+			if tc.primaryQuerierSeries != nil {
+				p = &mockQuerier{toReturn: tc.primaryQuerierSeries}
+			}
 			var qs []Querier
 			for _, in := range tc.querierSeries {
 				qs = append(qs, &mockQuerier{toReturn: in})
 			}
 			qs = append(qs, tc.extraQueriers...)
 
-			mergedQuerier, _, _ := NewMergeQuerier(qs[0], qs, ChainedSeriesMerge).Select(false, nil)
+			mergedQuerier := NewMergeQuerier(p, qs, ChainedSeriesMerge).Select(false, nil)
 
 			// Get all merged series upfront to make sure there are no incorrectly retained shared
 			// buffers causing bugs.
@@ -222,22 +242,35 @@ func TestMergeQuerierWithChainMerger(t *testing.T) {
 
 func TestMergeChunkQuerierWithNoVerticalChunkSeriesMerger(t *testing.T) {
 	for _, tc := range []struct {
-		name             string
+		name                    string
-		chkQuerierSeries [][]ChunkSeries
+		primaryChkQuerierSeries []ChunkSeries
-		extraQueriers    []ChunkQuerier
+		chkQuerierSeries        [][]ChunkSeries
+		extraQueriers           []ChunkQuerier
 
 		expected ChunkSeriesSet
 	}{
 		{
-			name:             "one querier with no series",
+			name:                    "one primary querier with no series",
+			primaryChkQuerierSeries: []ChunkSeries{},
+			expected:                NewMockChunkSeriesSet(),
+		},
+		{
+			name:             "one secondary querier with no series",
 			chkQuerierSeries: [][]ChunkSeries{{}},
 			expected:         NewMockChunkSeriesSet(),
 		},
 		{
-			name:             "many queriers with no series",
+			name:             "many secondary queriers with no series",
 			chkQuerierSeries: [][]ChunkSeries{{}, {}, {}, {}, {}, {}, {}},
 			expected:         NewMockChunkSeriesSet(),
 		},
+		{
+			name:                    "mix of queriers with no series",
+			primaryChkQuerierSeries: []ChunkSeries{},
+			chkQuerierSeries:        [][]ChunkSeries{{}, {}, {}, {}, {}, {}, {}},
+			expected:                NewMockChunkSeriesSet(),
+		},
+		// Test rest of cases on secondary queriers as the different between primary vs secondary is just error handling.
 		{
 			name: "one querier, two series",
 			chkQuerierSeries: [][]ChunkSeries{{
@@ -347,13 +380,19 @@ func TestMergeChunkQuerierWithNoVerticalChunkSeriesMerger(t *testing.T) {
 		},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
+			var p ChunkQuerier
+			if tc.primaryChkQuerierSeries != nil {
+				p = &mockChunkQurier{toReturn: tc.primaryChkQuerierSeries}
+			}
+
 			var qs []ChunkQuerier
 			for _, in := range tc.chkQuerierSeries {
 				qs = append(qs, &mockChunkQurier{toReturn: in})
 			}
 			qs = append(qs, tc.extraQueriers...)
 
-			merged, _, _ := NewMergeChunkQuerier(qs[0], qs, NewVerticalChunkSeriesMerger(nil)).Select(false, nil)
+			// TODO(bwplotka): Add case of overlap to check if those are handled well.
+			merged := NewMergeChunkQuerier(p, qs, NewVerticalChunkSeriesMerger(nil)).Select(false, nil)
 			for merged.Next() {
 				testutil.Assert(t, tc.expected.Next(), "Expected Next() to be true")
 				actualSeries := merged.At()
@@ -384,14 +423,14 @@ func (a seriesByLabel) Len() int           { return len(a) }
 func (a seriesByLabel) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
 func (a seriesByLabel) Less(i, j int) bool { return labels.Compare(a[i].Labels(), a[j].Labels()) < 0 }
 
-func (m *mockQuerier) Select(sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) (SeriesSet, Warnings, error) {
+func (m *mockQuerier) Select(sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) SeriesSet {
 	cpy := make([]Series, len(m.toReturn))
 	copy(cpy, m.toReturn)
 	if sortSeries {
 		sort.Sort(seriesByLabel(cpy))
 	}
 
-	return NewMockSeriesSet(cpy...), nil, nil
+	return NewMockSeriesSet(cpy...)
 }
 
 type mockChunkQurier struct {
@@ -408,14 +447,14 @@ func (a chunkSeriesByLabel) Less(i, j int) bool {
 	return labels.Compare(a[i].Labels(), a[j].Labels()) < 0
 }
 
-func (m *mockChunkQurier) Select(sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) {
+func (m *mockChunkQurier) Select(sortSeries bool, _ *SelectHints, _ ...*labels.Matcher) ChunkSeriesSet {
 	cpy := make([]ChunkSeries, len(m.toReturn))
 	copy(cpy, m.toReturn)
 	if sortSeries {
 		sort.Sort(chunkSeriesByLabel(cpy))
 	}
 
-	return NewMockChunkSeriesSet(cpy...), nil, nil
+	return NewMockChunkSeriesSet(cpy...)
 }
 
 type mockSeriesSet struct {
@@ -439,6 +478,8 @@ func (m *mockSeriesSet) At() Series { return m.series[m.idx] }
 
 func (m *mockSeriesSet) Err() error { return nil }
 
+func (m *mockSeriesSet) Warnings() Warnings { return nil }
+
 type mockChunkSeriesSet struct {
 	idx    int
 	series []ChunkSeries
@@ -460,6 +501,8 @@ func (m *mockChunkSeriesSet) At() ChunkSeries { return m.series[m.idx] }
 
 func (m *mockChunkSeriesSet) Err() error { return nil }
 
+func (m *mockChunkSeriesSet) Warnings() Warnings { return nil }
+
 func TestChainSampleIterator(t *testing.T) {
 	for _, tc := range []struct {
 		input    []chunkenc.Iterator
@@ -571,7 +614,7 @@ func makeMergeSeriesSet(numSeriesSets, numSeries, numSamples int) SeriesSet {
 	for i := 0; i < numSeriesSets; i++ {
 		seriesSets = append(seriesSets, &genericSeriesSetAdapter{makeSeriesSet(numSeries, numSamples)})
 	}
-	return &seriesSetAdapter{newGenericMergeSeriesSet(seriesSets, nil, (&seriesMergerAdapter{VerticalSeriesMergeFunc: ChainedSeriesMerge}).Merge)}
+	return &seriesSetAdapter{newGenericMergeSeriesSet(seriesSets, (&seriesMergerAdapter{VerticalSeriesMergeFunc: ChainedSeriesMerge}).Merge)}
 }
 
 func benchmarkDrain(seriesSet SeriesSet, b *testing.B) {
@@ -603,3 +646,260 @@ func BenchmarkMergeSeriesSet(b *testing.B) {
 		})
 	}
 }
+
+type mockGenericQuerier struct {
+	mtx sync.Mutex
+
+	closed                bool
+	labelNamesCalls       int
+	labelNamesRequested   []string
+	sortedSeriesRequested []bool
+
+	resp     []string
+	warnings Warnings
+	err      error
+}
+
+func (m *mockGenericQuerier) Select(b bool, _ *SelectHints, _ ...*labels.Matcher) genericSeriesSet {
+	m.mtx.Lock()
+	m.sortedSeriesRequested = append(m.sortedSeriesRequested, b)
+	m.mtx.Unlock()
+	return &mockGenericSeriesSet{resp: m.resp, warnings: m.warnings, err: m.err}
+}
+
+func (m *mockGenericQuerier) LabelValues(name string) ([]string, Warnings, error) {
+	m.mtx.Lock()
+	m.labelNamesRequested = append(m.labelNamesRequested, name)
+	m.mtx.Unlock()
+	return m.resp, m.warnings, m.err
+}
+
+func (m *mockGenericQuerier) LabelNames() ([]string, Warnings, error) {
+	m.mtx.Lock()
+	m.labelNamesCalls++
+	m.mtx.Unlock()
+	return m.resp, m.warnings, m.err
+}
+
+func (m *mockGenericQuerier) Close() error {
+	m.closed = true
+	return nil
+}
+
+type mockGenericSeriesSet struct {
+	resp     []string
+	warnings Warnings
+	err      error
+
+	curr int
+}
+
+func (m *mockGenericSeriesSet) Next() bool {
+	if m.err != nil {
+		return false
+	}
+	if m.curr >= len(m.resp) {
+		return false
+	}
+	m.curr++
+	return true
+}
+
+func (m *mockGenericSeriesSet) Err() error         { return m.err }
+func (m *mockGenericSeriesSet) Warnings() Warnings { return m.warnings }
+
+func (m *mockGenericSeriesSet) At() Labels {
+	return mockLabels(m.resp[m.curr-1])
+}
+
+type mockLabels string
+
+func (l mockLabels) Labels() labels.Labels {
+	return labels.FromStrings("test", string(l))
+}
+
+func unwrapMockGenericQuerier(t *testing.T, qr genericQuerier) *mockGenericQuerier {
+	m, ok := qr.(*mockGenericQuerier)
+	if !ok {
+		s, ok := qr.(*secondaryQuerier)
+		testutil.Assert(t, ok, "expected secondaryQuerier got something else")
+		m, ok = s.genericQuerier.(*mockGenericQuerier)
+		testutil.Assert(t, ok, "expected mockGenericQuerier got something else")
+	}
+	return m
+}
+
+func TestMergeGenericQuerierWithSecondaries_ErrorHandling(t *testing.T) {
+	var (
+		errStorage  = errors.New("storage error")
+		warnStorage = errors.New("storage warning")
+	)
+	for _, tcase := range []struct {
+		name     string
+		queriers []genericQuerier
+
+		expectedSelectsSeries []labels.Labels
+		expectedLabels        []string
+
+		expectedWarnings [3]Warnings
+		expectedErrs     [3]error
+	}{
+		{},
+		{
+			name:     "one successful primary querier",
+			queriers: []genericQuerier{&mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil}},
+			expectedSelectsSeries: []labels.Labels{
+				labels.FromStrings("test", "a"),
+				labels.FromStrings("test", "b"),
+			},
+			expectedLabels: []string{"a", "b"},
+		},
+		{
+			name: "multiple successful primary queriers",
+			queriers: []genericQuerier{
+				&mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil},
+				&mockGenericQuerier{resp: []string{"b", "c"}, warnings: nil, err: nil},
+			},
+			expectedSelectsSeries: []labels.Labels{
+				labels.FromStrings("test", "a"),
+				labels.FromStrings("test", "b"),
+				labels.FromStrings("test", "c"),
+			},
+			expectedLabels: []string{"a", "b", "c"},
+		},
+		{
+			name:         "one failed primary querier",
+			queriers:     []genericQuerier{&mockGenericQuerier{warnings: nil, err: errStorage}},
+			expectedErrs: [3]error{errStorage, errStorage, errStorage},
+		},
+		{
+			name: "one successful primary querier with successful secondaries",
+			queriers: []genericQuerier{
+				&mockGenericQuerier{resp: []string{"a", "b"}, warnings: nil, err: nil},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}},
+			},
+			expectedSelectsSeries: []labels.Labels{
+				labels.FromStrings("test", "a"),
+				labels.FromStrings("test", "b"),
+				labels.FromStrings("test", "c"),
+			},
+			expectedLabels: []string{"a", "b", "c"},
+		},
+		{
+			name: "one successful primary querier with empty response and successful secondaries",
+			queriers: []genericQuerier{
+				&mockGenericQuerier{resp: []string{}, warnings: nil, err: nil},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}},
+			},
+			expectedSelectsSeries: []labels.Labels{
+				labels.FromStrings("test", "b"),
+				labels.FromStrings("test", "c"),
+			},
+			expectedLabels: []string{"b", "c"},
+		},
+		{
+			name: "one failed primary querier with successful secondaries",
+			queriers: []genericQuerier{
+				&mockGenericQuerier{warnings: nil, err: errStorage},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: nil}},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: nil}},
+			},
+			expectedErrs: [3]error{errStorage, errStorage, errStorage},
+		},
+		{
+			name: "one successful primary querier with failed secondaries",
+			queriers: []genericQuerier{
+				&mockGenericQuerier{resp: []string{"a"}, warnings: nil, err: nil},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: nil, err: errStorage}},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"c"}, warnings: nil, err: errStorage}},
+			},
+			expectedSelectsSeries: []labels.Labels{
+				labels.FromStrings("test", "a"),
+			},
+			expectedLabels: []string{"a"},
+			expectedWarnings: [3]Warnings{
+				[]error{errStorage, errStorage},
+				[]error{errStorage, errStorage},
+				[]error{errStorage, errStorage},
+			},
+		},
+		{
+			name: "successful queriers with warnings",
+			queriers: []genericQuerier{
+				&mockGenericQuerier{resp: []string{"a"}, warnings: []error{warnStorage}, err: nil},
+				&secondaryQuerier{genericQuerier: &mockGenericQuerier{resp: []string{"b"}, warnings: []error{warnStorage}, err: nil}},
+			},
+			expectedSelectsSeries: []labels.Labels{
+				labels.FromStrings("test", "a"),
+				labels.FromStrings("test", "b"),
+			},
+			expectedLabels: []string{"a", "b"},
+			expectedWarnings: [3]Warnings{
+				[]error{warnStorage, warnStorage},
+				[]error{warnStorage, warnStorage},
+				[]error{warnStorage, warnStorage},
+			},
+		},
+	} {
+		t.Run(tcase.name, func(t *testing.T) {
+			q := &mergeGenericQuerier{
+				queriers: tcase.queriers,
+				mergeFn:  func(l ...Labels) Labels { return l[0] },
+			}
+
+			t.Run("Select", func(t *testing.T) {
+				res := q.Select(false, nil)
+				var lbls []labels.Labels
+				for res.Next() {
+					lbls = append(lbls, res.At().Labels())
+				}
+				testutil.Equals(t, tcase.expectedWarnings[0], res.Warnings())
+				testutil.Equals(t, tcase.expectedErrs[0], res.Err())
+				testutil.Assert(t, errors.Is(res.Err(), tcase.expectedErrs[0]), "expected error doesn't match")
+				testutil.Equals(t, tcase.expectedSelectsSeries, lbls)
+
+				for _, qr := range q.queriers {
+					m := unwrapMockGenericQuerier(t, qr)
+
+					exp := []bool{true}
+					if len(q.queriers) == 1 {
+						exp[0] = false
+					}
+					testutil.Equals(t, exp, m.sortedSeriesRequested)
+				}
+			})
+			t.Run("LabelNames", func(t *testing.T) {
+				res, w, err := q.LabelNames()
+				testutil.Equals(t, tcase.expectedWarnings[1], w)
+				testutil.Assert(t, errors.Is(err, tcase.expectedErrs[1]), "expected error doesn't match")
+				testutil.Equals(t, tcase.expectedLabels, res)
+
+				if err != nil {
+					return
+				}
+				for _, qr := range q.queriers {
+					m := unwrapMockGenericQuerier(t, qr)
+
+					testutil.Equals(t, 1, m.labelNamesCalls)
+				}
+			})
+			t.Run("LabelValues", func(t *testing.T) {
+				res, w, err := q.LabelValues("test")
+				testutil.Equals(t, tcase.expectedWarnings[2], w)
+				testutil.Assert(t, errors.Is(err, tcase.expectedErrs[2]), "expected error doesn't match")
+				testutil.Equals(t, tcase.expectedLabels, res)
+
+				if err != nil {
+					return
+				}
+				for _, qr := range q.queriers {
+					m := unwrapMockGenericQuerier(t, qr)
+
+					testutil.Equals(t, []string{"test"}, m.labelNamesRequested)
+				}
+			})
+		})
+	}
+}

storage/generic.go

@@ -20,13 +20,14 @@ import "github.com/prometheus/prometheus/pkg/labels"
 
 type genericQuerier interface {
 	baseQuerier
-	Select(bool, *SelectHints, ...*labels.Matcher) (genericSeriesSet, Warnings, error)
+	Select(bool, *SelectHints, ...*labels.Matcher) genericSeriesSet
 }
 
 type genericSeriesSet interface {
 	Next() bool
 	At() Labels
 	Err() error
+	Warnings() Warnings
 }
 
 type genericSeriesMergeFunc func(...Labels) Labels
@@ -55,13 +56,11 @@ type genericQuerierAdapter struct {
 	cq ChunkQuerier
 }
 
-func (q *genericQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (genericSeriesSet, Warnings, error) {
+func (q *genericQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet {
 	if q.q != nil {
-		s, w, err := q.q.Select(sortSeries, hints, matchers...)
+		return &genericSeriesSetAdapter{q.q.Select(sortSeries, hints, matchers...)}
-		return &genericSeriesSetAdapter{s}, w, err
 	}
-	s, w, err := q.cq.Select(sortSeries, hints, matchers...)
+	return &genericChunkSeriesSetAdapter{q.cq.Select(sortSeries, hints, matchers...)}
-	return &genericChunkSeriesSetAdapter{s}, w, err
 }
 
 func newGenericQuerierFrom(q Querier) genericQuerier {
@@ -84,9 +83,8 @@ func (a *seriesSetAdapter) At() Series {
 	return a.genericSeriesSet.At().(Series)
 }
 
-func (q *querierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (SeriesSet, Warnings, error) {
+func (q *querierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) SeriesSet {
-	s, w, err := q.genericQuerier.Select(sortSeries, hints, matchers...)
+	return &seriesSetAdapter{q.genericQuerier.Select(sortSeries, hints, matchers...)}
-	return &seriesSetAdapter{s}, w, err
 }
 
 type chunkQuerierAdapter struct {
@@ -101,9 +99,8 @@ func (a *chunkSeriesSetAdapter) At() ChunkSeries {
 	return a.genericSeriesSet.At().(ChunkSeries)
 }
 
-func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) {
+func (q *chunkQuerierAdapter) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) ChunkSeriesSet {
-	s, w, err := q.genericQuerier.Select(sortSeries, hints, matchers...)
+	return &chunkSeriesSetAdapter{q.genericQuerier.Select(sortSeries, hints, matchers...)}
-	return &chunkSeriesSetAdapter{s}, w, err
 }
 
 type seriesMergerAdapter struct {
@@ -129,3 +126,13 @@ func (a *chunkSeriesMergerAdapter) Merge(s ...Labels) Labels {
 	}
 	return a.VerticalChunkSeriesMergerFunc(buf...)
 }
+
+type noopGenericSeriesSet struct{}
+
+func (noopGenericSeriesSet) Next() bool { return false }
+
+func (noopGenericSeriesSet) At() Labels { return nil }
+
+func (noopGenericSeriesSet) Err() error { return nil }
+
+func (noopGenericSeriesSet) Warnings() Warnings { return nil }

storage/interface.go

@@ -65,7 +65,7 @@ type Querier interface {
 	// Select returns a set of series that matches the given label matchers.
 	// Caller can specify if it requires returned series to be sorted. Prefer not requiring sorting for better performance.
 	// It allows passing hints that can help in optimising select, but it's up to implementation how this is used if used at all.
-	Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (SeriesSet, Warnings, error)
+	Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) SeriesSet
 }
 
 // A ChunkQueryable handles queries against a storage.
@@ -82,7 +82,7 @@ type ChunkQuerier interface {
 	// Select returns a set of series that matches the given label matchers.
 	// Caller can specify if it requires returned series to be sorted. Prefer not requiring sorting for better performance.
 	// It allows passing hints that can help in optimising select, but it's up to implementation how this is used if used at all.
-	Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) (ChunkSeriesSet, Warnings, error)
+	Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) ChunkSeriesSet
 }
 
 type baseQuerier interface {
@@ -153,7 +153,12 @@ type Appender interface {
 type SeriesSet interface {
 	Next() bool
 	At() Series
+	// The error that iteration as failed with.
+	// When an error occurs, set cannot continue to iterate.
 	Err() error
+	// A collection of warnings for the whole set.
+	// Warnings could be return even iteration has not failed with error.
+	Warnings() Warnings
 }
 
 var emptySeriesSet = errSeriesSet{}
@@ -164,12 +169,19 @@ func EmptySeriesSet() SeriesSet {
 }
 
 type errSeriesSet struct {
+	ws  Warnings
 	err error
 }
 
-func (s errSeriesSet) Next() bool { return false }
+func (s errSeriesSet) Next() bool         { return false }
-func (s errSeriesSet) At() Series { return nil }
+func (s errSeriesSet) At() Series         { return nil }
-func (s errSeriesSet) Err() error { return s.err }
+func (s errSeriesSet) Err() error         { return s.err }
+func (s errSeriesSet) Warnings() Warnings { return s.ws }
+
+// ErrSeriesSet returns a series set that wraps an error.
+func ErrSeriesSet(err error) SeriesSet {
+	return errSeriesSet{err: err}
+}
 
 // Series exposes a single time series and allows iterating over samples.
 type Series interface {
@@ -181,7 +193,12 @@ type Series interface {
 type ChunkSeriesSet interface {
 	Next() bool
 	At() ChunkSeries
+	// The error that iteration has failed with.
+	// When an error occurs, set cannot continue to iterate.
 	Err() error
+	// A collection of warnings for the whole set.
+	// Warnings could be return even iteration has not failed with error.
+	Warnings() Warnings
 }
 
 // ChunkSeries exposes a single time series and allows iterating over chunks.

storage/lazy.go

@@ -0,0 +1,67 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+type lazySeriesSet struct {
+	create func() (s genericSeriesSet, ok bool)
+
+	set genericSeriesSet
+}
+
+func (c *lazySeriesSet) Next() bool {
+	if c.set != nil {
+		return c.set.Next()
+	}
+
+	var ok bool
+	c.set, ok = c.create()
+	return ok
+}
+
+func (c *lazySeriesSet) Err() error {
+	if c.set != nil {
+		return c.set.Err()
+	}
+	return nil
+}
+
+func (c *lazySeriesSet) At() Labels {
+	if c.set != nil {
+		return c.set.At()
+	}
+	return nil
+}
+
+func (c *lazySeriesSet) Warnings() Warnings {
+	if c.set != nil {
+		return c.set.Warnings()
+	}
+	return nil
+}
+
+type warningsOnlySeriesSet Warnings
+
+func (warningsOnlySeriesSet) Next() bool           { return false }
+func (warningsOnlySeriesSet) Err() error           { return nil }
+func (warningsOnlySeriesSet) At() Labels           { return nil }
+func (c warningsOnlySeriesSet) Warnings() Warnings { return Warnings(c) }
+
+type errorOnlySeriesSet struct {
+	err error
+}
+
+func (errorOnlySeriesSet) Next() bool         { return false }
+func (errorOnlySeriesSet) At() Labels         { return nil }
+func (s errorOnlySeriesSet) Err() error       { return s.err }
+func (errorOnlySeriesSet) Warnings() Warnings { return nil }

storage/noop.go

@@ -24,8 +24,8 @@ func NoopQuerier() Querier {
 	return noopQuerier{}
 }
 
-func (noopQuerier) Select(bool, *SelectHints, ...*labels.Matcher) (SeriesSet, Warnings, error) {
+func (noopQuerier) Select(bool, *SelectHints, ...*labels.Matcher) SeriesSet {
-	return NoopSeriesSet(), nil, nil
+	return NoopSeriesSet()
 }
 
 func (noopQuerier) LabelValues(string) ([]string, Warnings, error) {
@@ -47,8 +47,8 @@ func NoopChunkedQuerier() ChunkQuerier {
 	return noopChunkQuerier{}
 }
 
-func (noopChunkQuerier) Select(bool, *SelectHints, ...*labels.Matcher) (ChunkSeriesSet, Warnings, error) {
+func (noopChunkQuerier) Select(bool, *SelectHints, ...*labels.Matcher) ChunkSeriesSet {
-	return NoopChunkedSeriesSet(), nil, nil
+	return NoopChunkedSeriesSet()
 }
 
 func (noopChunkQuerier) LabelValues(string) ([]string, Warnings, error) {
@@ -76,6 +76,8 @@ func (noopSeriesSet) At() Series { return nil }
 
 func (noopSeriesSet) Err() error { return nil }
 
+func (noopSeriesSet) Warnings() Warnings { return nil }
+
 type noopChunkedSeriesSet struct{}
 
 // NoopChunkedSeriesSet is a ChunkSeriesSet that does nothing.
@@ -88,3 +90,5 @@ func (noopChunkedSeriesSet) Next() bool { return false }
 func (noopChunkedSeriesSet) At() ChunkSeries { return nil }
 
 func (noopChunkedSeriesSet) Err() error { return nil }
+
+func (noopChunkedSeriesSet) Warnings() Warnings { return nil }

storage/remote/client.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -28,6 +29,7 @@ import (
 	"github.com/golang/snappy"
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
+	"github.com/prometheus/client_golang/prometheus"
 	config_util "github.com/prometheus/common/config"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/version"
@@ -40,6 +42,16 @@ const maxErrMsgLen = 256
 
 var userAgent = fmt.Sprintf("Prometheus/%s", version.Version)
 
+var remoteReadQueriesTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: namespace,
+		Subsystem: subsystem,
+		Name:      "read_queries_total",
+		Help:      "The total number of remote read queries.",
+	},
+	[]string{remoteName, endpoint, "code"},
+)
+
 // Client allows reading and writing from/to a remote HTTP endpoint.
 type Client struct {
 	remoteName string // Used to differentiate clients in metrics.
@@ -55,6 +67,10 @@ type ClientConfig struct {
 	HTTPClientConfig config_util.HTTPClientConfig
 }
 
+func init() {
+	prometheus.MustRegister(remoteReadQueriesTotal)
+}
+
 // NewClient creates a new Client.
 func NewClient(remoteName string, conf *ClientConfig) (*Client, error) {
 	httpClient, err := config_util.NewClientFromConfig(conf.HTTPClientConfig, "remote_storage", false)
@@ -193,6 +209,9 @@ func (c *Client) Read(ctx context.Context, query *prompb.Query) (*prompb.QueryRe
 		httpResp.Body.Close()
 	}()
 
+	remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String(), strconv.Itoa(httpResp.StatusCode))
+	remoteReadTotalCounter.Inc()
+
 	compressed, err = ioutil.ReadAll(httpResp.Body)
 	if err != nil {
 		return nil, errors.Wrap(err, fmt.Sprintf("error reading response. HTTP status code: %s", httpResp.Status))

storage/remote/codec.go

@@ -107,7 +107,7 @@ func ToQuery(from, to int64, matchers []*labels.Matcher, hints *storage.SelectHi
 }
 
 // ToQueryResult builds a QueryResult proto.
-func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, error) {
+func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult, storage.Warnings, error) {
 	numSamples := 0
 	resp := &prompb.QueryResult{}
 	for ss.Next() {
@@ -118,7 +118,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult,
 		for iter.Next() {
 			numSamples++
 			if sampleLimit > 0 && numSamples > sampleLimit {
-				return nil, HTTPError{
+				return nil, ss.Warnings(), HTTPError{
 					msg:    fmt.Sprintf("exceeded sample limit (%d)", sampleLimit),
 					status: http.StatusBadRequest,
 				}
@@ -130,7 +130,7 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult,
 			})
 		}
 		if err := iter.Err(); err != nil {
-			return nil, err
+			return nil, ss.Warnings(), err
 		}
 
 		resp.Timeseries = append(resp.Timeseries, &prompb.TimeSeries{
@@ -139,9 +139,9 @@ func ToQueryResult(ss storage.SeriesSet, sampleLimit int) (*prompb.QueryResult,
 		})
 	}
 	if err := ss.Err(); err != nil {
-		return nil, err
+		return nil, ss.Warnings(), err
 	}
-	return resp, nil
+	return resp, ss.Warnings(), nil
 }
 
 // FromQueryResult unpacks and sorts a QueryResult proto.
@@ -195,7 +195,7 @@ func StreamChunkedReadResponses(
 	ss storage.SeriesSet,
 	sortedExternalLabels []prompb.Label,
 	maxBytesInFrame int,
-) error {
+) (storage.Warnings, error) {
 	var (
 		chks     []prompb.Chunk
 		lbls     []prompb.Label
@@ -218,7 +218,7 @@ func StreamChunkedReadResponses(
 			// TODO(bwplotka): Use ChunkIterator once available in TSDB instead of re-encoding: https://github.com/prometheus/prometheus/pull/5882
 			chks, err = encodeChunks(iter, chks, maxBytesInFrame-lblsSize)
 			if err != nil {
-				return err
+				return ss.Warnings(), err
 			}
 
 			if len(chks) == 0 {
@@ -234,25 +234,25 @@ func StreamChunkedReadResponses(
 				QueryIndex: queryIndex,
 			})
 			if err != nil {
-				return errors.Wrap(err, "marshal ChunkedReadResponse")
+				return ss.Warnings(), errors.Wrap(err, "marshal ChunkedReadResponse")
 			}
 
 			if _, err := stream.Write(b); err != nil {
-				return errors.Wrap(err, "write to stream")
+				return ss.Warnings(), errors.Wrap(err, "write to stream")
 			}
 
 			chks = chks[:0]
 		}
 
 		if err := iter.Err(); err != nil {
-			return err
+			return ss.Warnings(), err
 		}
 	}
 	if err := ss.Err(); err != nil {
-		return err
+		return ss.Warnings(), err
 	}
 
-	return nil
+	return ss.Warnings(), nil
 }
 
 // encodeChunks expects iterator to be ready to use (aka iter.Next() called before invoking).
@@ -365,6 +365,8 @@ func (e errSeriesSet) Err() error {
 	return e.err
 }
 
+func (e errSeriesSet) Warnings() storage.Warnings { return nil }
+
 // concreteSeriesSet implements storage.SeriesSet.
 type concreteSeriesSet struct {
 	cur    int
@@ -384,6 +386,8 @@ func (c *concreteSeriesSet) Err() error {
 	return nil
 }
 
+func (c *concreteSeriesSet) Warnings() storage.Warnings { return nil }
+
 // concreteSeries implements storage.Series.
 type concreteSeries struct {
 	labels  labels.Labels

storage/remote/queue_manager.go

@@ -115,7 +115,7 @@ func newQueueManagerMetrics(r prometheus.Registerer, rn, e string) *queueManager
 		Subsystem:   subsystem,
 		Name:        "sent_batch_duration_seconds",
 		Help:        "Duration of sample batch send calls to the remote storage.",
-		Buckets:     prometheus.DefBuckets,
+		Buckets:     append(prometheus.DefBuckets, 25, 60, 120, 300),
 		ConstLabels: constLabels,
 	})
 	m.highestSentTimestamp = &maxGauge{

storage/remote/read.go

@@ -16,6 +16,7 @@ package remote
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/pkg/labels"
@@ -32,26 +33,26 @@ var remoteReadQueries = prometheus.NewGaugeVec(
 	[]string{remoteName, endpoint},
 )
 
-var remoteReadQueriesTotal = prometheus.NewCounterVec(
+var remoteReadQueriesHistogram = prometheus.NewHistogramVec(
-	prometheus.CounterOpts{
+	prometheus.HistogramOpts{
 		Namespace: namespace,
 		Subsystem: subsystem,
-		Name:      "remote_read_queries_total",
+		Name:      "read_request_duration_seconds",
-		Help:      "The total number of remote read queries.",
+		Help:      "Histogram of the latency for remote read requests.",
+		Buckets:   append(prometheus.DefBuckets, 25, 60),
 	},
 	[]string{remoteName, endpoint},
 )
 
 func init() {
-	prometheus.MustRegister(remoteReadQueries)
+	prometheus.MustRegister(remoteReadQueries, remoteReadQueriesHistogram)
-	prometheus.MustRegister(remoteReadQueriesTotal)
 }
 
 // QueryableClient returns a storage.Queryable which queries the given
 // Client to select series sets.
 func QueryableClient(c *Client) storage.Queryable {
 	remoteReadQueries.WithLabelValues(c.remoteName, c.url.String())
-	remoteReadQueriesTotal.WithLabelValues(c.remoteName, c.url.String())
+	remoteReadQueriesHistogram.WithLabelValues(c.remoteName, c.url.String())
 
 	return storage.QueryableFunc(func(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
 		return &querier{
@@ -71,25 +72,25 @@ type querier struct {
 }
 
 // Select implements storage.Querier and uses the given matchers to read series sets from the Client.
-func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
 	query, err := ToQuery(q.mint, q.maxt, matchers, hints)
 	if err != nil {
-		return nil, nil, err
+		return storage.ErrSeriesSet(err)
 	}
 
 	remoteReadGauge := remoteReadQueries.WithLabelValues(q.client.remoteName, q.client.url.String())
 	remoteReadGauge.Inc()
 	defer remoteReadGauge.Dec()
 
-	remoteReadTotalCounter := remoteReadQueriesTotal.WithLabelValues(q.client.remoteName, q.client.url.String())
+	remoteReadQueriesHistogram := remoteReadQueriesHistogram.WithLabelValues(q.client.remoteName, q.client.url.String())
-	remoteReadTotalCounter.Inc()
+	remoteReadQueriesHistogram.Observe(time.Since(time.Now()).Seconds())
 
 	res, err := q.client.Read(q.ctx, query)
 	if err != nil {
-		return nil, nil, fmt.Errorf("remote_read: %v", err)
+		return storage.ErrSeriesSet(fmt.Errorf("remote_read: %v", err))
 	}
 
-	return FromQueryResult(sortSeries, res), nil, nil
+	return FromQueryResult(sortSeries, res)
 }
 
 // LabelValues implements storage.Querier and is a noop.
@@ -132,13 +133,9 @@ type externalLabelsQuerier struct {
 // Select adds equality matchers for all external labels to the list of matchers
 // before calling the wrapped storage.Queryable. The added external labels are
 // removed from the returned series sets.
-func (q externalLabelsQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q externalLabelsQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
 	m, added := q.addExternalLabels(matchers)
-	s, warnings, err := q.Querier.Select(sortSeries, hints, m...)
+	return newSeriesSetFilter(q.Querier.Select(sortSeries, hints, m...), added)
-	if err != nil {
-		return nil, warnings, err
-	}
-	return newSeriesSetFilter(s, added), warnings, nil
 }
 
 // PreferLocalStorageFilter returns a QueryableFunc which creates a NoopQuerier
@@ -185,7 +182,7 @@ type requiredMatchersQuerier struct {
 
 // Select returns a NoopSeriesSet if the given matchers don't match the label
 // set of the requiredMatchersQuerier. Otherwise it'll call the wrapped querier.
-func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHints, matchers ...*labels.Matcher) storage.SeriesSet {
 	ms := q.requiredMatchers
 	for _, m := range matchers {
 		for i, r := range ms {
@@ -199,7 +196,7 @@ func (q requiredMatchersQuerier) Select(sortSeries bool, hints *storage.SelectHi
 		}
 	}
 	if len(ms) > 0 {
-		return storage.NoopSeriesSet(), nil, nil
+		return storage.NoopSeriesSet()
 	}
 	return q.Querier.Select(sortSeries, hints, matchers...)
 }

storage/remote/read_test.go

@@ -117,10 +117,8 @@ func TestExternalLabelsQuerierSelect(t *testing.T) {
 		},
 	}
 	want := newSeriesSetFilter(mockSeriesSet{}, q.externalLabels)
-	have, _, err := q.Select(false, nil, matchers...)
+	have := q.Select(false, nil, matchers...)
-	if err != nil {
+
-		t.Error(err)
-	}
 	if !reflect.DeepEqual(want, have) {
 		t.Errorf("expected series set %+v, got %+v", want, have)
 	}
@@ -218,16 +216,12 @@ func TestSeriesSetFilter(t *testing.T) {
 		},
 	}
 
-	for i, tc := range tests {
+	for _, tc := range tests {
 		filtered := newSeriesSetFilter(FromQueryResult(true, tc.in), tc.toRemove)
-		have, err := ToQueryResult(filtered, 1e6)
+		act, ws, err := ToQueryResult(filtered, 1e6)
-		if err != nil {
+		testutil.Ok(t, err)
-			t.Fatal(err)
+		testutil.Equals(t, 0, len(ws))
-		}
+		testutil.Equals(t, tc.expected, act)
-
-		if !reflect.DeepEqual(have, tc.expected) {
-			t.Fatalf("%d. unexpected labels; want %v, got %v", i, tc.expected, have)
-		}
 	}
 }
 
@@ -242,8 +236,8 @@ type mockSeriesSet struct {
 	storage.SeriesSet
 }
 
-func (mockQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (mockQuerier) Select(bool, *storage.SelectHints, ...*labels.Matcher) storage.SeriesSet {
-	return mockSeriesSet{}, nil, nil
+	return mockSeriesSet{}
 }
 
 func TestPreferLocalStorageFilter(t *testing.T) {
@@ -398,10 +392,8 @@ func TestRequiredLabelsQuerierSelect(t *testing.T) {
 			requiredMatchers: test.requiredMatchers,
 		}
 
-		have, _, err := q.Select(false, nil, test.matchers...)
+		have := q.Select(false, nil, test.matchers...)
-		if err != nil {
+
-			t.Error(err)
-		}
 		if want := test.seriesSet; want != have {
 			t.Errorf("%d. expected series set %+v, got %+v", i, want, have)
 		}

storage/remote/storage.go

@@ -132,6 +132,9 @@ func (s *Storage) StartTime() (int64, error) {
 
 // Querier returns a storage.MergeQuerier combining the remote client queriers
 // of each configured remote read endpoint.
+// Returned querier will never return error as all queryables are assumed best effort.
+// Additionally all returned queriers ensure that its Select's SeriesSets have ready data after first `Next` invoke.
+// This is because Prometheus (fanout and secondary queries) can't handle the stream failing half way through by design.
 func (s *Storage) Querier(ctx context.Context, mint, maxt int64) (storage.Querier, error) {
 	s.mtx.Lock()
 	queryables := s.queryables

storage/secondary.go

@@ -0,0 +1,112 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package storage
+
+import (
+	"sync"
+
+	"github.com/prometheus/prometheus/pkg/labels"
+)
+
+// secondaryQuerier is a wrapper that allows a querier to be treated in a best effort manner.
+// This means that an error on any method returned by Querier except Close will be returned as a warning,
+// and the result will be empty.
+//
+// Additionally, Querier ensures that if ANY SeriesSet returned by this querier's Select failed on an initial Next,
+// All other SeriesSet will be return no response as well. This ensures consistent partial response strategy, where you
+// have either full results or none from each secondary Querier.
+// NOTE: This works well only for implementations that only fail during first Next() (e.g fetch from network). If implementation fails
+// during further iterations, set will panic. If Select is invoked after first Next of any returned SeriesSet, querier will panic.
+//
+// Not go-routine safe.
+// NOTE: Prometheus treats all remote storages as secondary / best effort.
+type secondaryQuerier struct {
+	genericQuerier
+
+	once      sync.Once
+	done      bool
+	asyncSets []genericSeriesSet
+}
+
+func newSecondaryQuerierFrom(q Querier) genericQuerier {
+	return &secondaryQuerier{genericQuerier: newGenericQuerierFrom(q)}
+}
+
+func newSecondaryQuerierFromChunk(cq ChunkQuerier) genericQuerier {
+	return &secondaryQuerier{genericQuerier: newGenericQuerierFromChunk(cq)}
+}
+
+func (s *secondaryQuerier) LabelValues(name string) ([]string, Warnings, error) {
+	vals, w, err := s.genericQuerier.LabelValues(name)
+	if err != nil {
+		return nil, append([]error{err}, w...), nil
+	}
+	return vals, w, nil
+}
+
+func (s *secondaryQuerier) LabelNames() ([]string, Warnings, error) {
+	names, w, err := s.genericQuerier.LabelNames()
+	if err != nil {
+		return nil, append([]error{err}, w...), nil
+	}
+	return names, w, nil
+}
+
+func (s *secondaryQuerier) createFn(asyncSet genericSeriesSet) func() (genericSeriesSet, bool) {
+	s.asyncSets = append(s.asyncSets, asyncSet)
+	curr := len(s.asyncSets) - 1
+	return func() (genericSeriesSet, bool) {
+		s.once.Do(func() {
+			// At first create invocation we iterate over all sets and ensure its Next() returns some value without
+			// errors. This is to ensure we support consistent partial failures.
+			for i, set := range s.asyncSets {
+				if set.Next() {
+					continue
+				}
+				ws := set.Warnings()
+				// Failed set.
+				if err := set.Err(); err != nil {
+					ws = append([]error{err}, ws...)
+					// Promote the warnings to the current one.
+					s.asyncSets[curr] = warningsOnlySeriesSet(ws)
+					// One of the sets failed, ensure rest of the sets returns nothing. (All or nothing logic).
+					for i := range s.asyncSets {
+						if curr != i {
+							s.asyncSets[i] = noopGenericSeriesSet{}
+						}
+					}
+					break
+				}
+				// Exhausted set.
+				s.asyncSets[i] = warningsOnlySeriesSet(ws)
+			}
+
+			s.done = true
+		})
+
+		switch s.asyncSets[curr].(type) {
+		case warningsOnlySeriesSet, noopGenericSeriesSet:
+			return s.asyncSets[curr], false
+		default:
+			return s.asyncSets[curr], true
+		}
+	}
+}
+
+func (s *secondaryQuerier) Select(sortSeries bool, hints *SelectHints, matchers ...*labels.Matcher) genericSeriesSet {
+	if s.done {
+		panic("secondaryQuerier: Select invoked after first Next of any returned SeriesSet was done")
+	}
+	return &lazySeriesSet{create: s.createFn(s.genericQuerier.Select(sortSeries, hints, matchers...))}
+}

tsdb/block_test.go

@@ -203,9 +203,7 @@ func TestCorruptedChunk(t *testing.T) {
 			querier, err := NewBlockQuerier(b, 0, 1)
 			testutil.Ok(t, err)
 			defer func() { testutil.Ok(t, querier.Close()) }()
-			set, ws, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
+			set := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
-			testutil.Ok(t, err)
-			testutil.Equals(t, 0, len(ws))
 
 			// Check query err.
 			testutil.Equals(t, false, set.Next())

tsdb/chunkenc/bstream.go

@@ -41,7 +41,10 @@
 
 package chunkenc
 
-import "io"
+import (
+	"encoding/binary"
+	"io"
+)
 
 // bstream is a stream of bits.
 type bstream struct {
@@ -49,10 +52,6 @@ type bstream struct {
 	count  uint8  // how many bits are valid in current byte
 }
 
-func newBReader(b []byte) bstream {
-	return bstream{stream: b, count: 8}
-}
-
 func (b *bstream) bytes() []byte {
 	return b.stream
 }
@@ -111,90 +110,131 @@ func (b *bstream) writeBits(u uint64, nbits int) {
 	}
 }
 
-func (b *bstream) readBit() (bit, error) {
+type bstreamReader struct {
-	if len(b.stream) == 0 {
+	stream       []byte
+	streamOffset int // The offset from which read the next byte from the stream.
+
+	buffer uint64 // The current buffer, filled from the stream, containing up to 8 bytes from which read bits.
+	valid  uint8  // The number of bits valid to read (from left) in the current buffer.
+}
+
+func newBReader(b []byte) bstreamReader {
+	return bstreamReader{
+		stream: b,
+	}
+}
+
+func (b *bstreamReader) readBit() (bit, error) {
+	if b.valid == 0 {
+		if !b.loadNextBuffer(1) {
+			return false, io.EOF
+		}
+	}
+
+	return b.readBitFast()
+}
+
+// readBitFast is like readBit but can return io.EOF if the internal buffer is empty.
+// If it returns io.EOF, the caller should retry reading bits calling readBit().
+// This function must be kept small and a leaf in order to help the compiler inlining it
+// and further improve performances.
+func (b *bstreamReader) readBitFast() (bit, error) {
+	if b.valid == 0 {
 		return false, io.EOF
 	}
 
-	if b.count == 0 {
+	b.valid--
-		b.stream = b.stream[1:]
+	bitmask := uint64(1) << b.valid
+	return (b.buffer & bitmask) != 0, nil
+}
 
-		if len(b.stream) == 0 {
+func (b *bstreamReader) readBits(nbits uint8) (uint64, error) {
-			return false, io.EOF
+	if b.valid == 0 {
+		if !b.loadNextBuffer(nbits) {
+			return 0, io.EOF
 		}
-		b.count = 8
 	}
 
-	d := (b.stream[0] << (8 - b.count)) & 0x80
+	if nbits <= b.valid {
-	b.count--
+		return b.readBitsFast(nbits)
-	return d != 0, nil
+	}
-}
 
-func (b *bstream) ReadByte() (byte, error) {
+	// We have to read all remaining valid bits from the current buffer and a part from the next one.
-	return b.readByte()
+	bitmask := (uint64(1) << b.valid) - 1
-}
+	nbits -= b.valid
+	v := (b.buffer & bitmask) << nbits
+	b.valid = 0
 
-func (b *bstream) readByte() (byte, error) {
+	if !b.loadNextBuffer(nbits) {
-	if len(b.stream) == 0 {
 		return 0, io.EOF
 	}
 
-	if b.count == 0 {
+	bitmask = (uint64(1) << nbits) - 1
-		b.stream = b.stream[1:]
+	v = v | ((b.buffer >> (b.valid - nbits)) & bitmask)
+	b.valid -= nbits
 
-		if len(b.stream) == 0 {
+	return v, nil
-			return 0, io.EOF
+}
-		}
-		return b.stream[0], nil
-	}
 
-	if b.count == 8 {
+// readBitsFast is like readBits but can return io.EOF if the internal buffer is empty.
-		b.count = 0
+// If it returns io.EOF, the caller should retry reading bits calling readBits().
-		return b.stream[0], nil
+// This function must be kept small and a leaf in order to help the compiler inlining it
-	}
+// and further improve performances.
-
+func (b *bstreamReader) readBitsFast(nbits uint8) (uint64, error) {
-	byt := b.stream[0] << (8 - b.count)
+	if nbits > b.valid {
-	b.stream = b.stream[1:]
-
-	if len(b.stream) == 0 {
 		return 0, io.EOF
 	}
 
-	// We just advanced the stream and can assume the shift to be 0.
+	bitmask := (uint64(1) << nbits) - 1
-	byt |= b.stream[0] >> b.count
+	b.valid -= nbits
 
-	return byt, nil
+	return (b.buffer >> b.valid) & bitmask, nil
 }
 
-func (b *bstream) readBits(nbits int) (uint64, error) {
+func (b *bstreamReader) ReadByte() (byte, error) {
-	var u uint64
+	v, err := b.readBits(8)
-
+	if err != nil {
-	for nbits >= 8 {
+		return 0, err
-		byt, err := b.readByte()
-		if err != nil {
-			return 0, err
-		}
-
-		u = (u << 8) | uint64(byt)
-		nbits -= 8
 	}
-
+	return byte(v), nil
-	if nbits == 0 {
+}
-		return u, nil
+
-	}
+// loadNextBuffer loads the next bytes from the stream into the internal buffer.
-
+// The input nbits is the minimum number of bits that must be read, but the implementation
-	if nbits > int(b.count) {
+// can read more (if possible) to improve performances.
-		u = (u << uint(b.count)) | uint64((b.stream[0]<<(8-b.count))>>(8-b.count))
+func (b *bstreamReader) loadNextBuffer(nbits uint8) bool {
-		nbits -= int(b.count)
+	if b.streamOffset >= len(b.stream) {
-		b.stream = b.stream[1:]
+		return false
-
+	}
-		if len(b.stream) == 0 {
+
-			return 0, io.EOF
+	// Handle the case there are more then 8 bytes in the buffer (most common case)
-		}
+	// in a optimized way. It's guaranteed that this branch will never read from the
-		b.count = 8
+	// very last byte of the stream (which suffers race conditions due to concurrent
-	}
+	// writes).
-
+	if b.streamOffset+8 < len(b.stream) {
-	u = (u << uint(nbits)) | uint64((b.stream[0]<<(8-b.count))>>(8-uint(nbits)))
+		b.buffer = binary.BigEndian.Uint64(b.stream[b.streamOffset:])
-	b.count -= uint8(nbits)
+		b.streamOffset += 8
-	return u, nil
+		b.valid = 64
+		return true
+	}
+
+	// We're here if the are 8 or less bytes left in the stream. Since this reader needs
+	// to handle race conditions with concurrent writes happening on the very last byte
+	// we make sure to never over more than the minimum requested bits (rounded up to
+	// the next byte). The following code is slower but called less frequently.
+	nbytes := int((nbits / 8) + 1)
+	if b.streamOffset+nbytes > len(b.stream) {
+		nbytes = len(b.stream) - b.streamOffset
+	}
+
+	buffer := uint64(0)
+	for i := 0; i < nbytes; i++ {
+		buffer = buffer | (uint64(b.stream[b.streamOffset+i]) << uint(8*(nbytes-i-1)))
+	}
+
+	b.buffer = buffer
+	b.streamOffset += nbytes
+	b.valid = uint8(nbytes * 8)
+
+	return true
 }

tsdb/chunkenc/bstream_test.go

@@ -0,0 +1,61 @@
+// Copyright 2020 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chunkenc
+
+import (
+	"testing"
+
+	"github.com/prometheus/prometheus/util/testutil"
+)
+
+func TestBstreamReader(t *testing.T) {
+	// Write to the bit stream.
+	w := bstream{}
+	for _, bit := range []bit{true, false} {
+		w.writeBit(bit)
+	}
+	for nbits := 1; nbits <= 64; nbits++ {
+		w.writeBits(uint64(nbits), nbits)
+	}
+	for v := 1; v < 10000; v += 123 {
+		w.writeBits(uint64(v), 29)
+	}
+
+	// Read back.
+	r := newBReader(w.bytes())
+	for _, bit := range []bit{true, false} {
+		v, err := r.readBitFast()
+		if err != nil {
+			v, err = r.readBit()
+		}
+		testutil.Ok(t, err)
+		testutil.Equals(t, bit, v)
+	}
+	for nbits := uint8(1); nbits <= 64; nbits++ {
+		v, err := r.readBitsFast(nbits)
+		if err != nil {
+			v, err = r.readBits(nbits)
+		}
+		testutil.Ok(t, err)
+		testutil.Equals(t, uint64(nbits), v, "nbits=%d", nbits)
+	}
+	for v := 1; v < 10000; v += 123 {
+		actual, err := r.readBitsFast(29)
+		if err != nil {
+			actual, err = r.readBits(29)
+		}
+		testutil.Ok(t, err)
+		testutil.Equals(t, uint64(v), actual, "v=%d", v)
+	}
+}

tsdb/chunkenc/xor.go

@@ -240,7 +240,7 @@ func (a *xorAppender) writeVDelta(v float64) {
 }
 
 type xorIterator struct {
-	br       bstream
+	br       bstreamReader
 	numTotal uint16
 	numRead  uint16
 
@@ -328,7 +328,10 @@ func (it *xorIterator) Next() bool {
 	// read delta-of-delta
 	for i := 0; i < 4; i++ {
 		d <<= 1
-		bit, err := it.br.readBit()
+		bit, err := it.br.readBitFast()
+		if err != nil {
+			bit, err = it.br.readBit()
+		}
 		if err != nil {
 			it.err = err
 			return false
@@ -350,6 +353,7 @@ func (it *xorIterator) Next() bool {
 	case 0x0e:
 		sz = 20
 	case 0x0f:
+		// Do not use fast because it's very unlikely it will succeed.
 		bits, err := it.br.readBits(64)
 		if err != nil {
 			it.err = err
@@ -360,7 +364,10 @@ func (it *xorIterator) Next() bool {
 	}
 
 	if sz != 0 {
-		bits, err := it.br.readBits(int(sz))
+		bits, err := it.br.readBitsFast(sz)
+		if err != nil {
+			bits, err = it.br.readBits(sz)
+		}
 		if err != nil {
 			it.err = err
 			return false
@@ -379,7 +386,10 @@ func (it *xorIterator) Next() bool {
 }
 
 func (it *xorIterator) readValue() bool {
-	bit, err := it.br.readBit()
+	bit, err := it.br.readBitFast()
+	if err != nil {
+		bit, err = it.br.readBit()
+	}
 	if err != nil {
 		it.err = err
 		return false
@@ -388,7 +398,10 @@ func (it *xorIterator) readValue() bool {
 	if bit == zero {
 		// it.val = it.val
 	} else {
-		bit, err := it.br.readBit()
+		bit, err := it.br.readBitFast()
+		if err != nil {
+			bit, err = it.br.readBit()
+		}
 		if err != nil {
 			it.err = err
 			return false
@@ -397,14 +410,20 @@ func (it *xorIterator) readValue() bool {
 			// reuse leading/trailing zero bits
 			// it.leading, it.trailing = it.leading, it.trailing
 		} else {
-			bits, err := it.br.readBits(5)
+			bits, err := it.br.readBitsFast(5)
+			if err != nil {
+				bits, err = it.br.readBits(5)
+			}
 			if err != nil {
 				it.err = err
 				return false
 			}
 			it.leading = uint8(bits)
 
-			bits, err = it.br.readBits(6)
+			bits, err = it.br.readBitsFast(6)
+			if err != nil {
+				bits, err = it.br.readBits(6)
+			}
 			if err != nil {
 				it.err = err
 				return false
@@ -417,14 +436,17 @@ func (it *xorIterator) readValue() bool {
 			it.trailing = 64 - it.leading - mbits
 		}
 
-		mbits := int(64 - it.leading - it.trailing)
+		mbits := 64 - it.leading - it.trailing
-		bits, err := it.br.readBits(mbits)
+		bits, err := it.br.readBitsFast(mbits)
+		if err != nil {
+			bits, err = it.br.readBits(mbits)
+		}
 		if err != nil {
 			it.err = err
 			return false
 		}
 		vbits := math.Float64bits(it.val)
-		vbits ^= (bits << it.trailing)
+		vbits ^= bits << it.trailing
 		it.val = math.Float64frombits(vbits)
 	}
 

tsdb/chunks/head_chunks.go

@@ -190,23 +190,23 @@ func (cdm *ChunkDiskMapper) openMMapFiles() (returnErr error) {
 	lastSeq := chkFileIndices[0]
 	for _, seq := range chkFileIndices[1:] {
 		if seq != lastSeq+1 {
-			return errors.Errorf("found unsequential head chunk files %d and %d", lastSeq, seq)
+			return errors.Errorf("found unsequential head chunk files %s (index: %d) and %s (index: %d)", files[lastSeq], lastSeq, files[seq], seq)
 		}
 		lastSeq = seq
 	}
 
 	for i, b := range cdm.mmappedChunkFiles {
 		if b.byteSlice.Len() < HeadChunkFileHeaderSize {
-			return errors.Wrapf(errInvalidSize, "invalid head chunk file header in file %d", i)
+			return errors.Wrapf(errInvalidSize, "%s: invalid head chunk file header", files[i])
 		}
 		// Verify magic number.
 		if m := binary.BigEndian.Uint32(b.byteSlice.Range(0, MagicChunksSize)); m != MagicHeadChunks {
-			return errors.Errorf("invalid magic number %x", m)
+			return errors.Errorf("%s: invalid magic number %x", files[i], m)
 		}
 
 		// Verify chunk format version.
 		if v := int(b.byteSlice.Range(MagicChunksSize, MagicChunksSize+ChunksFormatVersionSize)[0]); v != chunksFormatV1 {
-			return errors.Errorf("invalid chunk format version %d", v)
+			return errors.Errorf("%s: invalid chunk format version %d", files[i], v)
 		}
 
 		cdm.size += int64(b.byteSlice.Len())

tsdb/cmd/tsdb/main.go

@@ -617,18 +617,7 @@ func dumpSamples(db *tsdb.DBReadOnly, mint, maxt int64) (err error) {
 		err = merr.Err()
 	}()
 
-	ss, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "", ".*"))
+	ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "", ".*"))
-	if err != nil {
-		return err
-	}
-
-	if len(ws) > 0 {
-		var merr tsdb_errors.MultiError
-		for _, w := range ws {
-			merr.Add(w)
-		}
-		return merr.Err()
-	}
 
 	for ss.Next() {
 		series := ss.At()
@@ -643,6 +632,14 @@ func dumpSamples(db *tsdb.DBReadOnly, mint, maxt int64) (err error) {
 		}
 	}
 
+	if ws := ss.Warnings(); len(ws) > 0 {
+		var merr tsdb_errors.MultiError
+		for _, w := range ws {
+			merr.Add(w)
+		}
+		return merr.Err()
+	}
+
 	if ss.Err() != nil {
 		return ss.Err()
 	}

tsdb/db.go

@@ -441,10 +441,14 @@ func (db *DBReadOnly) Blocks() ([]BlockReader, error) {
 	if len(corrupted) > 0 {
 		for _, b := range loadable {
 			if err := b.Close(); err != nil {
-				level.Warn(db.logger).Log("msg", "Closing a block", err)
+				level.Warn(db.logger).Log("msg", "Closing block failed", "err", err, "block", b)
 			}
 		}
-		return nil, errors.Errorf("unexpected corrupted block:%v", corrupted)
+		var merr tsdb_errors.MultiError
+		for ulid, err := range corrupted {
+			merr.Add(errors.Wrapf(err, "corrupted block %s", ulid.String()))
+		}
+		return nil, merr.Err()
 	}
 
 	if len(loadable) == 0 {
@@ -880,7 +884,11 @@ func (db *DB) reload() (err error) {
 				block.Close()
 			}
 		}
-		return fmt.Errorf("unexpected corrupted block:%v", corrupted)
+		var merr tsdb_errors.MultiError
+		for ulid, err := range corrupted {
+			merr.Add(errors.Wrapf(err, "corrupted block %s", ulid.String()))
+		}
+		return merr.Err()
 	}
 
 	// All deletable blocks should not be loaded.
@@ -1054,7 +1062,7 @@ func (db *DB) deleteBlocks(blocks map[ulid.ULID]*Block) error {
 	for ulid, block := range blocks {
 		if block != nil {
 			if err := block.Close(); err != nil {
-				level.Warn(db.logger).Log("msg", "Closing block failed", "err", err)
+				level.Warn(db.logger).Log("msg", "Closing block failed", "err", err, "block", ulid)
 			}
 		}
 		if err := os.RemoveAll(filepath.Join(db.dir, ulid.String())); err != nil {

tsdb/db_test.go

@@ -67,15 +67,12 @@ func openTestDB(t testing.TB, opts *Options, rngs []int64) (db *DB, close func()
 
 // query runs a matcher query against the querier and fully expands its data.
 func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[string][]tsdbutil.Sample {
-	ss, ws, err := q.Select(false, nil, matchers...)
+	ss := q.Select(false, nil, matchers...)
 	defer func() {
 		testutil.Ok(t, q.Close())
 	}()
-	testutil.Ok(t, err)
-	testutil.Equals(t, 0, len(ws))
 
 	result := map[string][]tsdbutil.Sample{}
-
 	for ss.Next() {
 		series := ss.At()
 
@@ -95,6 +92,7 @@ func query(t testing.TB, q storage.Querier, matchers ...*labels.Matcher) map[str
 		result[name] = samples
 	}
 	testutil.Ok(t, ss.Err())
+	testutil.Equals(t, 0, len(ss.Warnings()))
 
 	return result
 }
@@ -315,9 +313,7 @@ Outer:
 		q, err := db.Querier(context.TODO(), 0, numSamples)
 		testutil.Ok(t, err)
 
-		res, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
+		res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
-		testutil.Ok(t, err)
-		testutil.Equals(t, 0, len(ws))
 
 		expSamples := make([]tsdbutil.Sample, 0, len(c.remaint))
 		for _, ts := range c.remaint {
@@ -333,6 +329,7 @@ Outer:
 			testutil.Equals(t, eok, rok)
 
 			if !eok {
+				testutil.Equals(t, 0, len(res.Warnings()))
 				continue Outer
 			}
 			sexp := expss.At()
@@ -491,10 +488,7 @@ func TestDB_Snapshot(t *testing.T) {
 	defer func() { testutil.Ok(t, querier.Close()) }()
 
 	// sum values
-	seriesSet, ws, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+	seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
-	testutil.Ok(t, err)
-	testutil.Equals(t, 0, len(ws))
-
 	sum := 0.0
 	for seriesSet.Next() {
 		series := seriesSet.At().Iterator()
@@ -505,6 +499,7 @@ func TestDB_Snapshot(t *testing.T) {
 		testutil.Ok(t, series.Err())
 	}
 	testutil.Ok(t, seriesSet.Err())
+	testutil.Equals(t, 0, len(seriesSet.Warnings()))
 	testutil.Equals(t, 1000.0, sum)
 }
 
@@ -546,10 +541,7 @@ func TestDB_Snapshot_ChunksOutsideOfCompactedRange(t *testing.T) {
 	defer func() { testutil.Ok(t, querier.Close()) }()
 
 	// Sum values.
-	seriesSet, ws, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+	seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
-	testutil.Ok(t, err)
-	testutil.Equals(t, 0, len(ws))
-
 	sum := 0.0
 	for seriesSet.Next() {
 		series := seriesSet.At().Iterator()
@@ -560,6 +552,7 @@ func TestDB_Snapshot_ChunksOutsideOfCompactedRange(t *testing.T) {
 		testutil.Ok(t, series.Err())
 	}
 	testutil.Ok(t, seriesSet.Err())
+	testutil.Equals(t, 0, len(seriesSet.Warnings()))
 
 	// Since we snapshotted with MaxTime - 10, so expect 10 less samples.
 	testutil.Equals(t, 1000.0-10, sum)
@@ -618,9 +611,7 @@ Outer:
 		testutil.Ok(t, err)
 		defer func() { testutil.Ok(t, q.Close()) }()
 
-		res, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
+		res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
-		testutil.Ok(t, err)
-		testutil.Equals(t, 0, len(ws))
 
 		expSamples := make([]tsdbutil.Sample, 0, len(c.remaint))
 		for _, ts := range c.remaint {
@@ -641,6 +632,7 @@ Outer:
 			testutil.Equals(t, eok, rok)
 
 			if !eok {
+				testutil.Equals(t, 0, len(res.Warnings()))
 				continue Outer
 			}
 			sexp := expss.At()
@@ -792,10 +784,7 @@ func TestDB_e2e(t *testing.T) {
 			q, err := db.Querier(context.TODO(), mint, maxt)
 			testutil.Ok(t, err)
 
-			ss, ws, err := q.Select(false, nil, qry.ms...)
+			ss := q.Select(false, nil, qry.ms...)
-			testutil.Ok(t, err)
-			testutil.Equals(t, 0, len(ws))
-
 			result := map[string][]tsdbutil.Sample{}
 
 			for ss.Next() {
@@ -810,6 +799,7 @@ func TestDB_e2e(t *testing.T) {
 			}
 
 			testutil.Ok(t, ss.Err())
+			testutil.Equals(t, 0, len(ss.Warnings()))
 			testutil.Equals(t, expected, result)
 
 			q.Close()
@@ -968,9 +958,7 @@ func TestTombstoneClean(t *testing.T) {
 		testutil.Ok(t, err)
 		defer q.Close()
 
-		res, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
+		res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
-		testutil.Ok(t, err)
-		testutil.Equals(t, 0, len(ws))
 
 		expSamples := make([]tsdbutil.Sample, 0, len(c.remaint))
 		for _, ts := range c.remaint {
@@ -1004,6 +992,7 @@ func TestTombstoneClean(t *testing.T) {
 			testutil.Equals(t, errExp, errRes)
 			testutil.Equals(t, smplExp, smplRes)
 		}
+		testutil.Equals(t, 0, len(res.Warnings()))
 
 		for _, b := range db.Blocks() {
 			testutil.Equals(t, tombstones.NewMemTombstones(), b.tombstones)
@@ -1306,19 +1295,17 @@ func TestNotMatcherSelectsLabelsUnsetSeries(t *testing.T) {
 	defer func() { testutil.Ok(t, q.Close()) }()
 
 	for _, c := range cases {
-		ss, ws, err := q.Select(false, nil, c.selector...)
+		ss := q.Select(false, nil, c.selector...)
+		lres, _, ws, err := expandSeriesSet(ss)
 		testutil.Ok(t, err)
 		testutil.Equals(t, 0, len(ws))
-
-		lres, _, err := expandSeriesSet(ss)
-		testutil.Ok(t, err)
 		testutil.Equals(t, c.series, lres)
 	}
 }
 
 // expandSeriesSet returns the raw labels in the order they are retrieved from
 // the series set and the samples keyed by Labels().String().
-func expandSeriesSet(ss storage.SeriesSet) ([]labels.Labels, map[string][]sample, error) {
+func expandSeriesSet(ss storage.SeriesSet) ([]labels.Labels, map[string][]sample, storage.Warnings, error) {
 	resultLabels := []labels.Labels{}
 	resultSamples := map[string][]sample{}
 	for ss.Next() {
@@ -1332,7 +1319,7 @@ func expandSeriesSet(ss storage.SeriesSet) ([]labels.Labels, map[string][]sample
 		resultLabels = append(resultLabels, series.Labels())
 		resultSamples[series.Labels().String()] = samples
 	}
-	return resultLabels, resultSamples, ss.Err()
+	return resultLabels, resultSamples, ss.Warnings(), ss.Err()
 }
 
 func TestOverlappingBlocksDetectsAllOverlaps(t *testing.T) {
@@ -2503,9 +2490,7 @@ func TestDBReadOnly_FlushWAL(t *testing.T) {
 	defer func() { testutil.Ok(t, querier.Close()) }()
 
 	// Sum the values.
-	seriesSet, ws, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, defaultLabelName, "flush"))
+	seriesSet := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, defaultLabelName, "flush"))
-	testutil.Ok(t, err)
-	testutil.Equals(t, 0, len(ws))
 
 	sum := 0.0
 	for seriesSet.Next() {
@@ -2517,6 +2502,7 @@ func TestDBReadOnly_FlushWAL(t *testing.T) {
 		testutil.Ok(t, series.Err())
 	}
 	testutil.Ok(t, seriesSet.Err())
+	testutil.Equals(t, 0, len(seriesSet.Warnings()))
 	testutil.Equals(t, 1000.0, sum)
 }
 
@@ -2568,11 +2554,11 @@ func TestDBCannotSeePartialCommits(t *testing.T) {
 			testutil.Ok(t, err)
 			defer querier.Close()
 
-			ss, _, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+			ss := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+			_, seriesSet, ws, err := expandSeriesSet(ss)
 			testutil.Ok(t, err)
+			testutil.Equals(t, 0, len(ws))
 
-			_, seriesSet, err := expandSeriesSet(ss)
-			testutil.Ok(t, err)
 			values := map[float64]struct{}{}
 			for _, series := range seriesSet {
 				values[series[len(series)-1].v] = struct{}{}
@@ -2608,16 +2594,16 @@ func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) {
 	defer querierAfterAddButBeforeCommit.Close()
 
 	// None of the queriers should return anything after the Add but before the commit.
-	ss, _, err := querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+	ss := querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
-	testutil.Ok(t, err)
+	_, seriesSet, ws, err := expandSeriesSet(ss)
-	_, seriesSet, err := expandSeriesSet(ss)
 	testutil.Ok(t, err)
+	testutil.Equals(t, 0, len(ws))
 	testutil.Equals(t, map[string][]sample{}, seriesSet)
 
-	ss, _, err = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+	ss = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
-	testutil.Ok(t, err)
+	_, seriesSet, ws, err = expandSeriesSet(ss)
-	_, seriesSet, err = expandSeriesSet(ss)
 	testutil.Ok(t, err)
+	testutil.Equals(t, 0, len(ws))
 	testutil.Equals(t, map[string][]sample{}, seriesSet)
 
 	// This commit is after the queriers are created, so should not be returned.
@@ -2625,17 +2611,17 @@ func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) {
 	testutil.Ok(t, err)
 
 	// Nothing returned for querier created before the Add.
-	ss, _, err = querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+	ss = querierBeforeAdd.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
-	testutil.Ok(t, err)
+	_, seriesSet, ws, err = expandSeriesSet(ss)
-	_, seriesSet, err = expandSeriesSet(ss)
 	testutil.Ok(t, err)
+	testutil.Equals(t, 0, len(ws))
 	testutil.Equals(t, map[string][]sample{}, seriesSet)
 
 	// Series exists but has no samples for querier created after Add.
-	ss, _, err = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+	ss = querierAfterAddButBeforeCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
-	testutil.Ok(t, err)
+	_, seriesSet, ws, err = expandSeriesSet(ss)
-	_, seriesSet, err = expandSeriesSet(ss)
 	testutil.Ok(t, err)
+	testutil.Equals(t, 0, len(ws))
 	testutil.Equals(t, map[string][]sample{`{foo="bar"}`: {}}, seriesSet)
 
 	querierAfterCommit, err := db.Querier(context.Background(), 0, 1000000)
@@ -2643,10 +2629,10 @@ func TestDBQueryDoesntSeeAppendsAfterCreation(t *testing.T) {
 	defer querierAfterCommit.Close()
 
 	// Samples are returned for querier created after Commit.
-	ss, _, err = querierAfterCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+	ss = querierAfterCommit.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
-	testutil.Ok(t, err)
+	_, seriesSet, ws, err = expandSeriesSet(ss)
-	_, seriesSet, err = expandSeriesSet(ss)
 	testutil.Ok(t, err)
+	testutil.Equals(t, 0, len(ws))
 	testutil.Equals(t, map[string][]sample{`{foo="bar"}`: {{t: 0, v: 0}}}, seriesSet)
 }
 
@@ -2872,3 +2858,94 @@ func TestChunkReader_ConcurrentReads(t *testing.T) {
 	}
 	testutil.Ok(t, r.Close())
 }
+
+// TestCompactHead ensures that the head compaction
+// creates a block that is ready for loading and
+// does not cause data loss.
+// This test:
+// * opens a storage;
+// * appends values;
+// * compacts the head; and
+// * queries the db to ensure the samples are present from the compacted head.
+func TestCompactHead(t *testing.T) {
+	dbDir, err := ioutil.TempDir("", "testFlush")
+	testutil.Ok(t, err)
+	defer func() { testutil.Ok(t, os.RemoveAll(dbDir)) }()
+
+	// Open a DB and append data to the WAL.
+	tsdbCfg := &Options{
+		RetentionDuration: int64(time.Hour * 24 * 15 / time.Millisecond),
+		NoLockfile:        true,
+		MinBlockDuration:  int64(time.Hour * 2 / time.Millisecond),
+		MaxBlockDuration:  int64(time.Hour * 2 / time.Millisecond),
+		WALCompression:    true,
+	}
+
+	db, err := Open(dbDir, log.NewNopLogger(), prometheus.NewRegistry(), tsdbCfg)
+	testutil.Ok(t, err)
+	app := db.Appender()
+	var expSamples []sample
+	maxt := 100
+	for i := 0; i < maxt; i++ {
+		val := rand.Float64()
+		_, err := app.Add(labels.FromStrings("a", "b"), int64(i), val)
+		testutil.Ok(t, err)
+		expSamples = append(expSamples, sample{int64(i), val})
+	}
+	testutil.Ok(t, app.Commit())
+
+	// Compact the Head to create a new block.
+	testutil.Ok(t, db.CompactHead(NewRangeHead(db.Head(), 0, int64(maxt)-1)))
+	testutil.Ok(t, db.Close())
+
+	// Delete everything but the new block and
+	// reopen the db to query it to ensure it includes the head data.
+	testutil.Ok(t, deleteNonBlocks(db.Dir()))
+	db, err = Open(dbDir, log.NewNopLogger(), prometheus.NewRegistry(), tsdbCfg)
+	testutil.Ok(t, err)
+	testutil.Equals(t, 1, len(db.Blocks()))
+	testutil.Equals(t, int64(maxt), db.Head().MinTime())
+	defer func() { testutil.Ok(t, db.Close()) }()
+	querier, err := db.Querier(context.Background(), 0, int64(maxt)-1)
+	testutil.Ok(t, err)
+	defer func() { testutil.Ok(t, querier.Close()) }()
+
+	seriesSet := querier.Select(false, nil, &labels.Matcher{Type: labels.MatchEqual, Name: "a", Value: "b"})
+	var actSamples []sample
+
+	for seriesSet.Next() {
+		series := seriesSet.At().Iterator()
+		for series.Next() {
+			time, val := series.At()
+			actSamples = append(actSamples, sample{int64(time), val})
+		}
+		testutil.Ok(t, series.Err())
+	}
+	testutil.Equals(t, expSamples, actSamples)
+	testutil.Ok(t, seriesSet.Err())
+}
+
+func deleteNonBlocks(dbDir string) error {
+	dirs, err := ioutil.ReadDir(dbDir)
+	if err != nil {
+		return err
+	}
+	for _, dir := range dirs {
+		if ok := isBlockDir(dir); !ok {
+			if err := os.RemoveAll(filepath.Join(dbDir, dir.Name())); err != nil {
+				return err
+			}
+		}
+	}
+	dirs, err = ioutil.ReadDir(dbDir)
+	if err != nil {
+		return err
+	}
+	for _, dir := range dirs {
+		if ok := isBlockDir(dir); !ok {
+			return errors.Errorf("root folder:%v still hase non block directory:%v", dbDir, dir.Name())
+		}
+	}
+
+	return nil
+}

tsdb/docs/format/README.md

@@ -2,5 +2,6 @@
 
 * [Index](index.md)
 * [Chunks](chunks.md)
+* [Head Chunks](head_chunks.md)
 * [Tombstones](tombstones.md)
 * [Wal](wal.md)

tsdb/docs/format/head_chunks.md

@@ -1,7 +1,7 @@
 # Head Chunks on Disk Format
 
-The following describes the format of a chunks file,
+The following describes the format of a chunks file, which is created in the
-which is created in the `wal/chunks/` inside the data directory.
+`chunks_head/` directory inside the data directory.
 
 Chunks in the files are referenced from the index by uint64 composed of
 in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes).
@@ -27,10 +27,13 @@ in-file offset (lower 4 bytes) and segment sequence number (upper 4 bytes).
 
 # Chunk
 
-Unlike chunks in the on-disk blocks, here we additionally store series reference that the chunks belongs to and the mint/maxt of the chunks. This is because we don't have an index associated with these chunks, hence these meta information are used while replaying the chunks.
+Unlike chunks in the on-disk blocks, here we additionally store series
+reference that the chunks belongs to and the mint/maxt of the chunks. This is
+because we don't have an index associated with these chunks, hence these meta
+information are used while replaying the chunks.
 
 ```
 ┌─────────────────────┬───────────────────────┬───────────────────────┬───────────────────┬───────────────┬──────────────┬────────────────┐
 | series ref <8 byte> | mint <8 byte, uint64> | maxt <8 byte, uint64> | encoding <1 byte> | len <uvarint> | data <bytes> │ CRC32 <4 byte> │
 └─────────────────────┴───────────────────────┴───────────────────────┴───────────────────┴───────────────┴──────────────┴────────────────┘
-```
+```

tsdb/head_test.go

@@ -562,9 +562,7 @@ func TestHeadDeleteSimple(t *testing.T) {
 				for _, h := range []*Head{head, reloadedHead} {
 					q, err := NewBlockQuerier(h, h.MinTime(), h.MaxTime())
 					testutil.Ok(t, err)
-					actSeriesSet, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, lblDefault.Name, lblDefault.Value))
+					actSeriesSet := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, lblDefault.Name, lblDefault.Value))
-					testutil.Ok(t, err)
-					testutil.Equals(t, 0, len(ws))
 					testutil.Ok(t, q.Close())
 					expSeriesSet := newMockSeriesSet([]storage.Series{
 						newSeries(map[string]string{lblDefault.Name: lblDefault.Value}, func() []tsdbutil.Sample {
@@ -583,6 +581,8 @@ func TestHeadDeleteSimple(t *testing.T) {
 
 						if !eok {
 							testutil.Ok(t, h.Close())
+							testutil.Ok(t, actSeriesSet.Err())
+							testutil.Equals(t, 0, len(actSeriesSet.Warnings()))
 							continue Outer
 						}
 						expSeries := expSeriesSet.At()
@@ -623,13 +623,15 @@ func TestDeleteUntilCurMax(t *testing.T) {
 	// Test the series returns no samples. The series is cleared only after compaction.
 	q, err := NewBlockQuerier(hb, 0, 100000)
 	testutil.Ok(t, err)
-	res, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
+	res := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
-	testutil.Ok(t, err)
-	testutil.Equals(t, 0, len(ws))
 	testutil.Assert(t, res.Next(), "series is not present")
 	s := res.At()
 	it := s.Iterator()
 	testutil.Assert(t, !it.Next(), "expected no samples")
+	for res.Next() {
+	}
+	testutil.Ok(t, res.Err())
+	testutil.Equals(t, 0, len(res.Warnings()))
 
 	// Add again and test for presence.
 	app = hb.Appender()
@@ -638,15 +640,17 @@ func TestDeleteUntilCurMax(t *testing.T) {
 	testutil.Ok(t, app.Commit())
 	q, err = NewBlockQuerier(hb, 0, 100000)
 	testutil.Ok(t, err)
-	res, ws, err = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
+	res = q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "b"))
-	testutil.Ok(t, err)
-	testutil.Equals(t, 0, len(ws))
 	testutil.Assert(t, res.Next(), "series don't exist")
 	exps := res.At()
 	it = exps.Iterator()
 	resSamples, err := expandSeriesIterator(it)
 	testutil.Ok(t, err)
 	testutil.Equals(t, []tsdbutil.Sample{sample{11, 1}}, resSamples)
+	for res.Next() {
+	}
+	testutil.Ok(t, res.Err())
+	testutil.Equals(t, 0, len(res.Warnings()))
 }
 
 func TestDeletedSamplesAndSeriesStillInWALAfterCheckpoint(t *testing.T) {
@@ -807,9 +811,7 @@ func TestDelete_e2e(t *testing.T) {
 			q, err := NewBlockQuerier(hb, 0, 100000)
 			testutil.Ok(t, err)
 			defer q.Close()
-			ss, ws, err := q.Select(true, nil, del.ms...)
+			ss := q.Select(true, nil, del.ms...)
-			testutil.Ok(t, err)
-			testutil.Equals(t, 0, len(ws))
 			// Build the mockSeriesSet.
 			matchedSeries := make([]storage.Series, 0, len(matched))
 			for _, m := range matched {
@@ -850,6 +852,8 @@ func TestDelete_e2e(t *testing.T) {
 				testutil.Equals(t, errExp, errRes)
 				testutil.Equals(t, smplExp, smplRes)
 			}
+			testutil.Ok(t, ss.Err())
+			testutil.Equals(t, 0, len(ss.Warnings()))
 		}
 	}
 }
@@ -1118,11 +1122,12 @@ func TestUncommittedSamplesNotLostOnTruncate(t *testing.T) {
 	testutil.Ok(t, err)
 	defer q.Close()
 
-	ss, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1"))
+	ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1"))
-	testutil.Ok(t, err)
-	testutil.Equals(t, 0, len(ws))
-
 	testutil.Equals(t, true, ss.Next())
+	for ss.Next() {
+	}
+	testutil.Ok(t, ss.Err())
+	testutil.Equals(t, 0, len(ss.Warnings()))
 }
 
 func TestRemoveSeriesAfterRollbackAndTruncate(t *testing.T) {
@@ -1148,11 +1153,9 @@ func TestRemoveSeriesAfterRollbackAndTruncate(t *testing.T) {
 	testutil.Ok(t, err)
 	defer q.Close()
 
-	ss, ws, err := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1"))
+	ss := q.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "a", "1"))
-	testutil.Ok(t, err)
-	testutil.Equals(t, 0, len(ws))
-
 	testutil.Equals(t, false, ss.Next())
+	testutil.Equals(t, 0, len(ss.Warnings()))
 
 	// Truncate again, this time the series should be deleted
 	testutil.Ok(t, h.Truncate(2050))
@@ -1434,11 +1437,11 @@ func TestMemSeriesIsolation(t *testing.T) {
 		testutil.Ok(t, err)
 		defer querier.Close()
 
-		ss, _, err := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+		ss := querier.Select(false, nil, labels.MustNewMatcher(labels.MatchEqual, "foo", "bar"))
+		_, seriesSet, ws, err := expandSeriesSet(ss)
 		testutil.Ok(t, err)
+		testutil.Equals(t, 0, len(ws))
 
-		_, seriesSet, err := expandSeriesSet(ss)
-		testutil.Ok(t, err)
 		for _, series := range seriesSet {
 			return int(series[len(series)-1].v)
 		}
@@ -1790,8 +1793,9 @@ func testHeadSeriesChunkRace(t *testing.T) {
 		h.gc()
 		wg.Done()
 	}()
-	ss, _, err := q.Select(false, nil, matcher)
+	ss := q.Select(false, nil, matcher)
-	testutil.Ok(t, err)
+	for ss.Next() {
+	}
 	testutil.Ok(t, ss.Err())
 	wg.Wait()
 }

tsdb/querier.go

@@ -85,9 +85,9 @@ func (q *querier) lvals(qs []storage.Querier, n string) ([]string, storage.Warni
 	return mergeStrings(s1, s2), ws, nil
 }
 
-func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet {
 	if len(q.blocks) == 0 {
-		return storage.EmptySeriesSet(), nil, nil
+		return storage.EmptySeriesSet()
 	}
 	if len(q.blocks) == 1 {
 		// Sorting Head series is slow, and unneeded when only the
@@ -96,18 +96,12 @@ func (q *querier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*lab
 	}
 
 	ss := make([]storage.SeriesSet, len(q.blocks))
-	var ws storage.Warnings
 	for i, b := range q.blocks {
 		// We have to sort if blocks > 1 as MergedSeriesSet requires it.
-		s, w, err := b.Select(true, hints, ms...)
+		ss[i] = b.Select(true, hints, ms...)
-		ws = append(ws, w...)
-		if err != nil {
-			return nil, ws, err
-		}
-		ss[i] = s
 	}
 
-	return NewMergedSeriesSet(ss), ws, nil
+	return NewMergedSeriesSet(ss)
 }
 
 func (q *querier) Close() error {
@@ -125,31 +119,23 @@ type verticalQuerier struct {
 	querier
 }
 
-func (q *verticalQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q *verticalQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet {
 	return q.sel(sortSeries, hints, q.blocks, ms)
 }
 
-func (q *verticalQuerier) sel(sortSeries bool, hints *storage.SelectHints, qs []storage.Querier, ms []*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q *verticalQuerier) sel(sortSeries bool, hints *storage.SelectHints, qs []storage.Querier, ms []*labels.Matcher) storage.SeriesSet {
 	if len(qs) == 0 {
-		return storage.EmptySeriesSet(), nil, nil
+		return storage.EmptySeriesSet()
 	}
 	if len(qs) == 1 {
 		return qs[0].Select(sortSeries, hints, ms...)
 	}
 	l := len(qs) / 2
 
-	var ws storage.Warnings
+	return newMergedVerticalSeriesSet(
-	a, w, err := q.sel(sortSeries, hints, qs[:l], ms)
+		q.sel(sortSeries, hints, qs[:l], ms),
-	ws = append(ws, w...)
+		q.sel(sortSeries, hints, qs[l:], ms),
-	if err != nil {
+	)
-		return nil, ws, err
-	}
-	b, w, err := q.sel(sortSeries, hints, qs[l:], ms)
-	ws = append(ws, w...)
-	if err != nil {
-		return nil, ws, err
-	}
-	return newMergedVerticalSeriesSet(a, b), ws, nil
 }
 
 // NewBlockQuerier returns a querier against the reader.
@@ -189,7 +175,7 @@ type blockQuerier struct {
 	mint, maxt int64
 }
 
-func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) (storage.SeriesSet, storage.Warnings, error) {
+func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ...*labels.Matcher) storage.SeriesSet {
 	var base storage.DeprecatedChunkSeriesSet
 	var err error
 
@@ -199,7 +185,7 @@ func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ..
 		base, err = LookupChunkSeries(q.index, q.tombstones, ms...)
 	}
 	if err != nil {
-		return nil, nil, err
+		return storage.ErrSeriesSet(err)
 	}
 
 	mint := q.mint
@@ -218,7 +204,7 @@ func (q *blockQuerier) Select(sortSeries bool, hints *storage.SelectHints, ms ..
 
 		mint: mint,
 		maxt: maxt,
-	}, nil, nil
+	}
 }
 
 func (q *blockQuerier) LabelValues(name string) ([]string, storage.Warnings, error) {
@@ -501,6 +487,14 @@ func (s *mergedSeriesSet) Err() error {
 	return s.err
 }
 
+func (s *mergedSeriesSet) Warnings() storage.Warnings {
+	var ws storage.Warnings
+	for _, ss := range s.all {
+		ws = append(ws, ss.Warnings()...)
+	}
+	return ws
+}
+
 // nextAll is to call Next() for all SeriesSet.
 // Because the order of the SeriesSet slice will affect the results,
 // we need to use an buffer slice to hold the order.
@@ -509,7 +503,10 @@ func (s *mergedSeriesSet) nextAll() {
 	for _, ss := range s.all {
 		if ss.Next() {
 			s.buf = append(s.buf, ss)
-		} else if ss.Err() != nil {
+			continue
+		}
+
+		if ss.Err() != nil {
 			s.done = true
 			s.err = ss.Err()
 			break
@@ -623,6 +620,13 @@ func (s *mergedVerticalSeriesSet) Err() error {
 	return s.b.Err()
 }
 
+func (s *mergedVerticalSeriesSet) Warnings() storage.Warnings {
+	var ws storage.Warnings
+	ws = append(ws, s.a.Warnings()...)
+	ws = append(ws, s.b.Warnings()...)
+	return ws
+}
+
 func (s *mergedVerticalSeriesSet) compare() int {
 	if s.adone {
 		return 1
@@ -848,8 +852,9 @@ func (s *blockSeriesSet) Next() bool {
 	return false
 }
 
-func (s *blockSeriesSet) At() storage.Series { return s.cur }
+func (s *blockSeriesSet) At() storage.Series         { return s.cur }
-func (s *blockSeriesSet) Err() error         { return s.err }
+func (s *blockSeriesSet) Err() error                 { return s.err }
+func (s *blockSeriesSet) Warnings() storage.Warnings { return nil }
 
 // chunkSeries is a series that is backed by a sequence of chunks holding
 // time series data.

tsdb/querier_bench_test.go

@@ -155,8 +155,7 @@ func BenchmarkQuerierSelect(b *testing.B) {
 
 				b.ResetTimer()
 				for i := 0; i < b.N; i++ {
-					ss, _, err := q.Select(sorted, nil, matcher)
+					ss := q.Select(sorted, nil, matcher)
-					testutil.Ok(b, err)
 					for ss.Next() {
 					}
 					testutil.Ok(b, ss.Err())

tsdb/querier_test.go

@@ -39,12 +39,14 @@ import (
 type mockSeriesSet struct {
 	next   func() bool
 	series func() storage.Series
+	ws     func() storage.Warnings
 	err    func() error
 }
 
-func (m *mockSeriesSet) Next() bool         { return m.next() }
+func (m *mockSeriesSet) Next() bool                 { return m.next() }
-func (m *mockSeriesSet) At() storage.Series { return m.series() }
+func (m *mockSeriesSet) At() storage.Series         { return m.series() }
-func (m *mockSeriesSet) Err() error         { return m.err() }
+func (m *mockSeriesSet) Err() error                 { return m.err() }
+func (m *mockSeriesSet) Warnings() storage.Warnings { return m.ws() }
 
 func newMockSeriesSet(list []storage.Series) *mockSeriesSet {
 	i := -1
@@ -57,11 +59,11 @@ func newMockSeriesSet(list []storage.Series) *mockSeriesSet {
 			return list[i]
 		},
 		err: func() error { return nil },
+		ws:  func() storage.Warnings { return nil },
 	}
 }
 
 func TestMergedSeriesSet(t *testing.T) {
-
 	cases := []struct {
 		// The input sets in order (samples in series in b are strictly
 		// after those in a).
@@ -373,15 +375,14 @@ Outer:
 			maxt: c.maxt,
 		}
 
-		res, ws, err := querier.Select(false, nil, c.ms...)
+		res := querier.Select(false, nil, c.ms...)
-		testutil.Ok(t, err)
-		testutil.Equals(t, 0, len(ws))
 
 		for {
 			eok, rok := c.exp.Next(), res.Next()
 			testutil.Equals(t, eok, rok)
 
 			if !eok {
+				testutil.Equals(t, 0, len(res.Warnings()))
 				continue Outer
 			}
 			sexp := c.exp.At()
@@ -536,15 +537,14 @@ Outer:
 			maxt: c.maxt,
 		}
 
-		res, ws, err := querier.Select(false, nil, c.ms...)
+		res := querier.Select(false, nil, c.ms...)
-		testutil.Ok(t, err)
-		testutil.Equals(t, 0, len(ws))
 
 		for {
 			eok, rok := c.exp.Next(), res.Next()
 			testutil.Equals(t, eok, rok)
 
 			if !eok {
+				testutil.Equals(t, 0, len(res.Warnings()))
 				continue Outer
 			}
 			sexp := c.exp.At()
@@ -1654,7 +1654,7 @@ func BenchmarkQuerySeek(b *testing.B) {
 				b.ResetTimer()
 				b.ReportAllocs()
 
-				ss, ws, err := sq.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*"))
+				ss := sq.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "__name__", ".*"))
 				for ss.Next() {
 					it := ss.At().Iterator()
 					for t := mint; t <= maxt; t++ {
@@ -1664,7 +1664,7 @@ func BenchmarkQuerySeek(b *testing.B) {
 				}
 				testutil.Ok(b, ss.Err())
 				testutil.Ok(b, err)
-				testutil.Equals(b, 0, len(ws))
+				testutil.Equals(b, 0, len(ss.Warnings()))
 			})
 		}
 	}
@@ -1792,9 +1792,11 @@ func BenchmarkSetMatcher(b *testing.B) {
 			b.ResetTimer()
 			b.ReportAllocs()
 			for n := 0; n < b.N; n++ {
-				_, ws, err := que.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "test", c.pattern))
+				ss := que.Select(false, nil, labels.MustNewMatcher(labels.MatchRegexp, "test", c.pattern))
-				testutil.Ok(b, err)
+				for ss.Next() {
-				testutil.Equals(b, 0, len(ws))
+				}
+				testutil.Ok(b, ss.Err())
+				testutil.Equals(b, 0, len(ss.Warnings()))
 			}
 		})
 	}
@@ -2252,9 +2254,7 @@ func benchQuery(b *testing.B, expExpansions int, q storage.Querier, selectors la
 	b.ResetTimer()
 	b.ReportAllocs()
 	for i := 0; i < b.N; i++ {
-		ss, ws, err := q.Select(false, nil, selectors...)
+		ss := q.Select(false, nil, selectors...)
-		testutil.Ok(b, err)
-		testutil.Equals(b, 0, len(ws))
 		var actualExpansions int
 		for ss.Next() {
 			s := ss.At()
@@ -2264,6 +2264,8 @@ func benchQuery(b *testing.B, expExpansions int, q storage.Querier, selectors la
 			}
 			actualExpansions++
 		}
+		testutil.Ok(b, ss.Err())
+		testutil.Equals(b, 0, len(ss.Warnings()))
 		testutil.Equals(b, expExpansions, actualExpansions)
 		testutil.Ok(b, ss.Err())
 	}

vendor/github.com/digitalocean/godo/.gitignore

@@ -0,0 +1 @@
+vendor/

vendor/github.com/digitalocean/godo/.whitesource

@@ -0,0 +1,8 @@
+{
+  "checkRunSettings": {
+    "vulnerableCheckRunConclusionLevel": "failure"
+  },
+  "issueSettings": {
+    "minSeverityLevel": "LOW"
+  }
+}

vendor/github.com/digitalocean/godo/1-click.go

@@ -0,0 +1,52 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+const oneClickBasePath = "v2/1-clicks"
+
+// OneClickService is an interface for interacting with 1-clicks with the
+// DigitalOcean API.
+// See: https://developers.digitalocean.com/documentation/v2/#1-click-applications
+type OneClickService interface {
+	List(context.Context, string) ([]*OneClick, *Response, error)
+}
+
+var _ OneClickService = &OneClickServiceOp{}
+
+// OneClickServiceOp interfaces with 1-click endpoints in the DigitalOcean API.
+type OneClickServiceOp struct {
+	client *Client
+}
+
+// OneClick is the structure of a 1-click
+type OneClick struct {
+	Slug string `json:"slug"`
+	Type string `json:"type"`
+}
+
+// OneClicksRoot is the root of the json payload that contains a list of 1-clicks
+type OneClicksRoot struct {
+	List []*OneClick `json:"1_clicks"`
+}
+
+// List returns a list of the available 1-click applications.
+func (ocs *OneClickServiceOp) List(ctx context.Context, oneClickType string) ([]*OneClick, *Response, error) {
+	path := fmt.Sprintf(`%s?type=%s`, oneClickBasePath, oneClickType)
+
+	req, err := ocs.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(OneClicksRoot)
+	resp, err := ocs.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.List, resp, nil
+}

vendor/github.com/digitalocean/godo/CHANGELOG.md

@@ -0,0 +1,255 @@
+# Change Log
+
+## [v1.37.0] - 2020-06-01
+
+- #336 registry: URL encode repository names when building URLs. @adamwg
+- #335 Add 1-click service and request. @scottcrawford03
+
+## [v1.36.0] - 2020-05-12
+
+- #331 Expose expiry_seconds for Registry.DockerCredentials. @andrewsomething
+
+## [v1.35.1] - 2020-04-21
+
+- #328 Update vulnerable x/crypto dependency - @bentranter
+
+## [v1.35.0] - 2020-04-20
+
+- #326 Add TagCount field to registry/Repository - @nicktate
+- #325 Add DOCR EA routes - @nicktate
+- #324 Upgrade godo to Go 1.14 - @bentranter
+
+## [v1.34.0] - 2020-03-30
+
+- #320 Add VPC v3 attributes - @viola
+
+## [v1.33.1] - 2020-03-23
+
+- #318 upgrade github.com/stretchr/objx past 0.1.1 - @hilary
+
+## [v1.33.0] - 2020-03-20
+
+- #310 Add BillingHistory service and List endpoint - @rbutler
+- #316 load balancers: add new enable_backend_keepalive field - @anitgandhi
+
+## [v1.32.0] - 2020-03-04
+
+- #311 Add reset database user auth method - @zbarahal-do
+
+## [v1.31.0] - 2020-02-28
+
+- #305 invoices: GetPDF and GetCSV methods - @rbutler
+- #304 Add NewFromToken convenience method to init client - @bentranter
+- #301 invoices: Get, Summary, and List methods - @rbutler
+- #299 Fix param expiry_seconds for kubernetes.GetCredentials request - @velp
+
+## [v1.30.0] - 2020-02-03
+
+- #295 registry: support the created_at field - @adamwg
+- #293 doks: node pool labels - @snormore
+
+## [v1.29.0] - 2019-12-13
+
+- #288 Add Balance Get method - @rbutler
+- #286,#289 Deserialize meta field - @timoreimann
+
+## [v1.28.0] - 2019-12-04
+
+- #282 Add valid Redis eviction policy constants - @bentranter
+- #281 Remove databases info from top-level godoc string - @bentranter
+- #280 Fix VolumeSnapshotResourceType value volumesnapshot -> volume_snapshot - @aqche
+
+## [v1.27.0] - 2019-11-18
+
+- #278 add mysql user auth settings for database users - @gregmankes
+
+## [v1.26.0] - 2019-11-13
+
+- #272 dbaas: get and set mysql sql mode - @mikejholly
+
+## [v1.25.0] - 2019-11-13
+
+- #275 registry/docker-credentials: add support for the read/write parameter - @kamaln7
+- #273 implement the registry/docker-credentials endpoint - @kamaln7
+- #271 Add registry resource - @snormore
+
+## [v1.24.1] - 2019-11-04
+
+- #264 Update isLast to check p.Next - @aqche
+
+## [v1.24.0] - 2019-10-30
+
+- #267 Return []DatabaseFirewallRule in addition to raw response. - @andrewsomething
+
+## [v1.23.1] - 2019-10-30
+
+- #265 add support for getting/setting firewall rules - @gregmankes
+- #262 remove ResolveReference call - @mdanzinger
+- #261 Update CONTRIBUTING.md - @mdanzinger
+
+## [v1.22.0] - 2019-09-24
+
+- #259 Add Kubernetes GetCredentials method - @snormore
+
+## [v1.21.1] - 2019-09-19
+
+- #257 Upgrade to Go 1.13 - @bentranter
+
+## [v1.21.0] - 2019-09-16
+
+- #255 Add DropletID to Kubernetes Node instance - @snormore
+- #254 Add tags to Database, DatabaseReplica - @Zyqsempai
+
+## [v1.20.0] - 2019-09-06
+
+- #252 Add Kubernetes autoscale config fields - @snormore
+- #251 Support unset fields on Kubernetes cluster and node pool updates - @snormore
+- #250 Add Kubernetes GetUser method - @snormore
+
+## [v1.19.0] - 2019-07-19
+
+- #244 dbaas: add private-network-uuid field to create request
+
+## [v1.18.0] - 2019-07-17
+
+- #241 Databases: support for custom VPC UUID on migrate @mikejholly
+- #240 Add the ability to get URN for a Database @stack72
+- #236 Fix omitempty typos in JSON struct tags @amccarthy1
+
+## [v1.17.0] - 2019-06-21
+
+- #238 Add support for Redis eviction policy in Databases @mikejholly
+
+## [v1.16.0] - 2019-06-04
+
+- #233 Add Kubernetes DeleteNode method, deprecate RecycleNodePoolNodes @bouk
+
+## [v1.15.0] - 2019-05-13
+
+- #231 Add private connection fields to Databases - @mikejholly
+- #223 Introduce Go modules - @andreiavrammsd
+
+## [v1.14.0] - 2019-05-13
+
+- #229 Add support for upgrading Kubernetes clusters - @adamwg
+
+## [v1.13.0] - 2019-04-19
+
+- #213 Add tagging support for volume snapshots - @jcodybaker
+
+## [v1.12.0] - 2019-04-18
+
+- #224 Add maintenance window support for Kubernetes- @fatih
+
+## [v1.11.1] - 2019-04-04
+
+- #222 Fix Create Database Pools json fields - @sunny-b
+
+## [v1.11.0] - 2019-04-03
+
+- #220 roll out vpc functionality - @jheimann
+
+## [v1.10.1] - 2019-03-27
+
+- #219 Fix Database Pools json field - @sunny-b
+
+## [v1.10.0] - 2019-03-20
+
+- #215 Add support for Databases - @mikejholly
+
+## [v1.9.0] - 2019-03-18
+
+- #214 add support for enable_proxy_protocol. - @mregmi
+
+## [v1.8.0] - 2019-03-13
+
+- #210 Expose tags on storage volume create/list/get. - @jcodybaker
+
+## [v1.7.5] - 2019-03-04
+
+- #207 Add support for custom subdomains for Spaces CDN [beta] - @xornivore
+
+## [v1.7.4] - 2019-02-08
+
+- #202 Allow tagging volumes - @mchitten
+
+## [v1.7.3] - 2018-12-18
+
+- #196 Expose tag support for creating Load Balancers.
+
+## [v1.7.2] - 2018-12-04
+
+- #192 Exposes more options for Kubernetes clusters.
+
+## [v1.7.1] - 2018-11-27
+
+- #190 Expose constants for the state of Kubernetes clusters.
+
+## [v1.7.0] - 2018-11-13
+
+- #188 Kubernetes support [beta] - @aybabtme
+
+## [v1.6.0] - 2018-10-16
+
+- #185 Projects support [beta] - @mchitten
+
+## [v1.5.0] - 2018-10-01
+
+- #181 Adding tagging images support - @hugocorbucci
+
+## [v1.4.2] - 2018-08-30
+
+- #178 Allowing creating domain records with weight of 0 - @TFaga
+- #177 Adding `VolumeLimit` to account - @lxfontes
+
+## [v1.4.1] - 2018-08-23
+
+- #176 Fix cdn flush cache API endpoint - @sunny-b
+
+## [v1.4.0] - 2018-08-22
+
+- #175 Add support for Spaces CDN - @sunny-b
+
+## [v1.3.0] - 2018-05-24
+
+- #170 Add support for volume formatting - @adamwg
+
+## [v1.2.0] - 2018-05-08
+
+- #166 Remove support for Go 1.6 - @iheanyi
+- #165 Add support for Let's Encrypt Certificates - @viola
+
+## [v1.1.3] - 2018-03-07
+
+- #156 Handle non-json errors from the API - @aknuds1
+- #158 Update droplet example to use latest instance type - @dan-v
+
+## [v1.1.2] - 2018-03-06
+
+- #157 storage: list volumes should handle only name or only region params - @andrewsykim
+- #154 docs: replace first example with fully-runnable example - @xmudrii
+- #152 Handle flags & tag properties of domain record - @jaymecd
+
+## [v1.1.1] - 2017-09-29
+
+- #151 Following user agent field recommendations - @joonas
+- #148 AsRequest method to create load balancers requests - @lukegb
+
+## [v1.1.0] - 2017-06-06
+
+### Added
+- #145 Add FirewallsService for managing Firewalls with the DigitalOcean API. - @viola
+- #139 Add TTL field to the Domains. - @xmudrii
+
+### Fixed
+- #143 Fix oauth2.NoContext depreciation. - @jbowens
+- #141 Fix DropletActions on tagged resources. - @xmudrii
+
+## [v1.0.0] - 2017-03-10
+
+### Added
+- #130 Add Convert to ImageActionsService. - @xmudrii
+- #126 Add CertificatesService for managing certificates with the DigitalOcean API. - @viola
+- #125 Add LoadBalancersService for managing load balancers with the DigitalOcean API. - @viola
+- #122 Add GetVolumeByName to StorageService. - @protochron
+- #113 Add context.Context to all calls. - @aybabtme

vendor/github.com/digitalocean/godo/CONTRIBUTING.md

@@ -0,0 +1,54 @@
+# Contributing
+
+We love contributions! You are welcome to open a pull request, but it's a good idea to
+open an issue and discuss your idea with us first.
+
+Once you are ready to open a PR, please keep the following guidelines in mind:
+
+1. Code should be `go fmt` compliant.
+1. Types, structs and funcs should be documented.
+1. Tests pass.
+
+## Getting set up
+
+`godo` uses go modules. Just fork this repo, clone your fork and off you go!
+
+## Running tests
+
+When working on code in this repository, tests can be run via:
+
+```sh
+go test -mod=vendor .
+```
+
+## Versioning
+
+Godo follows [semver](https://www.semver.org) versioning semantics.
+New functionality should be accompanied by increment to the minor
+version number. Any code merged to master is subject to release.
+
+## Releasing
+
+Releasing a new version of godo is currently a manual process. 
+
+Submit a separate pull request for the version change from the pull
+request with your changes.
+
+1. Update the `CHANGELOG.md` with your changes. If a version header
+   for the next (unreleased) version does not exist, create one.
+   Include one bullet point for each piece of new functionality in the
+   release, including the pull request ID, description, and author(s).
+
+```
+## [v1.8.0] - 2019-03-13
+
+- #210 Expose tags on storage volume create/list/get. - @jcodybaker
+- #123 Update test dependencies - @digitalocean
+```
+
+2. Update the `libraryVersion` number in `godo.go`.
+3. Make a pull request with these changes.  This PR should be separate from the PR containing the godo changes.
+4. Once the pull request has been merged, [draft a new release](https://github.com/digitalocean/godo/releases/new).  
+5. Update the `Tag version` and `Release title` field with the new godo version.  Be sure the version has a `v` prefixed in both places. Ex `v1.8.0`.  
+6. Copy the changelog bullet points to the description field.
+7. Publish the release.

vendor/github.com/digitalocean/godo/LICENSE.txt

@@ -0,0 +1,55 @@
+Copyright (c) 2014-2016 The godo AUTHORS. All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+======================
+Portions of the client are based on code at:
+https://github.com/google/go-github/
+
+Copyright (c) 2013 The go-github AUTHORS. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

vendor/github.com/digitalocean/godo/README.md

@@ -0,0 +1,139 @@
+# Godo
+
+[![Build Status](https://travis-ci.org/digitalocean/godo.svg)](https://travis-ci.org/digitalocean/godo)
+[![GoDoc](https://godoc.org/github.com/digitalocean/godo?status.svg)](https://godoc.org/github.com/digitalocean/godo)
+
+Godo is a Go client library for accessing the DigitalOcean V2 API.
+
+You can view the client API docs here: [http://godoc.org/github.com/digitalocean/godo](http://godoc.org/github.com/digitalocean/godo)
+
+You can view DigitalOcean API docs here: [https://developers.digitalocean.com/documentation/v2/](https://developers.digitalocean.com/documentation/v2/)
+
+## Install
+```sh
+go get github.com/digitalocean/godo@vX.Y.Z
+```
+
+where X.Y.Z is the [version](https://github.com/digitalocean/godo/releases) you need.
+
+or
+```sh
+go get github.com/digitalocean/godo
+```
+for non Go modules usage or latest version.
+
+## Usage
+
+```go
+import "github.com/digitalocean/godo"
+```
+
+Create a new DigitalOcean client, then use the exposed services to
+access different parts of the DigitalOcean API.
+
+### Authentication
+
+Currently, Personal Access Token (PAT) is the only method of
+authenticating with the API. You can manage your tokens
+at the DigitalOcean Control Panel [Applications Page](https://cloud.digitalocean.com/settings/applications).
+
+You can then use your token to create a new client:
+
+```go
+package main
+
+import (
+    "github.com/digitalocean/godo"
+)
+
+func main() {
+    client := godo.NewFromToken("my-digitalocean-api-token")
+}
+```
+
+If you need to provide a `context.Context` to your new client, you should use [`godo.NewClient`](https://godoc.org/github.com/digitalocean/godo#NewClient) to manually construct a client instead.
+
+## Examples
+
+
+To create a new Droplet:
+
+```go
+dropletName := "super-cool-droplet"
+
+createRequest := &godo.DropletCreateRequest{
+    Name:   dropletName,
+    Region: "nyc3",
+    Size:   "s-1vcpu-1gb",
+    Image: godo.DropletCreateImage{
+        Slug: "ubuntu-14-04-x64",
+    },
+}
+
+ctx := context.TODO()
+
+newDroplet, _, err := client.Droplets.Create(ctx, createRequest)
+
+if err != nil {
+    fmt.Printf("Something bad happened: %s\n\n", err)
+    return err
+}
+```
+
+### Pagination
+
+If a list of items is paginated by the API, you must request pages individually. For example, to fetch all Droplets:
+
+```go
+func DropletList(ctx context.Context, client *godo.Client) ([]godo.Droplet, error) {
+    // create a list to hold our droplets
+    list := []godo.Droplet{}
+
+    // create options. initially, these will be blank
+    opt := &godo.ListOptions{}
+    for {
+        droplets, resp, err := client.Droplets.List(ctx, opt)
+        if err != nil {
+            return nil, err
+        }
+
+        // append the current page's droplets to our list
+        for _, d := range droplets {
+            list = append(list, d)
+        }
+
+        // if we are at the last page, break out the for loop
+        if resp.Links == nil || resp.Links.IsLastPage() {
+            break
+        }
+
+        page, err := resp.Links.CurrentPage()
+        if err != nil {
+            return nil, err
+        }
+
+        // set the page we want for the next request
+        opt.Page = page + 1
+    }
+
+    return list, nil
+}
+```
+
+## Versioning
+
+Each version of the client is tagged and the version is updated accordingly.
+
+To see the list of past versions, run `git tag`.
+
+
+## Documentation
+
+For a comprehensive list of examples, check out the [API documentation](https://developers.digitalocean.com/documentation/v2/).
+
+For details on all the functionality in this library, see the [GoDoc](http://godoc.org/github.com/digitalocean/godo) documentation.
+
+
+## Contributing
+
+We love pull requests! Please see the [contribution guidelines](CONTRIBUTING.md).

vendor/github.com/digitalocean/godo/account.go

@@ -0,0 +1,60 @@
+package godo
+
+import (
+	"context"
+	"net/http"
+)
+
+// AccountService is an interface for interfacing with the Account
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2/#account
+type AccountService interface {
+	Get(context.Context) (*Account, *Response, error)
+}
+
+// AccountServiceOp handles communication with the Account related methods of
+// the DigitalOcean API.
+type AccountServiceOp struct {
+	client *Client
+}
+
+var _ AccountService = &AccountServiceOp{}
+
+// Account represents a DigitalOcean Account
+type Account struct {
+	DropletLimit    int    `json:"droplet_limit,omitempty"`
+	FloatingIPLimit int    `json:"floating_ip_limit,omitempty"`
+	VolumeLimit     int    `json:"volume_limit,omitempty"`
+	Email           string `json:"email,omitempty"`
+	UUID            string `json:"uuid,omitempty"`
+	EmailVerified   bool   `json:"email_verified,omitempty"`
+	Status          string `json:"status,omitempty"`
+	StatusMessage   string `json:"status_message,omitempty"`
+}
+
+type accountRoot struct {
+	Account *Account `json:"account"`
+}
+
+func (r Account) String() string {
+	return Stringify(r)
+}
+
+// Get DigitalOcean account info
+func (s *AccountServiceOp) Get(ctx context.Context) (*Account, *Response, error) {
+
+	path := "v2/account"
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(accountRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Account, resp, err
+}

vendor/github.com/digitalocean/godo/action.go

@@ -0,0 +1,108 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+const (
+	actionsBasePath = "v2/actions"
+
+	// ActionInProgress is an in progress action status
+	ActionInProgress = "in-progress"
+
+	//ActionCompleted is a completed action status
+	ActionCompleted = "completed"
+)
+
+// ActionsService handles communction with action related methods of the
+// DigitalOcean API: https://developers.digitalocean.com/documentation/v2#actions
+type ActionsService interface {
+	List(context.Context, *ListOptions) ([]Action, *Response, error)
+	Get(context.Context, int) (*Action, *Response, error)
+}
+
+// ActionsServiceOp handles communition with the image action related methods of the
+// DigitalOcean API.
+type ActionsServiceOp struct {
+	client *Client
+}
+
+var _ ActionsService = &ActionsServiceOp{}
+
+type actionsRoot struct {
+	Actions []Action `json:"actions"`
+	Links   *Links   `json:"links"`
+	Meta    *Meta    `json:"meta"`
+}
+
+type actionRoot struct {
+	Event *Action `json:"action"`
+}
+
+// Action represents a DigitalOcean Action
+type Action struct {
+	ID           int        `json:"id"`
+	Status       string     `json:"status"`
+	Type         string     `json:"type"`
+	StartedAt    *Timestamp `json:"started_at"`
+	CompletedAt  *Timestamp `json:"completed_at"`
+	ResourceID   int        `json:"resource_id"`
+	ResourceType string     `json:"resource_type"`
+	Region       *Region    `json:"region,omitempty"`
+	RegionSlug   string     `json:"region_slug,omitempty"`
+}
+
+// List all actions
+func (s *ActionsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Action, *Response, error) {
+	path := actionsBasePath
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Actions, resp, err
+}
+
+// Get an action by ID.
+func (s *ActionsServiceOp) Get(ctx context.Context, id int) (*Action, *Response, error) {
+	if id < 1 {
+		return nil, nil, NewArgError("id", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d", actionsBasePath, id)
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Event, resp, err
+}
+
+func (a Action) String() string {
+	return Stringify(a)
+}

vendor/github.com/digitalocean/godo/balance.go

@@ -0,0 +1,52 @@
+package godo
+
+import (
+	"context"
+	"net/http"
+	"time"
+)
+
+// BalanceService is an interface for interfacing with the Balance
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2/#balance
+type BalanceService interface {
+	Get(context.Context) (*Balance, *Response, error)
+}
+
+// BalanceServiceOp handles communication with the Balance related methods of
+// the DigitalOcean API.
+type BalanceServiceOp struct {
+	client *Client
+}
+
+var _ BalanceService = &BalanceServiceOp{}
+
+// Balance represents a DigitalOcean Balance
+type Balance struct {
+	MonthToDateBalance string    `json:"month_to_date_balance"`
+	AccountBalance     string    `json:"account_balance"`
+	MonthToDateUsage   string    `json:"month_to_date_usage"`
+	GeneratedAt        time.Time `json:"generated_at"`
+}
+
+func (r Balance) String() string {
+	return Stringify(r)
+}
+
+// Get DigitalOcean balance info
+func (s *BalanceServiceOp) Get(ctx context.Context) (*Balance, *Response, error) {
+	path := "v2/customers/my/balance"
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(Balance)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root, resp, err
+}

vendor/github.com/digitalocean/godo/billing_history.go

@@ -0,0 +1,72 @@
+package godo
+
+import (
+	"context"
+	"net/http"
+	"time"
+)
+
+const billingHistoryBasePath = "v2/customers/my/billing_history"
+
+// BillingHistoryService is an interface for interfacing with the BillingHistory
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2/#billing_history
+type BillingHistoryService interface {
+	List(context.Context, *ListOptions) (*BillingHistory, *Response, error)
+}
+
+// BillingHistoryServiceOp handles communication with the BillingHistory related methods of
+// the DigitalOcean API.
+type BillingHistoryServiceOp struct {
+	client *Client
+}
+
+var _ BillingHistoryService = &BillingHistoryServiceOp{}
+
+// BillingHistory represents a DigitalOcean Billing History
+type BillingHistory struct {
+	BillingHistory []BillingHistoryEntry `json:"billing_history"`
+	Links          *Links                `json:"links"`
+	Meta           *Meta                 `json:"meta"`
+}
+
+// BillingHistoryEntry represents an entry in a customer's Billing History
+type BillingHistoryEntry struct {
+	Description string    `json:"description"`
+	Amount      string    `json:"amount"`
+	InvoiceID   *string   `json:"invoice_id"`
+	InvoiceUUID *string   `json:"invoice_uuid"`
+	Date        time.Time `json:"date"`
+	Type        string    `json:"type"`
+}
+
+func (b BillingHistory) String() string {
+	return Stringify(b)
+}
+
+// List the Billing History for a customer
+func (s *BillingHistoryServiceOp) List(ctx context.Context, opt *ListOptions) (*BillingHistory, *Response, error) {
+	path, err := addOptions(billingHistoryBasePath, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(BillingHistory)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root, resp, err
+}

vendor/github.com/digitalocean/godo/cdn.go

@@ -0,0 +1,218 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+const cdnBasePath = "v2/cdn/endpoints"
+
+// CDNService is an interface for managing Spaces CDN with the DigitalOcean API.
+type CDNService interface {
+	List(context.Context, *ListOptions) ([]CDN, *Response, error)
+	Get(context.Context, string) (*CDN, *Response, error)
+	Create(context.Context, *CDNCreateRequest) (*CDN, *Response, error)
+	UpdateTTL(context.Context, string, *CDNUpdateTTLRequest) (*CDN, *Response, error)
+	UpdateCustomDomain(context.Context, string, *CDNUpdateCustomDomainRequest) (*CDN, *Response, error)
+	FlushCache(context.Context, string, *CDNFlushCacheRequest) (*Response, error)
+	Delete(context.Context, string) (*Response, error)
+}
+
+// CDNServiceOp handles communication with the CDN related methods of the
+// DigitalOcean API.
+type CDNServiceOp struct {
+	client *Client
+}
+
+var _ CDNService = &CDNServiceOp{}
+
+// CDN represents a DigitalOcean CDN
+type CDN struct {
+	ID            string    `json:"id"`
+	Origin        string    `json:"origin"`
+	Endpoint      string    `json:"endpoint"`
+	CreatedAt     time.Time `json:"created_at"`
+	TTL           uint32    `json:"ttl"`
+	CertificateID string    `json:"certificate_id,omitempty"`
+	CustomDomain  string    `json:"custom_domain,omitempty"`
+}
+
+// CDNRoot represents a response from the DigitalOcean API
+type cdnRoot struct {
+	Endpoint *CDN `json:"endpoint"`
+}
+
+type cdnsRoot struct {
+	Endpoints []CDN  `json:"endpoints"`
+	Links     *Links `json:"links"`
+	Meta      *Meta  `json:"meta"`
+}
+
+// CDNCreateRequest represents a request to create a CDN.
+type CDNCreateRequest struct {
+	Origin        string `json:"origin"`
+	TTL           uint32 `json:"ttl"`
+	CustomDomain  string `json:"custom_domain,omitempty"`
+	CertificateID string `json:"certificate_id,omitempty"`
+}
+
+// CDNUpdateTTLRequest represents a request to update the ttl of a CDN.
+type CDNUpdateTTLRequest struct {
+	TTL uint32 `json:"ttl"`
+}
+
+// CDNUpdateCustomDomainRequest represents a request to update the custom domain of a CDN.
+type CDNUpdateCustomDomainRequest struct {
+	CustomDomain  string `json:"custom_domain"`
+	CertificateID string `json:"certificate_id"`
+}
+
+// CDNFlushCacheRequest represents a request to flush cache of a CDN.
+type CDNFlushCacheRequest struct {
+	Files []string `json:"files"`
+}
+
+// List all CDN endpoints
+func (c CDNServiceOp) List(ctx context.Context, opt *ListOptions) ([]CDN, *Response, error) {
+	path, err := addOptions(cdnBasePath, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := c.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(cdnsRoot)
+	resp, err := c.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Endpoints, resp, err
+}
+
+// Get individual CDN. It requires a non-empty cdn id.
+func (c CDNServiceOp) Get(ctx context.Context, id string) (*CDN, *Response, error) {
+	if len(id) == 0 {
+		return nil, nil, NewArgError("id", "cannot be an empty string")
+	}
+
+	path := fmt.Sprintf("%s/%s", cdnBasePath, id)
+
+	req, err := c.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(cdnRoot)
+	resp, err := c.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Endpoint, resp, err
+}
+
+// Create a new CDN
+func (c CDNServiceOp) Create(ctx context.Context, createRequest *CDNCreateRequest) (*CDN, *Response, error) {
+	if createRequest == nil {
+		return nil, nil, NewArgError("createRequest", "cannot be nil")
+	}
+
+	req, err := c.client.NewRequest(ctx, http.MethodPost, cdnBasePath, createRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(cdnRoot)
+	resp, err := c.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Endpoint, resp, err
+}
+
+// UpdateTTL updates the ttl of an individual CDN
+func (c CDNServiceOp) UpdateTTL(ctx context.Context, id string, updateRequest *CDNUpdateTTLRequest) (*CDN, *Response, error) {
+	return c.update(ctx, id, updateRequest)
+}
+
+// UpdateCustomDomain sets or removes the custom domain of an individual CDN
+func (c CDNServiceOp) UpdateCustomDomain(ctx context.Context, id string, updateRequest *CDNUpdateCustomDomainRequest) (*CDN, *Response, error) {
+	return c.update(ctx, id, updateRequest)
+}
+
+func (c CDNServiceOp) update(ctx context.Context, id string, updateRequest interface{}) (*CDN, *Response, error) {
+	if updateRequest == nil {
+		return nil, nil, NewArgError("updateRequest", "cannot be nil")
+	}
+
+	if len(id) == 0 {
+		return nil, nil, NewArgError("id", "cannot be an empty string")
+	}
+	path := fmt.Sprintf("%s/%s", cdnBasePath, id)
+
+	req, err := c.client.NewRequest(ctx, http.MethodPut, path, updateRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(cdnRoot)
+	resp, err := c.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Endpoint, resp, err
+}
+
+// FlushCache flushes the cache of an individual CDN. Requires a non-empty slice of file paths and/or wildcards
+func (c CDNServiceOp) FlushCache(ctx context.Context, id string, flushCacheRequest *CDNFlushCacheRequest) (*Response, error) {
+	if flushCacheRequest == nil {
+		return nil, NewArgError("flushCacheRequest", "cannot be nil")
+	}
+
+	if len(id) == 0 {
+		return nil, NewArgError("id", "cannot be an empty string")
+	}
+
+	path := fmt.Sprintf("%s/%s/cache", cdnBasePath, id)
+
+	req, err := c.client.NewRequest(ctx, http.MethodDelete, path, flushCacheRequest)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.client.Do(ctx, req, nil)
+
+	return resp, err
+}
+
+// Delete an individual CDN
+func (c CDNServiceOp) Delete(ctx context.Context, id string) (*Response, error) {
+	if len(id) == 0 {
+		return nil, NewArgError("id", "cannot be an empty string")
+	}
+
+	path := fmt.Sprintf("%s/%s", cdnBasePath, id)
+
+	req, err := c.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.client.Do(ctx, req, nil)
+
+	return resp, err
+}

vendor/github.com/digitalocean/godo/certificates.go

@@ -0,0 +1,130 @@
+package godo
+
+import (
+	"context"
+	"net/http"
+	"path"
+)
+
+const certificatesBasePath = "/v2/certificates"
+
+// CertificatesService is an interface for managing certificates with the DigitalOcean API.
+// See: https://developers.digitalocean.com/documentation/v2/#certificates
+type CertificatesService interface {
+	Get(context.Context, string) (*Certificate, *Response, error)
+	List(context.Context, *ListOptions) ([]Certificate, *Response, error)
+	Create(context.Context, *CertificateRequest) (*Certificate, *Response, error)
+	Delete(context.Context, string) (*Response, error)
+}
+
+// Certificate represents a DigitalOcean certificate configuration.
+type Certificate struct {
+	ID              string   `json:"id,omitempty"`
+	Name            string   `json:"name,omitempty"`
+	DNSNames        []string `json:"dns_names,omitempty"`
+	NotAfter        string   `json:"not_after,omitempty"`
+	SHA1Fingerprint string   `json:"sha1_fingerprint,omitempty"`
+	Created         string   `json:"created_at,omitempty"`
+	State           string   `json:"state,omitempty"`
+	Type            string   `json:"type,omitempty"`
+}
+
+// CertificateRequest represents configuration for a new certificate.
+type CertificateRequest struct {
+	Name             string   `json:"name,omitempty"`
+	DNSNames         []string `json:"dns_names,omitempty"`
+	PrivateKey       string   `json:"private_key,omitempty"`
+	LeafCertificate  string   `json:"leaf_certificate,omitempty"`
+	CertificateChain string   `json:"certificate_chain,omitempty"`
+	Type             string   `json:"type,omitempty"`
+}
+
+type certificateRoot struct {
+	Certificate *Certificate `json:"certificate"`
+}
+
+type certificatesRoot struct {
+	Certificates []Certificate `json:"certificates"`
+	Links        *Links        `json:"links"`
+	Meta         *Meta         `json:"meta"`
+}
+
+// CertificatesServiceOp handles communication with certificates methods of the DigitalOcean API.
+type CertificatesServiceOp struct {
+	client *Client
+}
+
+var _ CertificatesService = &CertificatesServiceOp{}
+
+// Get an existing certificate by its identifier.
+func (c *CertificatesServiceOp) Get(ctx context.Context, cID string) (*Certificate, *Response, error) {
+	urlStr := path.Join(certificatesBasePath, cID)
+
+	req, err := c.client.NewRequest(ctx, http.MethodGet, urlStr, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(certificateRoot)
+	resp, err := c.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Certificate, resp, nil
+}
+
+// List all certificates.
+func (c *CertificatesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Certificate, *Response, error) {
+	urlStr, err := addOptions(certificatesBasePath, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := c.client.NewRequest(ctx, http.MethodGet, urlStr, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(certificatesRoot)
+	resp, err := c.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Certificates, resp, nil
+}
+
+// Create a new certificate with provided configuration.
+func (c *CertificatesServiceOp) Create(ctx context.Context, cr *CertificateRequest) (*Certificate, *Response, error) {
+	req, err := c.client.NewRequest(ctx, http.MethodPost, certificatesBasePath, cr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(certificateRoot)
+	resp, err := c.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Certificate, resp, nil
+}
+
+// Delete a certificate by its identifier.
+func (c *CertificatesServiceOp) Delete(ctx context.Context, cID string) (*Response, error) {
+	urlStr := path.Join(certificatesBasePath, cID)
+
+	req, err := c.client.NewRequest(ctx, http.MethodDelete, urlStr, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return c.client.Do(ctx, req, nil)
+}

vendor/github.com/digitalocean/godo/databases.go

@@ -0,0 +1,845 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+const (
+	databaseBasePath           = "/v2/databases"
+	databaseSinglePath         = databaseBasePath + "/%s"
+	databaseResizePath         = databaseBasePath + "/%s/resize"
+	databaseMigratePath        = databaseBasePath + "/%s/migrate"
+	databaseMaintenancePath    = databaseBasePath + "/%s/maintenance"
+	databaseBackupsPath        = databaseBasePath + "/%s/backups"
+	databaseUsersPath          = databaseBasePath + "/%s/users"
+	databaseUserPath           = databaseBasePath + "/%s/users/%s"
+	databaseResetUserAuthPath  = databaseUserPath + "/reset_auth"
+	databaseDBPath             = databaseBasePath + "/%s/dbs/%s"
+	databaseDBsPath            = databaseBasePath + "/%s/dbs"
+	databasePoolPath           = databaseBasePath + "/%s/pools/%s"
+	databasePoolsPath          = databaseBasePath + "/%s/pools"
+	databaseReplicaPath        = databaseBasePath + "/%s/replicas/%s"
+	databaseReplicasPath       = databaseBasePath + "/%s/replicas"
+	databaseEvictionPolicyPath = databaseBasePath + "/%s/eviction_policy"
+	databaseSQLModePath        = databaseBasePath + "/%s/sql_mode"
+	databaseFirewallRulesPath  = databaseBasePath + "/%s/firewall"
+)
+
+// SQL Mode constants allow for MySQL-specific SQL flavor configuration.
+const (
+	SQLModeAllowInvalidDates     = "ALLOW_INVALID_DATES"
+	SQLModeANSIQuotes            = "ANSI_QUOTES"
+	SQLModeHighNotPrecedence     = "HIGH_NOT_PRECEDENCE"
+	SQLModeIgnoreSpace           = "IGNORE_SPACE"
+	SQLModeNoAuthCreateUser      = "NO_AUTO_CREATE_USER"
+	SQLModeNoAutoValueOnZero     = "NO_AUTO_VALUE_ON_ZERO"
+	SQLModeNoBackslashEscapes    = "NO_BACKSLASH_ESCAPES"
+	SQLModeNoDirInCreate         = "NO_DIR_IN_CREATE"
+	SQLModeNoEngineSubstitution  = "NO_ENGINE_SUBSTITUTION"
+	SQLModeNoFieldOptions        = "NO_FIELD_OPTIONS"
+	SQLModeNoKeyOptions          = "NO_KEY_OPTIONS"
+	SQLModeNoTableOptions        = "NO_TABLE_OPTIONS"
+	SQLModeNoUnsignedSubtraction = "NO_UNSIGNED_SUBTRACTION"
+	SQLModeNoZeroDate            = "NO_ZERO_DATE"
+	SQLModeNoZeroInDate          = "NO_ZERO_IN_DATE"
+	SQLModeOnlyFullGroupBy       = "ONLY_FULL_GROUP_BY"
+	SQLModePadCharToFullLength   = "PAD_CHAR_TO_FULL_LENGTH"
+	SQLModePipesAsConcat         = "PIPES_AS_CONCAT"
+	SQLModeRealAsFloat           = "REAL_AS_FLOAT"
+	SQLModeStrictAllTables       = "STRICT_ALL_TABLES"
+	SQLModeStrictTransTables     = "STRICT_TRANS_TABLES"
+	SQLModeANSI                  = "ANSI"
+	SQLModeDB2                   = "DB2"
+	SQLModeMaxDB                 = "MAXDB"
+	SQLModeMSSQL                 = "MSSQL"
+	SQLModeMYSQL323              = "MYSQL323"
+	SQLModeMYSQL40               = "MYSQL40"
+	SQLModeOracle                = "ORACLE"
+	SQLModePostgreSQL            = "POSTGRESQL"
+	SQLModeTraditional           = "TRADITIONAL"
+)
+
+// SQL Auth constants allow for MySQL-specific user auth plugins
+const (
+	SQLAuthPluginNative      = "mysql_native_password"
+	SQLAuthPluginCachingSHA2 = "caching_sha2_password"
+)
+
+// Redis eviction policies supported by the managed Redis product.
+const (
+	EvictionPolicyNoEviction     = "noeviction"
+	EvictionPolicyAllKeysLRU     = "allkeys_lru"
+	EvictionPolicyAllKeysRandom  = "allkeys_random"
+	EvictionPolicyVolatileLRU    = "volatile_lru"
+	EvictionPolicyVolatileRandom = "volatile_random"
+	EvictionPolicyVolatileTTL    = "volatile_ttl"
+)
+
+// The DatabasesService provides access to the DigitalOcean managed database
+// suite of products through the public API. Customers can create new database
+// clusters, migrate them  between regions, create replicas and interact with
+// their configurations. Each database service is refered to as a Database. A
+// SQL database service can have multiple databases residing in the system. To
+// help make these entities distinct from Databases in godo, we refer to them
+// here as DatabaseDBs.
+//
+// See: https://developers.digitalocean.com/documentation/v2#databases
+type DatabasesService interface {
+	List(context.Context, *ListOptions) ([]Database, *Response, error)
+	Get(context.Context, string) (*Database, *Response, error)
+	Create(context.Context, *DatabaseCreateRequest) (*Database, *Response, error)
+	Delete(context.Context, string) (*Response, error)
+	Resize(context.Context, string, *DatabaseResizeRequest) (*Response, error)
+	Migrate(context.Context, string, *DatabaseMigrateRequest) (*Response, error)
+	UpdateMaintenance(context.Context, string, *DatabaseUpdateMaintenanceRequest) (*Response, error)
+	ListBackups(context.Context, string, *ListOptions) ([]DatabaseBackup, *Response, error)
+	GetUser(context.Context, string, string) (*DatabaseUser, *Response, error)
+	ListUsers(context.Context, string, *ListOptions) ([]DatabaseUser, *Response, error)
+	CreateUser(context.Context, string, *DatabaseCreateUserRequest) (*DatabaseUser, *Response, error)
+	DeleteUser(context.Context, string, string) (*Response, error)
+	ResetUserAuth(context.Context, string, string, *DatabaseResetUserAuthRequest) (*DatabaseUser, *Response, error)
+	ListDBs(context.Context, string, *ListOptions) ([]DatabaseDB, *Response, error)
+	CreateDB(context.Context, string, *DatabaseCreateDBRequest) (*DatabaseDB, *Response, error)
+	GetDB(context.Context, string, string) (*DatabaseDB, *Response, error)
+	DeleteDB(context.Context, string, string) (*Response, error)
+	ListPools(context.Context, string, *ListOptions) ([]DatabasePool, *Response, error)
+	CreatePool(context.Context, string, *DatabaseCreatePoolRequest) (*DatabasePool, *Response, error)
+	GetPool(context.Context, string, string) (*DatabasePool, *Response, error)
+	DeletePool(context.Context, string, string) (*Response, error)
+	GetReplica(context.Context, string, string) (*DatabaseReplica, *Response, error)
+	ListReplicas(context.Context, string, *ListOptions) ([]DatabaseReplica, *Response, error)
+	CreateReplica(context.Context, string, *DatabaseCreateReplicaRequest) (*DatabaseReplica, *Response, error)
+	DeleteReplica(context.Context, string, string) (*Response, error)
+	GetEvictionPolicy(context.Context, string) (string, *Response, error)
+	SetEvictionPolicy(context.Context, string, string) (*Response, error)
+	GetSQLMode(context.Context, string) (string, *Response, error)
+	SetSQLMode(context.Context, string, ...string) (*Response, error)
+	GetFirewallRules(context.Context, string) ([]DatabaseFirewallRule, *Response, error)
+	UpdateFirewallRules(context.Context, string, *DatabaseUpdateFirewallRulesRequest) (*Response, error)
+}
+
+// DatabasesServiceOp handles communication with the Databases related methods
+// of the DigitalOcean API.
+type DatabasesServiceOp struct {
+	client *Client
+}
+
+var _ DatabasesService = &DatabasesServiceOp{}
+
+// Database represents a DigitalOcean managed database product. These managed databases
+// are usually comprised of a cluster of database nodes, a primary and 0 or more replicas.
+// The EngineSlug is a string which indicates the type of database service. Some examples are
+// "pg", "mysql" or "redis". A Database also includes connection information and other
+// properties of the service like region, size and current status.
+type Database struct {
+	ID                 string                     `json:"id,omitempty"`
+	Name               string                     `json:"name,omitempty"`
+	EngineSlug         string                     `json:"engine,omitempty"`
+	VersionSlug        string                     `json:"version,omitempty"`
+	Connection         *DatabaseConnection        `json:"connection,omitempty"`
+	PrivateConnection  *DatabaseConnection        `json:"private_connection,omitempty"`
+	Users              []DatabaseUser             `json:"users,omitempty"`
+	NumNodes           int                        `json:"num_nodes,omitempty"`
+	SizeSlug           string                     `json:"size,omitempty"`
+	DBNames            []string                   `json:"db_names,omitempty"`
+	RegionSlug         string                     `json:"region,omitempty"`
+	Status             string                     `json:"status,omitempty"`
+	MaintenanceWindow  *DatabaseMaintenanceWindow `json:"maintenance_window,omitempty"`
+	CreatedAt          time.Time                  `json:"created_at,omitempty"`
+	PrivateNetworkUUID string                     `json:"private_network_uuid,omitempty"`
+	Tags               []string                   `json:"tags,omitempty"`
+}
+
+// DatabaseConnection represents a database connection
+type DatabaseConnection struct {
+	URI      string `json:"uri,omitempty"`
+	Database string `json:"database,omitempty"`
+	Host     string `json:"host,omitempty"`
+	Port     int    `json:"port,omitempty"`
+	User     string `json:"user,omitempty"`
+	Password string `json:"password,omitempty"`
+	SSL      bool   `json:"ssl,omitempty"`
+}
+
+// DatabaseUser represents a user in the database
+type DatabaseUser struct {
+	Name          string                     `json:"name,omitempty"`
+	Role          string                     `json:"role,omitempty"`
+	Password      string                     `json:"password,omitempty"`
+	MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"`
+}
+
+// DatabaseMySQLUserSettings contains MySQL-specific user settings
+type DatabaseMySQLUserSettings struct {
+	AuthPlugin string `json:"auth_plugin"`
+}
+
+// DatabaseMaintenanceWindow represents the maintenance_window of a database
+// cluster
+type DatabaseMaintenanceWindow struct {
+	Day         string   `json:"day,omitempty"`
+	Hour        string   `json:"hour,omitempty"`
+	Pending     bool     `json:"pending,omitempty"`
+	Description []string `json:"description,omitempty"`
+}
+
+// DatabaseBackup represents a database backup.
+type DatabaseBackup struct {
+	CreatedAt     time.Time `json:"created_at,omitempty"`
+	SizeGigabytes float64   `json:"size_gigabytes,omitempty"`
+}
+
+// DatabaseCreateRequest represents a request to create a database cluster
+type DatabaseCreateRequest struct {
+	Name               string   `json:"name,omitempty"`
+	EngineSlug         string   `json:"engine,omitempty"`
+	Version            string   `json:"version,omitempty"`
+	SizeSlug           string   `json:"size,omitempty"`
+	Region             string   `json:"region,omitempty"`
+	NumNodes           int      `json:"num_nodes,omitempty"`
+	PrivateNetworkUUID string   `json:"private_network_uuid"`
+	Tags               []string `json:"tags,omitempty"`
+}
+
+// DatabaseResizeRequest can be used to initiate a database resize operation.
+type DatabaseResizeRequest struct {
+	SizeSlug string `json:"size,omitempty"`
+	NumNodes int    `json:"num_nodes,omitempty"`
+}
+
+// DatabaseMigrateRequest can be used to initiate a database migrate operation.
+type DatabaseMigrateRequest struct {
+	Region             string `json:"region,omitempty"`
+	PrivateNetworkUUID string `json:"private_network_uuid"`
+}
+
+// DatabaseUpdateMaintenanceRequest can be used to update the database's maintenance window.
+type DatabaseUpdateMaintenanceRequest struct {
+	Day  string `json:"day,omitempty"`
+	Hour string `json:"hour,omitempty"`
+}
+
+// DatabaseDB represents an engine-specific database created within a database cluster. For SQL
+// databases like PostgreSQL or MySQL, a "DB" refers to a database created on the RDBMS. For instance,
+// a PostgreSQL database server can contain many database schemas, each with it's own settings, access
+// permissions and data. ListDBs will return all databases present on the server.
+type DatabaseDB struct {
+	Name string `json:"name"`
+}
+
+// DatabaseReplica represents a read-only replica of a particular database
+type DatabaseReplica struct {
+	Name               string              `json:"name"`
+	Connection         *DatabaseConnection `json:"connection"`
+	PrivateConnection  *DatabaseConnection `json:"private_connection,omitempty"`
+	Region             string              `json:"region"`
+	Status             string              `json:"status"`
+	CreatedAt          time.Time           `json:"created_at"`
+	PrivateNetworkUUID string              `json:"private_network_uuid,omitempty"`
+	Tags               []string            `json:"tags,omitempty"`
+}
+
+// DatabasePool represents a database connection pool
+type DatabasePool struct {
+	User              string              `json:"user"`
+	Name              string              `json:"name"`
+	Size              int                 `json:"size"`
+	Database          string              `json:"db"`
+	Mode              string              `json:"mode"`
+	Connection        *DatabaseConnection `json:"connection"`
+	PrivateConnection *DatabaseConnection `json:"private_connection,omitempty"`
+}
+
+// DatabaseCreatePoolRequest is used to create a new database connection pool
+type DatabaseCreatePoolRequest struct {
+	User     string `json:"user"`
+	Name     string `json:"name"`
+	Size     int    `json:"size"`
+	Database string `json:"db"`
+	Mode     string `json:"mode"`
+}
+
+// DatabaseCreateUserRequest is used to create a new database user
+type DatabaseCreateUserRequest struct {
+	Name          string                     `json:"name"`
+	MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"`
+}
+
+// DatabaseResetUserAuth request is used to reset a users DB auth
+type DatabaseResetUserAuthRequest struct {
+	MySQLSettings *DatabaseMySQLUserSettings `json:"mysql_settings,omitempty"`
+}
+
+// DatabaseCreateDBRequest is used to create a new engine-specific database within the cluster
+type DatabaseCreateDBRequest struct {
+	Name string `json:"name"`
+}
+
+// DatabaseCreateReplicaRequest is used to create a new read-only replica
+type DatabaseCreateReplicaRequest struct {
+	Name               string   `json:"name"`
+	Region             string   `json:"region"`
+	Size               string   `json:"size"`
+	PrivateNetworkUUID string   `json:"private_network_uuid"`
+	Tags               []string `json:"tags,omitempty"`
+}
+
+// DatabaseUpdateFirewallRulesRequest is used to set the firewall rules for a database
+type DatabaseUpdateFirewallRulesRequest struct {
+	Rules []*DatabaseFirewallRule `json:"rules"`
+}
+
+// DatabaseFirewallRule is a rule describing an inbound source to a database
+type DatabaseFirewallRule struct {
+	UUID        string    `json:"uuid"`
+	ClusterUUID string    `json:"cluster_uuid"`
+	Type        string    `json:"type"`
+	Value       string    `json:"value"`
+	CreatedAt   time.Time `json:"created_at"`
+}
+
+type databaseUserRoot struct {
+	User *DatabaseUser `json:"user"`
+}
+
+type databaseUsersRoot struct {
+	Users []DatabaseUser `json:"users"`
+}
+
+type databaseDBRoot struct {
+	DB *DatabaseDB `json:"db"`
+}
+
+type databaseDBsRoot struct {
+	DBs []DatabaseDB `json:"dbs"`
+}
+
+type databasesRoot struct {
+	Databases []Database `json:"databases"`
+}
+
+type databaseRoot struct {
+	Database *Database `json:"database"`
+}
+
+type databaseBackupsRoot struct {
+	Backups []DatabaseBackup `json:"backups"`
+}
+
+type databasePoolRoot struct {
+	Pool *DatabasePool `json:"pool"`
+}
+
+type databasePoolsRoot struct {
+	Pools []DatabasePool `json:"pools"`
+}
+
+type databaseReplicaRoot struct {
+	Replica *DatabaseReplica `json:"replica"`
+}
+
+type databaseReplicasRoot struct {
+	Replicas []DatabaseReplica `json:"replicas"`
+}
+
+type evictionPolicyRoot struct {
+	EvictionPolicy string `json:"eviction_policy"`
+}
+
+type sqlModeRoot struct {
+	SQLMode string `json:"sql_mode"`
+}
+
+type databaseFirewallRuleRoot struct {
+	Rules []DatabaseFirewallRule `json:"rules"`
+}
+
+// URN returns a URN identifier for the database
+func (d Database) URN() string {
+	return ToURN("dbaas", d.ID)
+}
+
+// List returns a list of the Databases visible with the caller's API token
+func (svc *DatabasesServiceOp) List(ctx context.Context, opts *ListOptions) ([]Database, *Response, error) {
+	path := databaseBasePath
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databasesRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Databases, resp, nil
+}
+
+// Get retrieves the details of a database cluster
+func (svc *DatabasesServiceOp) Get(ctx context.Context, databaseID string) (*Database, *Response, error) {
+	path := fmt.Sprintf(databaseSinglePath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Database, resp, nil
+}
+
+// Create creates a database cluster
+func (svc *DatabasesServiceOp) Create(ctx context.Context, create *DatabaseCreateRequest) (*Database, *Response, error) {
+	path := databaseBasePath
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Database, resp, nil
+}
+
+// Delete deletes a database cluster. There is no way to recover a cluster once
+// it has been destroyed.
+func (svc *DatabasesServiceOp) Delete(ctx context.Context, databaseID string) (*Response, error) {
+	path := fmt.Sprintf("%s/%s", databaseBasePath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// Resize resizes a database cluster by number of nodes or size
+func (svc *DatabasesServiceOp) Resize(ctx context.Context, databaseID string, resize *DatabaseResizeRequest) (*Response, error) {
+	path := fmt.Sprintf(databaseResizePath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPut, path, resize)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// Migrate migrates a database cluster to a new region
+func (svc *DatabasesServiceOp) Migrate(ctx context.Context, databaseID string, migrate *DatabaseMigrateRequest) (*Response, error) {
+	path := fmt.Sprintf(databaseMigratePath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPut, path, migrate)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// UpdateMaintenance updates the maintenance window on a cluster
+func (svc *DatabasesServiceOp) UpdateMaintenance(ctx context.Context, databaseID string, maintenance *DatabaseUpdateMaintenanceRequest) (*Response, error) {
+	path := fmt.Sprintf(databaseMaintenancePath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPut, path, maintenance)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// ListBackups returns a list of the current backups of a database
+func (svc *DatabasesServiceOp) ListBackups(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseBackup, *Response, error) {
+	path := fmt.Sprintf(databaseBackupsPath, databaseID)
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseBackupsRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Backups, resp, nil
+}
+
+// GetUser returns the database user identified by userID
+func (svc *DatabasesServiceOp) GetUser(ctx context.Context, databaseID, userID string) (*DatabaseUser, *Response, error) {
+	path := fmt.Sprintf(databaseUserPath, databaseID, userID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseUserRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.User, resp, nil
+}
+
+// ListUsers returns all database users for the database
+func (svc *DatabasesServiceOp) ListUsers(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseUser, *Response, error) {
+	path := fmt.Sprintf(databaseUsersPath, databaseID)
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseUsersRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Users, resp, nil
+}
+
+// CreateUser will create a new database user
+func (svc *DatabasesServiceOp) CreateUser(ctx context.Context, databaseID string, createUser *DatabaseCreateUserRequest) (*DatabaseUser, *Response, error) {
+	path := fmt.Sprintf(databaseUsersPath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createUser)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseUserRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.User, resp, nil
+}
+
+func (svc *DatabasesServiceOp) ResetUserAuth(ctx context.Context, databaseID, userID string, resetAuth *DatabaseResetUserAuthRequest) (*DatabaseUser, *Response, error) {
+	path := fmt.Sprintf(databaseResetUserAuthPath, databaseID, userID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, resetAuth)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseUserRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.User, resp, nil
+}
+
+// DeleteUser will delete an existing database user
+func (svc *DatabasesServiceOp) DeleteUser(ctx context.Context, databaseID, userID string) (*Response, error) {
+	path := fmt.Sprintf(databaseUserPath, databaseID, userID)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// ListDBs returns all databases for a given database cluster
+func (svc *DatabasesServiceOp) ListDBs(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseDB, *Response, error) {
+	path := fmt.Sprintf(databaseDBsPath, databaseID)
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseDBsRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.DBs, resp, nil
+}
+
+// GetDB returns a single database by name
+func (svc *DatabasesServiceOp) GetDB(ctx context.Context, databaseID, name string) (*DatabaseDB, *Response, error) {
+	path := fmt.Sprintf(databaseDBPath, databaseID, name)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseDBRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.DB, resp, nil
+}
+
+// CreateDB will create a new database
+func (svc *DatabasesServiceOp) CreateDB(ctx context.Context, databaseID string, createDB *DatabaseCreateDBRequest) (*DatabaseDB, *Response, error) {
+	path := fmt.Sprintf(databaseDBsPath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createDB)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseDBRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.DB, resp, nil
+}
+
+// DeleteDB will delete an existing database
+func (svc *DatabasesServiceOp) DeleteDB(ctx context.Context, databaseID, name string) (*Response, error) {
+	path := fmt.Sprintf(databaseDBPath, databaseID, name)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// ListPools returns all connection pools for a given database cluster
+func (svc *DatabasesServiceOp) ListPools(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabasePool, *Response, error) {
+	path := fmt.Sprintf(databasePoolsPath, databaseID)
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databasePoolsRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Pools, resp, nil
+}
+
+// GetPool returns a single database connection pool by name
+func (svc *DatabasesServiceOp) GetPool(ctx context.Context, databaseID, name string) (*DatabasePool, *Response, error) {
+	path := fmt.Sprintf(databasePoolPath, databaseID, name)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databasePoolRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Pool, resp, nil
+}
+
+// CreatePool will create a new database connection pool
+func (svc *DatabasesServiceOp) CreatePool(ctx context.Context, databaseID string, createPool *DatabaseCreatePoolRequest) (*DatabasePool, *Response, error) {
+	path := fmt.Sprintf(databasePoolsPath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createPool)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databasePoolRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Pool, resp, nil
+}
+
+// DeletePool will delete an existing database connection pool
+func (svc *DatabasesServiceOp) DeletePool(ctx context.Context, databaseID, name string) (*Response, error) {
+	path := fmt.Sprintf(databasePoolPath, databaseID, name)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// GetReplica returns a single database replica
+func (svc *DatabasesServiceOp) GetReplica(ctx context.Context, databaseID, name string) (*DatabaseReplica, *Response, error) {
+	path := fmt.Sprintf(databaseReplicaPath, databaseID, name)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseReplicaRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Replica, resp, nil
+}
+
+// ListReplicas returns all read-only replicas for a given database cluster
+func (svc *DatabasesServiceOp) ListReplicas(ctx context.Context, databaseID string, opts *ListOptions) ([]DatabaseReplica, *Response, error) {
+	path := fmt.Sprintf(databaseReplicasPath, databaseID)
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseReplicasRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Replicas, resp, nil
+}
+
+// CreateReplica will create a new database connection pool
+func (svc *DatabasesServiceOp) CreateReplica(ctx context.Context, databaseID string, createReplica *DatabaseCreateReplicaRequest) (*DatabaseReplica, *Response, error) {
+	path := fmt.Sprintf(databaseReplicasPath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, createReplica)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(databaseReplicaRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Replica, resp, nil
+}
+
+// DeleteReplica will delete an existing database replica
+func (svc *DatabasesServiceOp) DeleteReplica(ctx context.Context, databaseID, name string) (*Response, error) {
+	path := fmt.Sprintf(databaseReplicaPath, databaseID, name)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// GetEvictionPolicy loads the eviction policy for a given Redis cluster.
+func (svc *DatabasesServiceOp) GetEvictionPolicy(ctx context.Context, databaseID string) (string, *Response, error) {
+	path := fmt.Sprintf(databaseEvictionPolicyPath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return "", nil, err
+	}
+	root := new(evictionPolicyRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return "", resp, err
+	}
+	return root.EvictionPolicy, resp, nil
+}
+
+// SetEvictionPolicy updates the eviction policy for a given Redis cluster.
+//
+// The valid eviction policies are documented by the exported string constants
+// with the prefix `EvictionPolicy`.
+func (svc *DatabasesServiceOp) SetEvictionPolicy(ctx context.Context, databaseID, policy string) (*Response, error) {
+	path := fmt.Sprintf(databaseEvictionPolicyPath, databaseID)
+	root := &evictionPolicyRoot{EvictionPolicy: policy}
+	req, err := svc.client.NewRequest(ctx, http.MethodPut, path, root)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// GetSQLMode loads the SQL Mode settings for a given MySQL cluster.
+func (svc *DatabasesServiceOp) GetSQLMode(ctx context.Context, databaseID string) (string, *Response, error) {
+	path := fmt.Sprintf(databaseSQLModePath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return "", nil, err
+	}
+	root := &sqlModeRoot{}
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return "", resp, err
+	}
+	return root.SQLMode, resp, nil
+}
+
+// SetSQLMode updates the SQL Mode settings for a given MySQL cluster.
+func (svc *DatabasesServiceOp) SetSQLMode(ctx context.Context, databaseID string, sqlModes ...string) (*Response, error) {
+	path := fmt.Sprintf(databaseSQLModePath, databaseID)
+	root := &sqlModeRoot{SQLMode: strings.Join(sqlModes, ",")}
+	req, err := svc.client.NewRequest(ctx, http.MethodPut, path, root)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// GetFirewallRules loads the inbound sources for a given cluster.
+func (svc *DatabasesServiceOp) GetFirewallRules(ctx context.Context, databaseID string) ([]DatabaseFirewallRule, *Response, error) {
+	path := fmt.Sprintf(databaseFirewallRulesPath, databaseID)
+	root := new(databaseFirewallRuleRoot)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Rules, resp, nil
+}
+
+// UpdateFirewallRules sets the inbound sources for a given cluster.
+func (svc *DatabasesServiceOp) UpdateFirewallRules(ctx context.Context, databaseID string, firewallRulesReq *DatabaseUpdateFirewallRulesRequest) (*Response, error) {
+	path := fmt.Sprintf(databaseFirewallRulesPath, databaseID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPut, path, firewallRulesReq)
+	if err != nil {
+		return nil, err
+	}
+	return svc.client.Do(ctx, req, nil)
+}

vendor/github.com/digitalocean/godo/doc.go

@@ -0,0 +1,2 @@
+// Package godo is the DigtalOcean API v2 client for Go.
+package godo

vendor/github.com/digitalocean/godo/domains.go

@@ -0,0 +1,341 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+const domainsBasePath = "v2/domains"
+
+// DomainsService is an interface for managing DNS with the DigitalOcean API.
+// See: https://developers.digitalocean.com/documentation/v2#domains and
+// https://developers.digitalocean.com/documentation/v2#domain-records
+type DomainsService interface {
+	List(context.Context, *ListOptions) ([]Domain, *Response, error)
+	Get(context.Context, string) (*Domain, *Response, error)
+	Create(context.Context, *DomainCreateRequest) (*Domain, *Response, error)
+	Delete(context.Context, string) (*Response, error)
+
+	Records(context.Context, string, *ListOptions) ([]DomainRecord, *Response, error)
+	Record(context.Context, string, int) (*DomainRecord, *Response, error)
+	DeleteRecord(context.Context, string, int) (*Response, error)
+	EditRecord(context.Context, string, int, *DomainRecordEditRequest) (*DomainRecord, *Response, error)
+	CreateRecord(context.Context, string, *DomainRecordEditRequest) (*DomainRecord, *Response, error)
+}
+
+// DomainsServiceOp handles communication with the domain related methods of the
+// DigitalOcean API.
+type DomainsServiceOp struct {
+	client *Client
+}
+
+var _ DomainsService = &DomainsServiceOp{}
+
+// Domain represents a DigitalOcean domain
+type Domain struct {
+	Name     string `json:"name"`
+	TTL      int    `json:"ttl"`
+	ZoneFile string `json:"zone_file"`
+}
+
+// domainRoot represents a response from the DigitalOcean API
+type domainRoot struct {
+	Domain *Domain `json:"domain"`
+}
+
+type domainsRoot struct {
+	Domains []Domain `json:"domains"`
+	Links   *Links   `json:"links"`
+	Meta    *Meta    `json:"meta"`
+}
+
+// DomainCreateRequest respresents a request to create a domain.
+type DomainCreateRequest struct {
+	Name      string `json:"name"`
+	IPAddress string `json:"ip_address,omitempty"`
+}
+
+// DomainRecordRoot is the root of an individual Domain Record response
+type domainRecordRoot struct {
+	DomainRecord *DomainRecord `json:"domain_record"`
+}
+
+// DomainRecordsRoot is the root of a group of Domain Record responses
+type domainRecordsRoot struct {
+	DomainRecords []DomainRecord `json:"domain_records"`
+	Links         *Links         `json:"links"`
+}
+
+// DomainRecord represents a DigitalOcean DomainRecord
+type DomainRecord struct {
+	ID       int    `json:"id,float64,omitempty"`
+	Type     string `json:"type,omitempty"`
+	Name     string `json:"name,omitempty"`
+	Data     string `json:"data,omitempty"`
+	Priority int    `json:"priority"`
+	Port     int    `json:"port,omitempty"`
+	TTL      int    `json:"ttl,omitempty"`
+	Weight   int    `json:"weight"`
+	Flags    int    `json:"flags"`
+	Tag      string `json:"tag,omitempty"`
+}
+
+// DomainRecordEditRequest represents a request to update a domain record.
+type DomainRecordEditRequest struct {
+	Type     string `json:"type,omitempty"`
+	Name     string `json:"name,omitempty"`
+	Data     string `json:"data,omitempty"`
+	Priority int    `json:"priority"`
+	Port     int    `json:"port,omitempty"`
+	TTL      int    `json:"ttl,omitempty"`
+	Weight   int    `json:"weight"`
+	Flags    int    `json:"flags"`
+	Tag      string `json:"tag,omitempty"`
+}
+
+func (d Domain) String() string {
+	return Stringify(d)
+}
+
+func (d Domain) URN() string {
+	return ToURN("Domain", d.Name)
+}
+
+// List all domains.
+func (s DomainsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Domain, *Response, error) {
+	path := domainsBasePath
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(domainsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Domains, resp, err
+}
+
+// Get individual domain. It requires a non-empty domain name.
+func (s *DomainsServiceOp) Get(ctx context.Context, name string) (*Domain, *Response, error) {
+	if len(name) < 1 {
+		return nil, nil, NewArgError("name", "cannot be an empty string")
+	}
+
+	path := fmt.Sprintf("%s/%s", domainsBasePath, name)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(domainRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Domain, resp, err
+}
+
+// Create a new domain
+func (s *DomainsServiceOp) Create(ctx context.Context, createRequest *DomainCreateRequest) (*Domain, *Response, error) {
+	if createRequest == nil {
+		return nil, nil, NewArgError("createRequest", "cannot be nil")
+	}
+
+	path := domainsBasePath
+
+	req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(domainRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Domain, resp, err
+}
+
+// Delete domain
+func (s *DomainsServiceOp) Delete(ctx context.Context, name string) (*Response, error) {
+	if len(name) < 1 {
+		return nil, NewArgError("name", "cannot be an empty string")
+	}
+
+	path := fmt.Sprintf("%s/%s", domainsBasePath, name)
+
+	req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := s.client.Do(ctx, req, nil)
+
+	return resp, err
+}
+
+// Converts a DomainRecord to a string.
+func (d DomainRecord) String() string {
+	return Stringify(d)
+}
+
+// Converts a DomainRecordEditRequest to a string.
+func (d DomainRecordEditRequest) String() string {
+	return Stringify(d)
+}
+
+// Records returns a slice of DomainRecords for a domain
+func (s *DomainsServiceOp) Records(ctx context.Context, domain string, opt *ListOptions) ([]DomainRecord, *Response, error) {
+	if len(domain) < 1 {
+		return nil, nil, NewArgError("domain", "cannot be an empty string")
+	}
+
+	path := fmt.Sprintf("%s/%s/records", domainsBasePath, domain)
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(domainRecordsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+
+	return root.DomainRecords, resp, err
+}
+
+// Record returns the record id from a domain
+func (s *DomainsServiceOp) Record(ctx context.Context, domain string, id int) (*DomainRecord, *Response, error) {
+	if len(domain) < 1 {
+		return nil, nil, NewArgError("domain", "cannot be an empty string")
+	}
+
+	if id < 1 {
+		return nil, nil, NewArgError("id", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	record := new(domainRecordRoot)
+	resp, err := s.client.Do(ctx, req, record)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return record.DomainRecord, resp, err
+}
+
+// DeleteRecord deletes a record from a domain identified by id
+func (s *DomainsServiceOp) DeleteRecord(ctx context.Context, domain string, id int) (*Response, error) {
+	if len(domain) < 1 {
+		return nil, NewArgError("domain", "cannot be an empty string")
+	}
+
+	if id < 1 {
+		return nil, NewArgError("id", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id)
+
+	req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := s.client.Do(ctx, req, nil)
+
+	return resp, err
+}
+
+// EditRecord edits a record using a DomainRecordEditRequest
+func (s *DomainsServiceOp) EditRecord(ctx context.Context,
+	domain string,
+	id int,
+	editRequest *DomainRecordEditRequest,
+) (*DomainRecord, *Response, error) {
+	if len(domain) < 1 {
+		return nil, nil, NewArgError("domain", "cannot be an empty string")
+	}
+
+	if id < 1 {
+		return nil, nil, NewArgError("id", "cannot be less than 1")
+	}
+
+	if editRequest == nil {
+		return nil, nil, NewArgError("editRequest", "cannot be nil")
+	}
+
+	path := fmt.Sprintf("%s/%s/records/%d", domainsBasePath, domain, id)
+
+	req, err := s.client.NewRequest(ctx, http.MethodPut, path, editRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(domainRecordRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.DomainRecord, resp, err
+}
+
+// CreateRecord creates a record using a DomainRecordEditRequest
+func (s *DomainsServiceOp) CreateRecord(ctx context.Context,
+	domain string,
+	createRequest *DomainRecordEditRequest) (*DomainRecord, *Response, error) {
+	if len(domain) < 1 {
+		return nil, nil, NewArgError("domain", "cannot be empty string")
+	}
+
+	if createRequest == nil {
+		return nil, nil, NewArgError("createRequest", "cannot be nil")
+	}
+
+	path := fmt.Sprintf("%s/%s/records", domainsBasePath, domain)
+	req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest)
+
+	if err != nil {
+		return nil, nil, err
+	}
+
+	d := new(domainRecordRoot)
+	resp, err := s.client.Do(ctx, req, d)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return d.DomainRecord, resp, err
+}

vendor/github.com/digitalocean/godo/droplet_actions.go

@@ -0,0 +1,329 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+)
+
+// ActionRequest reprents DigitalOcean Action Request
+type ActionRequest map[string]interface{}
+
+// DropletActionsService is an interface for interfacing with the Droplet actions
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2#droplet-actions
+type DropletActionsService interface {
+	Shutdown(context.Context, int) (*Action, *Response, error)
+	ShutdownByTag(context.Context, string) ([]Action, *Response, error)
+	PowerOff(context.Context, int) (*Action, *Response, error)
+	PowerOffByTag(context.Context, string) ([]Action, *Response, error)
+	PowerOn(context.Context, int) (*Action, *Response, error)
+	PowerOnByTag(context.Context, string) ([]Action, *Response, error)
+	PowerCycle(context.Context, int) (*Action, *Response, error)
+	PowerCycleByTag(context.Context, string) ([]Action, *Response, error)
+	Reboot(context.Context, int) (*Action, *Response, error)
+	Restore(context.Context, int, int) (*Action, *Response, error)
+	Resize(context.Context, int, string, bool) (*Action, *Response, error)
+	Rename(context.Context, int, string) (*Action, *Response, error)
+	Snapshot(context.Context, int, string) (*Action, *Response, error)
+	SnapshotByTag(context.Context, string, string) ([]Action, *Response, error)
+	EnableBackups(context.Context, int) (*Action, *Response, error)
+	EnableBackupsByTag(context.Context, string) ([]Action, *Response, error)
+	DisableBackups(context.Context, int) (*Action, *Response, error)
+	DisableBackupsByTag(context.Context, string) ([]Action, *Response, error)
+	PasswordReset(context.Context, int) (*Action, *Response, error)
+	RebuildByImageID(context.Context, int, int) (*Action, *Response, error)
+	RebuildByImageSlug(context.Context, int, string) (*Action, *Response, error)
+	ChangeKernel(context.Context, int, int) (*Action, *Response, error)
+	EnableIPv6(context.Context, int) (*Action, *Response, error)
+	EnableIPv6ByTag(context.Context, string) ([]Action, *Response, error)
+	EnablePrivateNetworking(context.Context, int) (*Action, *Response, error)
+	EnablePrivateNetworkingByTag(context.Context, string) ([]Action, *Response, error)
+	Get(context.Context, int, int) (*Action, *Response, error)
+	GetByURI(context.Context, string) (*Action, *Response, error)
+}
+
+// DropletActionsServiceOp handles communication with the Droplet action related
+// methods of the DigitalOcean API.
+type DropletActionsServiceOp struct {
+	client *Client
+}
+
+var _ DropletActionsService = &DropletActionsServiceOp{}
+
+// Shutdown a Droplet
+func (s *DropletActionsServiceOp) Shutdown(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "shutdown"}
+	return s.doAction(ctx, id, request)
+}
+
+// ShutdownByTag shuts down Droplets matched by a Tag.
+func (s *DropletActionsServiceOp) ShutdownByTag(ctx context.Context, tag string) ([]Action, *Response, error) {
+	request := &ActionRequest{"type": "shutdown"}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+// PowerOff a Droplet
+func (s *DropletActionsServiceOp) PowerOff(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "power_off"}
+	return s.doAction(ctx, id, request)
+}
+
+// PowerOffByTag powers off Droplets matched by a Tag.
+func (s *DropletActionsServiceOp) PowerOffByTag(ctx context.Context, tag string) ([]Action, *Response, error) {
+	request := &ActionRequest{"type": "power_off"}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+// PowerOn a Droplet
+func (s *DropletActionsServiceOp) PowerOn(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "power_on"}
+	return s.doAction(ctx, id, request)
+}
+
+// PowerOnByTag powers on Droplets matched by a Tag.
+func (s *DropletActionsServiceOp) PowerOnByTag(ctx context.Context, tag string) ([]Action, *Response, error) {
+	request := &ActionRequest{"type": "power_on"}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+// PowerCycle a Droplet
+func (s *DropletActionsServiceOp) PowerCycle(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "power_cycle"}
+	return s.doAction(ctx, id, request)
+}
+
+// PowerCycleByTag power cycles Droplets matched by a Tag.
+func (s *DropletActionsServiceOp) PowerCycleByTag(ctx context.Context, tag string) ([]Action, *Response, error) {
+	request := &ActionRequest{"type": "power_cycle"}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+// Reboot a Droplet
+func (s *DropletActionsServiceOp) Reboot(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "reboot"}
+	return s.doAction(ctx, id, request)
+}
+
+// Restore an image to a Droplet
+func (s *DropletActionsServiceOp) Restore(ctx context.Context, id, imageID int) (*Action, *Response, error) {
+	requestType := "restore"
+	request := &ActionRequest{
+		"type":  requestType,
+		"image": float64(imageID),
+	}
+	return s.doAction(ctx, id, request)
+}
+
+// Resize a Droplet
+func (s *DropletActionsServiceOp) Resize(ctx context.Context, id int, sizeSlug string, resizeDisk bool) (*Action, *Response, error) {
+	requestType := "resize"
+	request := &ActionRequest{
+		"type": requestType,
+		"size": sizeSlug,
+		"disk": resizeDisk,
+	}
+	return s.doAction(ctx, id, request)
+}
+
+// Rename a Droplet
+func (s *DropletActionsServiceOp) Rename(ctx context.Context, id int, name string) (*Action, *Response, error) {
+	requestType := "rename"
+	request := &ActionRequest{
+		"type": requestType,
+		"name": name,
+	}
+	return s.doAction(ctx, id, request)
+}
+
+// Snapshot a Droplet.
+func (s *DropletActionsServiceOp) Snapshot(ctx context.Context, id int, name string) (*Action, *Response, error) {
+	requestType := "snapshot"
+	request := &ActionRequest{
+		"type": requestType,
+		"name": name,
+	}
+	return s.doAction(ctx, id, request)
+}
+
+// SnapshotByTag snapshots Droplets matched by a Tag.
+func (s *DropletActionsServiceOp) SnapshotByTag(ctx context.Context, tag string, name string) ([]Action, *Response, error) {
+	requestType := "snapshot"
+	request := &ActionRequest{
+		"type": requestType,
+		"name": name,
+	}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+// EnableBackups enables backups for a Droplet.
+func (s *DropletActionsServiceOp) EnableBackups(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "enable_backups"}
+	return s.doAction(ctx, id, request)
+}
+
+// EnableBackupsByTag enables backups for Droplets matched by a Tag.
+func (s *DropletActionsServiceOp) EnableBackupsByTag(ctx context.Context, tag string) ([]Action, *Response, error) {
+	request := &ActionRequest{"type": "enable_backups"}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+// DisableBackups disables backups for a Droplet.
+func (s *DropletActionsServiceOp) DisableBackups(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "disable_backups"}
+	return s.doAction(ctx, id, request)
+}
+
+// DisableBackupsByTag disables backups for Droplet matched by a Tag.
+func (s *DropletActionsServiceOp) DisableBackupsByTag(ctx context.Context, tag string) ([]Action, *Response, error) {
+	request := &ActionRequest{"type": "disable_backups"}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+// PasswordReset resets the password for a Droplet.
+func (s *DropletActionsServiceOp) PasswordReset(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "password_reset"}
+	return s.doAction(ctx, id, request)
+}
+
+// RebuildByImageID rebuilds a Droplet from an image with a given id.
+func (s *DropletActionsServiceOp) RebuildByImageID(ctx context.Context, id, imageID int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "rebuild", "image": imageID}
+	return s.doAction(ctx, id, request)
+}
+
+// RebuildByImageSlug rebuilds a Droplet from an Image matched by a given Slug.
+func (s *DropletActionsServiceOp) RebuildByImageSlug(ctx context.Context, id int, slug string) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "rebuild", "image": slug}
+	return s.doAction(ctx, id, request)
+}
+
+// ChangeKernel changes the kernel for a Droplet.
+func (s *DropletActionsServiceOp) ChangeKernel(ctx context.Context, id, kernelID int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "change_kernel", "kernel": kernelID}
+	return s.doAction(ctx, id, request)
+}
+
+// EnableIPv6 enables IPv6 for a Droplet.
+func (s *DropletActionsServiceOp) EnableIPv6(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "enable_ipv6"}
+	return s.doAction(ctx, id, request)
+}
+
+// EnableIPv6ByTag enables IPv6 for Droplets matched by a Tag.
+func (s *DropletActionsServiceOp) EnableIPv6ByTag(ctx context.Context, tag string) ([]Action, *Response, error) {
+	request := &ActionRequest{"type": "enable_ipv6"}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+// EnablePrivateNetworking enables private networking for a Droplet.
+func (s *DropletActionsServiceOp) EnablePrivateNetworking(ctx context.Context, id int) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "enable_private_networking"}
+	return s.doAction(ctx, id, request)
+}
+
+// EnablePrivateNetworkingByTag enables private networking for Droplets matched by a Tag.
+func (s *DropletActionsServiceOp) EnablePrivateNetworkingByTag(ctx context.Context, tag string) ([]Action, *Response, error) {
+	request := &ActionRequest{"type": "enable_private_networking"}
+	return s.doActionByTag(ctx, tag, request)
+}
+
+func (s *DropletActionsServiceOp) doAction(ctx context.Context, id int, request *ActionRequest) (*Action, *Response, error) {
+	if id < 1 {
+		return nil, nil, NewArgError("id", "cannot be less than 1")
+	}
+
+	if request == nil {
+		return nil, nil, NewArgError("request", "request can't be nil")
+	}
+
+	path := dropletActionPath(id)
+
+	req, err := s.client.NewRequest(ctx, http.MethodPost, path, request)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Event, resp, err
+}
+
+func (s *DropletActionsServiceOp) doActionByTag(ctx context.Context, tag string, request *ActionRequest) ([]Action, *Response, error) {
+	if tag == "" {
+		return nil, nil, NewArgError("tag", "cannot be empty")
+	}
+
+	if request == nil {
+		return nil, nil, NewArgError("request", "request can't be nil")
+	}
+
+	path := dropletActionPathByTag(tag)
+
+	req, err := s.client.NewRequest(ctx, http.MethodPost, path, request)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Actions, resp, err
+}
+
+// Get an action for a particular Droplet by id.
+func (s *DropletActionsServiceOp) Get(ctx context.Context, dropletID, actionID int) (*Action, *Response, error) {
+	if dropletID < 1 {
+		return nil, nil, NewArgError("dropletID", "cannot be less than 1")
+	}
+
+	if actionID < 1 {
+		return nil, nil, NewArgError("actionID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d", dropletActionPath(dropletID), actionID)
+	return s.get(ctx, path)
+}
+
+// GetByURI gets an action for a particular Droplet by id.
+func (s *DropletActionsServiceOp) GetByURI(ctx context.Context, rawurl string) (*Action, *Response, error) {
+	u, err := url.Parse(rawurl)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return s.get(ctx, u.Path)
+
+}
+
+func (s *DropletActionsServiceOp) get(ctx context.Context, path string) (*Action, *Response, error) {
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Event, resp, err
+
+}
+
+func dropletActionPath(dropletID int) string {
+	return fmt.Sprintf("v2/droplets/%d/actions", dropletID)
+}
+
+func dropletActionPathByTag(tag string) string {
+	return fmt.Sprintf("v2/droplets/actions?tag_name=%s", tag)
+}

vendor/github.com/digitalocean/godo/droplets.go

@@ -0,0 +1,592 @@
+package godo
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+)
+
+const dropletBasePath = "v2/droplets"
+
+var errNoNetworks = errors.New("no networks have been defined")
+
+// DropletsService is an interface for interfacing with the Droplet
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2#droplets
+type DropletsService interface {
+	List(context.Context, *ListOptions) ([]Droplet, *Response, error)
+	ListByTag(context.Context, string, *ListOptions) ([]Droplet, *Response, error)
+	Get(context.Context, int) (*Droplet, *Response, error)
+	Create(context.Context, *DropletCreateRequest) (*Droplet, *Response, error)
+	CreateMultiple(context.Context, *DropletMultiCreateRequest) ([]Droplet, *Response, error)
+	Delete(context.Context, int) (*Response, error)
+	DeleteByTag(context.Context, string) (*Response, error)
+	Kernels(context.Context, int, *ListOptions) ([]Kernel, *Response, error)
+	Snapshots(context.Context, int, *ListOptions) ([]Image, *Response, error)
+	Backups(context.Context, int, *ListOptions) ([]Image, *Response, error)
+	Actions(context.Context, int, *ListOptions) ([]Action, *Response, error)
+	Neighbors(context.Context, int) ([]Droplet, *Response, error)
+}
+
+// DropletsServiceOp handles communication with the Droplet related methods of the
+// DigitalOcean API.
+type DropletsServiceOp struct {
+	client *Client
+}
+
+var _ DropletsService = &DropletsServiceOp{}
+
+// Droplet represents a DigitalOcean Droplet
+type Droplet struct {
+	ID               int           `json:"id,float64,omitempty"`
+	Name             string        `json:"name,omitempty"`
+	Memory           int           `json:"memory,omitempty"`
+	Vcpus            int           `json:"vcpus,omitempty"`
+	Disk             int           `json:"disk,omitempty"`
+	Region           *Region       `json:"region,omitempty"`
+	Image            *Image        `json:"image,omitempty"`
+	Size             *Size         `json:"size,omitempty"`
+	SizeSlug         string        `json:"size_slug,omitempty"`
+	BackupIDs        []int         `json:"backup_ids,omitempty"`
+	NextBackupWindow *BackupWindow `json:"next_backup_window,omitempty"`
+	SnapshotIDs      []int         `json:"snapshot_ids,omitempty"`
+	Features         []string      `json:"features,omitempty"`
+	Locked           bool          `json:"locked,bool,omitempty"`
+	Status           string        `json:"status,omitempty"`
+	Networks         *Networks     `json:"networks,omitempty"`
+	Created          string        `json:"created_at,omitempty"`
+	Kernel           *Kernel       `json:"kernel,omitempty"`
+	Tags             []string      `json:"tags,omitempty"`
+	VolumeIDs        []string      `json:"volume_ids"`
+	VPCUUID          string        `json:"vpc_uuid,omitempty"`
+}
+
+// PublicIPv4 returns the public IPv4 address for the Droplet.
+func (d *Droplet) PublicIPv4() (string, error) {
+	if d.Networks == nil {
+		return "", errNoNetworks
+	}
+
+	for _, v4 := range d.Networks.V4 {
+		if v4.Type == "public" {
+			return v4.IPAddress, nil
+		}
+	}
+
+	return "", nil
+}
+
+// PrivateIPv4 returns the private IPv4 address for the Droplet.
+func (d *Droplet) PrivateIPv4() (string, error) {
+	if d.Networks == nil {
+		return "", errNoNetworks
+	}
+
+	for _, v4 := range d.Networks.V4 {
+		if v4.Type == "private" {
+			return v4.IPAddress, nil
+		}
+	}
+
+	return "", nil
+}
+
+// PublicIPv6 returns the public IPv6 address for the Droplet.
+func (d *Droplet) PublicIPv6() (string, error) {
+	if d.Networks == nil {
+		return "", errNoNetworks
+	}
+
+	for _, v6 := range d.Networks.V6 {
+		if v6.Type == "public" {
+			return v6.IPAddress, nil
+		}
+	}
+
+	return "", nil
+}
+
+// Kernel object
+type Kernel struct {
+	ID      int    `json:"id,float64,omitempty"`
+	Name    string `json:"name,omitempty"`
+	Version string `json:"version,omitempty"`
+}
+
+// BackupWindow object
+type BackupWindow struct {
+	Start *Timestamp `json:"start,omitempty"`
+	End   *Timestamp `json:"end,omitempty"`
+}
+
+// Convert Droplet to a string
+func (d Droplet) String() string {
+	return Stringify(d)
+}
+
+func (d Droplet) URN() string {
+	return ToURN("Droplet", d.ID)
+}
+
+// DropletRoot represents a Droplet root
+type dropletRoot struct {
+	Droplet *Droplet `json:"droplet"`
+	Links   *Links   `json:"links,omitempty"`
+}
+
+type dropletsRoot struct {
+	Droplets []Droplet `json:"droplets"`
+	Links    *Links    `json:"links"`
+	Meta     *Meta     `json:"meta"`
+}
+
+type kernelsRoot struct {
+	Kernels []Kernel `json:"kernels,omitempty"`
+	Links   *Links   `json:"links"`
+	Meta    *Meta    `json:"meta"`
+}
+
+type dropletSnapshotsRoot struct {
+	Snapshots []Image `json:"snapshots,omitempty"`
+	Links     *Links  `json:"links"`
+	Meta      *Meta   `json:"meta"`
+}
+
+type backupsRoot struct {
+	Backups []Image `json:"backups,omitempty"`
+	Links   *Links  `json:"links"`
+	Meta    *Meta   `json:"meta"`
+}
+
+// DropletCreateImage identifies an image for the create request. It prefers slug over ID.
+type DropletCreateImage struct {
+	ID   int
+	Slug string
+}
+
+// MarshalJSON returns either the slug or id of the image. It returns the id
+// if the slug is empty.
+func (d DropletCreateImage) MarshalJSON() ([]byte, error) {
+	if d.Slug != "" {
+		return json.Marshal(d.Slug)
+	}
+
+	return json.Marshal(d.ID)
+}
+
+// DropletCreateVolume identifies a volume to attach for the create request. It
+// prefers Name over ID,
+type DropletCreateVolume struct {
+	ID   string
+	Name string
+}
+
+// MarshalJSON returns an object with either the name or id of the volume. It
+// returns the id if the name is empty.
+func (d DropletCreateVolume) MarshalJSON() ([]byte, error) {
+	if d.Name != "" {
+		return json.Marshal(struct {
+			Name string `json:"name"`
+		}{Name: d.Name})
+	}
+
+	return json.Marshal(struct {
+		ID string `json:"id"`
+	}{ID: d.ID})
+}
+
+// DropletCreateSSHKey identifies a SSH Key for the create request. It prefers fingerprint over ID.
+type DropletCreateSSHKey struct {
+	ID          int
+	Fingerprint string
+}
+
+// MarshalJSON returns either the fingerprint or id of the ssh key. It returns
+// the id if the fingerprint is empty.
+func (d DropletCreateSSHKey) MarshalJSON() ([]byte, error) {
+	if d.Fingerprint != "" {
+		return json.Marshal(d.Fingerprint)
+	}
+
+	return json.Marshal(d.ID)
+}
+
+// DropletCreateRequest represents a request to create a Droplet.
+type DropletCreateRequest struct {
+	Name              string                `json:"name"`
+	Region            string                `json:"region"`
+	Size              string                `json:"size"`
+	Image             DropletCreateImage    `json:"image"`
+	SSHKeys           []DropletCreateSSHKey `json:"ssh_keys"`
+	Backups           bool                  `json:"backups"`
+	IPv6              bool                  `json:"ipv6"`
+	PrivateNetworking bool                  `json:"private_networking"`
+	Monitoring        bool                  `json:"monitoring"`
+	UserData          string                `json:"user_data,omitempty"`
+	Volumes           []DropletCreateVolume `json:"volumes,omitempty"`
+	Tags              []string              `json:"tags"`
+	VPCUUID           string                `json:"vpc_uuid,omitempty"`
+}
+
+// DropletMultiCreateRequest is a request to create multiple Droplets.
+type DropletMultiCreateRequest struct {
+	Names             []string              `json:"names"`
+	Region            string                `json:"region"`
+	Size              string                `json:"size"`
+	Image             DropletCreateImage    `json:"image"`
+	SSHKeys           []DropletCreateSSHKey `json:"ssh_keys"`
+	Backups           bool                  `json:"backups"`
+	IPv6              bool                  `json:"ipv6"`
+	PrivateNetworking bool                  `json:"private_networking"`
+	Monitoring        bool                  `json:"monitoring"`
+	UserData          string                `json:"user_data,omitempty"`
+	Tags              []string              `json:"tags"`
+	VPCUUID           string                `json:"vpc_uuid,omitempty"`
+}
+
+func (d DropletCreateRequest) String() string {
+	return Stringify(d)
+}
+
+func (d DropletMultiCreateRequest) String() string {
+	return Stringify(d)
+}
+
+// Networks represents the Droplet's Networks.
+type Networks struct {
+	V4 []NetworkV4 `json:"v4,omitempty"`
+	V6 []NetworkV6 `json:"v6,omitempty"`
+}
+
+// NetworkV4 represents a DigitalOcean IPv4 Network.
+type NetworkV4 struct {
+	IPAddress string `json:"ip_address,omitempty"`
+	Netmask   string `json:"netmask,omitempty"`
+	Gateway   string `json:"gateway,omitempty"`
+	Type      string `json:"type,omitempty"`
+}
+
+func (n NetworkV4) String() string {
+	return Stringify(n)
+}
+
+// NetworkV6 represents a DigitalOcean IPv6 network.
+type NetworkV6 struct {
+	IPAddress string `json:"ip_address,omitempty"`
+	Netmask   int    `json:"netmask,omitempty"`
+	Gateway   string `json:"gateway,omitempty"`
+	Type      string `json:"type,omitempty"`
+}
+
+func (n NetworkV6) String() string {
+	return Stringify(n)
+}
+
+// Performs a list request given a path.
+func (s *DropletsServiceOp) list(ctx context.Context, path string) ([]Droplet, *Response, error) {
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(dropletsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Droplets, resp, err
+}
+
+// List all Droplets.
+func (s *DropletsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Droplet, *Response, error) {
+	path := dropletBasePath
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return s.list(ctx, path)
+}
+
+// ListByTag lists all Droplets matched by a Tag.
+func (s *DropletsServiceOp) ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Droplet, *Response, error) {
+	path := fmt.Sprintf("%s?tag_name=%s", dropletBasePath, tag)
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return s.list(ctx, path)
+}
+
+// Get individual Droplet.
+func (s *DropletsServiceOp) Get(ctx context.Context, dropletID int) (*Droplet, *Response, error) {
+	if dropletID < 1 {
+		return nil, nil, NewArgError("dropletID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d", dropletBasePath, dropletID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(dropletRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Droplet, resp, err
+}
+
+// Create Droplet
+func (s *DropletsServiceOp) Create(ctx context.Context, createRequest *DropletCreateRequest) (*Droplet, *Response, error) {
+	if createRequest == nil {
+		return nil, nil, NewArgError("createRequest", "cannot be nil")
+	}
+
+	path := dropletBasePath
+
+	req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(dropletRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+
+	return root.Droplet, resp, err
+}
+
+// CreateMultiple creates multiple Droplets.
+func (s *DropletsServiceOp) CreateMultiple(ctx context.Context, createRequest *DropletMultiCreateRequest) ([]Droplet, *Response, error) {
+	if createRequest == nil {
+		return nil, nil, NewArgError("createRequest", "cannot be nil")
+	}
+
+	path := dropletBasePath
+
+	req, err := s.client.NewRequest(ctx, http.MethodPost, path, createRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(dropletsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+
+	return root.Droplets, resp, err
+}
+
+// Performs a delete request given a path
+func (s *DropletsServiceOp) delete(ctx context.Context, path string) (*Response, error) {
+	req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := s.client.Do(ctx, req, nil)
+
+	return resp, err
+}
+
+// Delete Droplet.
+func (s *DropletsServiceOp) Delete(ctx context.Context, dropletID int) (*Response, error) {
+	if dropletID < 1 {
+		return nil, NewArgError("dropletID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d", dropletBasePath, dropletID)
+
+	return s.delete(ctx, path)
+}
+
+// DeleteByTag deletes Droplets matched by a Tag.
+func (s *DropletsServiceOp) DeleteByTag(ctx context.Context, tag string) (*Response, error) {
+	if tag == "" {
+		return nil, NewArgError("tag", "cannot be empty")
+	}
+
+	path := fmt.Sprintf("%s?tag_name=%s", dropletBasePath, tag)
+
+	return s.delete(ctx, path)
+}
+
+// Kernels lists kernels available for a Droplet.
+func (s *DropletsServiceOp) Kernels(ctx context.Context, dropletID int, opt *ListOptions) ([]Kernel, *Response, error) {
+	if dropletID < 1 {
+		return nil, nil, NewArgError("dropletID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d/kernels", dropletBasePath, dropletID)
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(kernelsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Kernels, resp, err
+}
+
+// Actions lists the actions for a Droplet.
+func (s *DropletsServiceOp) Actions(ctx context.Context, dropletID int, opt *ListOptions) ([]Action, *Response, error) {
+	if dropletID < 1 {
+		return nil, nil, NewArgError("dropletID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d/actions", dropletBasePath, dropletID)
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Actions, resp, err
+}
+
+// Backups lists the backups for a Droplet.
+func (s *DropletsServiceOp) Backups(ctx context.Context, dropletID int, opt *ListOptions) ([]Image, *Response, error) {
+	if dropletID < 1 {
+		return nil, nil, NewArgError("dropletID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d/backups", dropletBasePath, dropletID)
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(backupsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Backups, resp, err
+}
+
+// Snapshots lists the snapshots available for a Droplet.
+func (s *DropletsServiceOp) Snapshots(ctx context.Context, dropletID int, opt *ListOptions) ([]Image, *Response, error) {
+	if dropletID < 1 {
+		return nil, nil, NewArgError("dropletID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d/snapshots", dropletBasePath, dropletID)
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(dropletSnapshotsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Snapshots, resp, err
+}
+
+// Neighbors lists the neighbors for a Droplet.
+func (s *DropletsServiceOp) Neighbors(ctx context.Context, dropletID int) ([]Droplet, *Response, error) {
+	if dropletID < 1 {
+		return nil, nil, NewArgError("dropletID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d/neighbors", dropletBasePath, dropletID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(dropletsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Droplets, resp, err
+}
+
+func (s *DropletsServiceOp) dropletActionStatus(ctx context.Context, uri string) (string, error) {
+	action, _, err := s.client.DropletActions.GetByURI(ctx, uri)
+
+	if err != nil {
+		return "", err
+	}
+
+	return action.Status, nil
+}

vendor/github.com/digitalocean/godo/errors.go

@@ -0,0 +1,24 @@
+package godo
+
+import "fmt"
+
+// ArgError is an error that represents an error with an input to godo. It
+// identifies the argument and the cause (if possible).
+type ArgError struct {
+	arg    string
+	reason string
+}
+
+var _ error = &ArgError{}
+
+// NewArgError creates an InputError.
+func NewArgError(arg, reason string) *ArgError {
+	return &ArgError{
+		arg:    arg,
+		reason: reason,
+	}
+}
+
+func (e *ArgError) Error() string {
+	return fmt.Sprintf("%s is invalid because %s", e.arg, e.reason)
+}

vendor/github.com/digitalocean/godo/firewalls.go

@@ -0,0 +1,271 @@
+package godo
+
+import (
+	"context"
+	"net/http"
+	"path"
+	"strconv"
+)
+
+const firewallsBasePath = "/v2/firewalls"
+
+// FirewallsService is an interface for managing Firewalls with the DigitalOcean API.
+// See: https://developers.digitalocean.com/documentation/v2/#firewalls
+type FirewallsService interface {
+	Get(context.Context, string) (*Firewall, *Response, error)
+	Create(context.Context, *FirewallRequest) (*Firewall, *Response, error)
+	Update(context.Context, string, *FirewallRequest) (*Firewall, *Response, error)
+	Delete(context.Context, string) (*Response, error)
+	List(context.Context, *ListOptions) ([]Firewall, *Response, error)
+	ListByDroplet(context.Context, int, *ListOptions) ([]Firewall, *Response, error)
+	AddDroplets(context.Context, string, ...int) (*Response, error)
+	RemoveDroplets(context.Context, string, ...int) (*Response, error)
+	AddTags(context.Context, string, ...string) (*Response, error)
+	RemoveTags(context.Context, string, ...string) (*Response, error)
+	AddRules(context.Context, string, *FirewallRulesRequest) (*Response, error)
+	RemoveRules(context.Context, string, *FirewallRulesRequest) (*Response, error)
+}
+
+// FirewallsServiceOp handles communication with Firewalls methods of the DigitalOcean API.
+type FirewallsServiceOp struct {
+	client *Client
+}
+
+// Firewall represents a DigitalOcean Firewall configuration.
+type Firewall struct {
+	ID             string          `json:"id"`
+	Name           string          `json:"name"`
+	Status         string          `json:"status"`
+	InboundRules   []InboundRule   `json:"inbound_rules"`
+	OutboundRules  []OutboundRule  `json:"outbound_rules"`
+	DropletIDs     []int           `json:"droplet_ids"`
+	Tags           []string        `json:"tags"`
+	Created        string          `json:"created_at"`
+	PendingChanges []PendingChange `json:"pending_changes"`
+}
+
+// String creates a human-readable description of a Firewall.
+func (fw Firewall) String() string {
+	return Stringify(fw)
+}
+
+func (fw Firewall) URN() string {
+	return ToURN("Firewall", fw.ID)
+}
+
+// FirewallRequest represents the configuration to be applied to an existing or a new Firewall.
+type FirewallRequest struct {
+	Name          string         `json:"name"`
+	InboundRules  []InboundRule  `json:"inbound_rules"`
+	OutboundRules []OutboundRule `json:"outbound_rules"`
+	DropletIDs    []int          `json:"droplet_ids"`
+	Tags          []string       `json:"tags"`
+}
+
+// FirewallRulesRequest represents rules configuration to be applied to an existing Firewall.
+type FirewallRulesRequest struct {
+	InboundRules  []InboundRule  `json:"inbound_rules"`
+	OutboundRules []OutboundRule `json:"outbound_rules"`
+}
+
+// InboundRule represents a DigitalOcean Firewall inbound rule.
+type InboundRule struct {
+	Protocol  string   `json:"protocol,omitempty"`
+	PortRange string   `json:"ports,omitempty"`
+	Sources   *Sources `json:"sources"`
+}
+
+// OutboundRule represents a DigitalOcean Firewall outbound rule.
+type OutboundRule struct {
+	Protocol     string        `json:"protocol,omitempty"`
+	PortRange    string        `json:"ports,omitempty"`
+	Destinations *Destinations `json:"destinations"`
+}
+
+// Sources represents a DigitalOcean Firewall InboundRule sources.
+type Sources struct {
+	Addresses        []string `json:"addresses,omitempty"`
+	Tags             []string `json:"tags,omitempty"`
+	DropletIDs       []int    `json:"droplet_ids,omitempty"`
+	LoadBalancerUIDs []string `json:"load_balancer_uids,omitempty"`
+}
+
+// PendingChange represents a DigitalOcean Firewall status details.
+type PendingChange struct {
+	DropletID int    `json:"droplet_id,omitempty"`
+	Removing  bool   `json:"removing,omitempty"`
+	Status    string `json:"status,omitempty"`
+}
+
+// Destinations represents a DigitalOcean Firewall OutboundRule destinations.
+type Destinations struct {
+	Addresses        []string `json:"addresses,omitempty"`
+	Tags             []string `json:"tags,omitempty"`
+	DropletIDs       []int    `json:"droplet_ids,omitempty"`
+	LoadBalancerUIDs []string `json:"load_balancer_uids,omitempty"`
+}
+
+var _ FirewallsService = &FirewallsServiceOp{}
+
+// Get an existing Firewall by its identifier.
+func (fw *FirewallsServiceOp) Get(ctx context.Context, fID string) (*Firewall, *Response, error) {
+	path := path.Join(firewallsBasePath, fID)
+
+	req, err := fw.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(firewallRoot)
+	resp, err := fw.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Firewall, resp, err
+}
+
+// Create a new Firewall with a given configuration.
+func (fw *FirewallsServiceOp) Create(ctx context.Context, fr *FirewallRequest) (*Firewall, *Response, error) {
+	req, err := fw.client.NewRequest(ctx, http.MethodPost, firewallsBasePath, fr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(firewallRoot)
+	resp, err := fw.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Firewall, resp, err
+}
+
+// Update an existing Firewall with new configuration.
+func (fw *FirewallsServiceOp) Update(ctx context.Context, fID string, fr *FirewallRequest) (*Firewall, *Response, error) {
+	path := path.Join(firewallsBasePath, fID)
+
+	req, err := fw.client.NewRequest(ctx, "PUT", path, fr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(firewallRoot)
+	resp, err := fw.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Firewall, resp, err
+}
+
+// Delete a Firewall by its identifier.
+func (fw *FirewallsServiceOp) Delete(ctx context.Context, fID string) (*Response, error) {
+	path := path.Join(firewallsBasePath, fID)
+	return fw.createAndDoReq(ctx, http.MethodDelete, path, nil)
+}
+
+// List Firewalls.
+func (fw *FirewallsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Firewall, *Response, error) {
+	path, err := addOptions(firewallsBasePath, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return fw.listHelper(ctx, path)
+}
+
+// ListByDroplet Firewalls.
+func (fw *FirewallsServiceOp) ListByDroplet(ctx context.Context, dID int, opt *ListOptions) ([]Firewall, *Response, error) {
+	basePath := path.Join(dropletBasePath, strconv.Itoa(dID), "firewalls")
+	path, err := addOptions(basePath, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return fw.listHelper(ctx, path)
+}
+
+// AddDroplets to a Firewall.
+func (fw *FirewallsServiceOp) AddDroplets(ctx context.Context, fID string, dropletIDs ...int) (*Response, error) {
+	path := path.Join(firewallsBasePath, fID, "droplets")
+	return fw.createAndDoReq(ctx, http.MethodPost, path, &dropletsRequest{IDs: dropletIDs})
+}
+
+// RemoveDroplets from a Firewall.
+func (fw *FirewallsServiceOp) RemoveDroplets(ctx context.Context, fID string, dropletIDs ...int) (*Response, error) {
+	path := path.Join(firewallsBasePath, fID, "droplets")
+	return fw.createAndDoReq(ctx, http.MethodDelete, path, &dropletsRequest{IDs: dropletIDs})
+}
+
+// AddTags to a Firewall.
+func (fw *FirewallsServiceOp) AddTags(ctx context.Context, fID string, tags ...string) (*Response, error) {
+	path := path.Join(firewallsBasePath, fID, "tags")
+	return fw.createAndDoReq(ctx, http.MethodPost, path, &tagsRequest{Tags: tags})
+}
+
+// RemoveTags from a Firewall.
+func (fw *FirewallsServiceOp) RemoveTags(ctx context.Context, fID string, tags ...string) (*Response, error) {
+	path := path.Join(firewallsBasePath, fID, "tags")
+	return fw.createAndDoReq(ctx, http.MethodDelete, path, &tagsRequest{Tags: tags})
+}
+
+// AddRules to a Firewall.
+func (fw *FirewallsServiceOp) AddRules(ctx context.Context, fID string, rr *FirewallRulesRequest) (*Response, error) {
+	path := path.Join(firewallsBasePath, fID, "rules")
+	return fw.createAndDoReq(ctx, http.MethodPost, path, rr)
+}
+
+// RemoveRules from a Firewall.
+func (fw *FirewallsServiceOp) RemoveRules(ctx context.Context, fID string, rr *FirewallRulesRequest) (*Response, error) {
+	path := path.Join(firewallsBasePath, fID, "rules")
+	return fw.createAndDoReq(ctx, http.MethodDelete, path, rr)
+}
+
+type dropletsRequest struct {
+	IDs []int `json:"droplet_ids"`
+}
+
+type tagsRequest struct {
+	Tags []string `json:"tags"`
+}
+
+type firewallRoot struct {
+	Firewall *Firewall `json:"firewall"`
+}
+
+type firewallsRoot struct {
+	Firewalls []Firewall `json:"firewalls"`
+	Links     *Links     `json:"links"`
+	Meta      *Meta      `json:"meta"`
+}
+
+func (fw *FirewallsServiceOp) createAndDoReq(ctx context.Context, method, path string, v interface{}) (*Response, error) {
+	req, err := fw.client.NewRequest(ctx, method, path, v)
+	if err != nil {
+		return nil, err
+	}
+
+	return fw.client.Do(ctx, req, nil)
+}
+
+func (fw *FirewallsServiceOp) listHelper(ctx context.Context, path string) ([]Firewall, *Response, error) {
+	req, err := fw.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(firewallsRoot)
+	resp, err := fw.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Firewalls, resp, err
+}

vendor/github.com/digitalocean/godo/floating_ips.go

@@ -0,0 +1,143 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+const floatingBasePath = "v2/floating_ips"
+
+// FloatingIPsService is an interface for interfacing with the floating IPs
+// endpoints of the Digital Ocean API.
+// See: https://developers.digitalocean.com/documentation/v2#floating-ips
+type FloatingIPsService interface {
+	List(context.Context, *ListOptions) ([]FloatingIP, *Response, error)
+	Get(context.Context, string) (*FloatingIP, *Response, error)
+	Create(context.Context, *FloatingIPCreateRequest) (*FloatingIP, *Response, error)
+	Delete(context.Context, string) (*Response, error)
+}
+
+// FloatingIPsServiceOp handles communication with the floating IPs related methods of the
+// DigitalOcean API.
+type FloatingIPsServiceOp struct {
+	client *Client
+}
+
+var _ FloatingIPsService = &FloatingIPsServiceOp{}
+
+// FloatingIP represents a Digital Ocean floating IP.
+type FloatingIP struct {
+	Region  *Region  `json:"region"`
+	Droplet *Droplet `json:"droplet"`
+	IP      string   `json:"ip"`
+}
+
+func (f FloatingIP) String() string {
+	return Stringify(f)
+}
+
+func (f FloatingIP) URN() string {
+	return ToURN("FloatingIP", f.IP)
+}
+
+type floatingIPsRoot struct {
+	FloatingIPs []FloatingIP `json:"floating_ips"`
+	Links       *Links       `json:"links"`
+	Meta        *Meta        `json:"meta"`
+}
+
+type floatingIPRoot struct {
+	FloatingIP *FloatingIP `json:"floating_ip"`
+	Links      *Links      `json:"links,omitempty"`
+}
+
+// FloatingIPCreateRequest represents a request to create a floating IP.
+// If DropletID is not empty, the floating IP will be assigned to the
+// droplet.
+type FloatingIPCreateRequest struct {
+	Region    string `json:"region"`
+	DropletID int    `json:"droplet_id,omitempty"`
+}
+
+// List all floating IPs.
+func (f *FloatingIPsServiceOp) List(ctx context.Context, opt *ListOptions) ([]FloatingIP, *Response, error) {
+	path := floatingBasePath
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := f.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(floatingIPsRoot)
+	resp, err := f.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.FloatingIPs, resp, err
+}
+
+// Get an individual floating IP.
+func (f *FloatingIPsServiceOp) Get(ctx context.Context, ip string) (*FloatingIP, *Response, error) {
+	path := fmt.Sprintf("%s/%s", floatingBasePath, ip)
+
+	req, err := f.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(floatingIPRoot)
+	resp, err := f.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.FloatingIP, resp, err
+}
+
+// Create a floating IP. If the DropletID field of the request is not empty,
+// the floating IP will also be assigned to the droplet.
+func (f *FloatingIPsServiceOp) Create(ctx context.Context, createRequest *FloatingIPCreateRequest) (*FloatingIP, *Response, error) {
+	path := floatingBasePath
+
+	req, err := f.client.NewRequest(ctx, http.MethodPost, path, createRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(floatingIPRoot)
+	resp, err := f.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+
+	return root.FloatingIP, resp, err
+}
+
+// Delete a floating IP.
+func (f *FloatingIPsServiceOp) Delete(ctx context.Context, ip string) (*Response, error) {
+	path := fmt.Sprintf("%s/%s", floatingBasePath, ip)
+
+	req, err := f.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := f.client.Do(ctx, req, nil)
+
+	return resp, err
+}

vendor/github.com/digitalocean/godo/floating_ips_actions.go

@@ -0,0 +1,109 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+// FloatingIPActionsService is an interface for interfacing with the
+// floating IPs actions endpoints of the Digital Ocean API.
+// See: https://developers.digitalocean.com/documentation/v2#floating-ips-action
+type FloatingIPActionsService interface {
+	Assign(ctx context.Context, ip string, dropletID int) (*Action, *Response, error)
+	Unassign(ctx context.Context, ip string) (*Action, *Response, error)
+	Get(ctx context.Context, ip string, actionID int) (*Action, *Response, error)
+	List(ctx context.Context, ip string, opt *ListOptions) ([]Action, *Response, error)
+}
+
+// FloatingIPActionsServiceOp handles communication with the floating IPs
+// action related methods of the DigitalOcean API.
+type FloatingIPActionsServiceOp struct {
+	client *Client
+}
+
+// Assign a floating IP to a droplet.
+func (s *FloatingIPActionsServiceOp) Assign(ctx context.Context, ip string, dropletID int) (*Action, *Response, error) {
+	request := &ActionRequest{
+		"type":       "assign",
+		"droplet_id": dropletID,
+	}
+	return s.doAction(ctx, ip, request)
+}
+
+// Unassign a floating IP from the droplet it is currently assigned to.
+func (s *FloatingIPActionsServiceOp) Unassign(ctx context.Context, ip string) (*Action, *Response, error) {
+	request := &ActionRequest{"type": "unassign"}
+	return s.doAction(ctx, ip, request)
+}
+
+// Get an action for a particular floating IP by id.
+func (s *FloatingIPActionsServiceOp) Get(ctx context.Context, ip string, actionID int) (*Action, *Response, error) {
+	path := fmt.Sprintf("%s/%d", floatingIPActionPath(ip), actionID)
+	return s.get(ctx, path)
+}
+
+// List the actions for a particular floating IP.
+func (s *FloatingIPActionsServiceOp) List(ctx context.Context, ip string, opt *ListOptions) ([]Action, *Response, error) {
+	path := floatingIPActionPath(ip)
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return s.list(ctx, path)
+}
+
+func (s *FloatingIPActionsServiceOp) doAction(ctx context.Context, ip string, request *ActionRequest) (*Action, *Response, error) {
+	path := floatingIPActionPath(ip)
+
+	req, err := s.client.NewRequest(ctx, http.MethodPost, path, request)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Event, resp, err
+}
+
+func (s *FloatingIPActionsServiceOp) get(ctx context.Context, path string) (*Action, *Response, error) {
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Event, resp, err
+}
+
+func (s *FloatingIPActionsServiceOp) list(ctx context.Context, path string) ([]Action, *Response, error) {
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+
+	return root.Actions, resp, err
+}
+
+func floatingIPActionPath(ip string) string {
+	return fmt.Sprintf("%s/%s/actions", floatingBasePath, ip)
+}

vendor/github.com/digitalocean/godo/go.mod

@@ -0,0 +1,16 @@
+module github.com/digitalocean/godo
+
+go 1.14
+
+require (
+	github.com/golang/protobuf v1.3.5 // indirect
+	github.com/google/go-querystring v1.0.0
+	github.com/stretchr/testify v1.4.0
+	golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
+	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
+	google.golang.org/appengine v1.6.5 // indirect
+)
+
+replace github.com/stretchr/objx => github.com/stretchr/objx v0.2.0
+
+replace golang.org/x/crypto => golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a

vendor/github.com/digitalocean/godo/go.sum

@@ -0,0 +1,42 @@
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+golang.org/x/crypto v0.0.0-20200420201142-3c4aac89819a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

vendor/github.com/digitalocean/godo/godo.go

@@ -0,0 +1,432 @@
+package godo
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"reflect"
+	"strconv"
+	"time"
+
+	"github.com/google/go-querystring/query"
+	"golang.org/x/oauth2"
+)
+
+const (
+	libraryVersion = "1.37.0"
+	defaultBaseURL = "https://api.digitalocean.com/"
+	userAgent      = "godo/" + libraryVersion
+	mediaType      = "application/json"
+
+	headerRateLimit     = "RateLimit-Limit"
+	headerRateRemaining = "RateLimit-Remaining"
+	headerRateReset     = "RateLimit-Reset"
+)
+
+// Client manages communication with DigitalOcean V2 API.
+type Client struct {
+	// HTTP client used to communicate with the DO API.
+	client *http.Client
+
+	// Base URL for API requests.
+	BaseURL *url.URL
+
+	// User agent for client
+	UserAgent string
+
+	// Rate contains the current rate limit for the client as determined by the most recent
+	// API call.
+	Rate Rate
+
+	// Services used for communicating with the API
+	Account           AccountService
+	Actions           ActionsService
+	Balance           BalanceService
+	BillingHistory    BillingHistoryService
+	CDNs              CDNService
+	Domains           DomainsService
+	Droplets          DropletsService
+	DropletActions    DropletActionsService
+	Images            ImagesService
+	ImageActions      ImageActionsService
+	Invoices          InvoicesService
+	Keys              KeysService
+	Regions           RegionsService
+	Sizes             SizesService
+	FloatingIPs       FloatingIPsService
+	FloatingIPActions FloatingIPActionsService
+	Snapshots         SnapshotsService
+	Storage           StorageService
+	StorageActions    StorageActionsService
+	Tags              TagsService
+	LoadBalancers     LoadBalancersService
+	Certificates      CertificatesService
+	Firewalls         FirewallsService
+	Projects          ProjectsService
+	Kubernetes        KubernetesService
+	Registry          RegistryService
+	Databases         DatabasesService
+	VPCs              VPCsService
+	OneClick          OneClickService
+
+	// Optional function called after every successful request made to the DO APIs
+	onRequestCompleted RequestCompletionCallback
+}
+
+// RequestCompletionCallback defines the type of the request callback function
+type RequestCompletionCallback func(*http.Request, *http.Response)
+
+// ListOptions specifies the optional parameters to various List methods that
+// support pagination.
+type ListOptions struct {
+	// For paginated result sets, page of results to retrieve.
+	Page int `url:"page,omitempty"`
+
+	// For paginated result sets, the number of results to include per page.
+	PerPage int `url:"per_page,omitempty"`
+}
+
+// Response is a DigitalOcean response. This wraps the standard http.Response returned from DigitalOcean.
+type Response struct {
+	*http.Response
+
+	// Links that were returned with the response. These are parsed from
+	// request body and not the header.
+	Links *Links
+
+	// Meta describes generic information about the response.
+	Meta *Meta
+
+	// Monitoring URI
+	Monitor string
+
+	Rate
+}
+
+// An ErrorResponse reports the error caused by an API request
+type ErrorResponse struct {
+	// HTTP response that caused this error
+	Response *http.Response
+
+	// Error message
+	Message string `json:"message"`
+
+	// RequestID returned from the API, useful to contact support.
+	RequestID string `json:"request_id"`
+}
+
+// Rate contains the rate limit for the current client.
+type Rate struct {
+	// The number of request per hour the client is currently limited to.
+	Limit int `json:"limit"`
+
+	// The number of remaining requests the client can make this hour.
+	Remaining int `json:"remaining"`
+
+	// The time at which the current rate limit will reset.
+	Reset Timestamp `json:"reset"`
+}
+
+func addOptions(s string, opt interface{}) (string, error) {
+	v := reflect.ValueOf(opt)
+
+	if v.Kind() == reflect.Ptr && v.IsNil() {
+		return s, nil
+	}
+
+	origURL, err := url.Parse(s)
+	if err != nil {
+		return s, err
+	}
+
+	origValues := origURL.Query()
+
+	newValues, err := query.Values(opt)
+	if err != nil {
+		return s, err
+	}
+
+	for k, v := range newValues {
+		origValues[k] = v
+	}
+
+	origURL.RawQuery = origValues.Encode()
+	return origURL.String(), nil
+}
+
+// NewFromToken returns a new DigitalOcean API client with the given API
+// token.
+func NewFromToken(token string) *Client {
+	ctx := context.Background()
+
+	config := &oauth2.Config{}
+	ts := config.TokenSource(ctx, &oauth2.Token{AccessToken: token})
+
+	return NewClient(oauth2.NewClient(ctx, ts))
+}
+
+// NewClient returns a new DigitalOcean API client, using the given
+// http.Client to perform all requests.
+//
+// Users who wish to pass their own http.Client should use this method. If
+// you're in need of further customization, the godo.New method allows more
+// options, such as setting a custom URL or a custom user agent string.
+func NewClient(httpClient *http.Client) *Client {
+	if httpClient == nil {
+		httpClient = http.DefaultClient
+	}
+
+	baseURL, _ := url.Parse(defaultBaseURL)
+
+	c := &Client{client: httpClient, BaseURL: baseURL, UserAgent: userAgent}
+	c.Account = &AccountServiceOp{client: c}
+	c.Actions = &ActionsServiceOp{client: c}
+	c.Balance = &BalanceServiceOp{client: c}
+	c.BillingHistory = &BillingHistoryServiceOp{client: c}
+	c.CDNs = &CDNServiceOp{client: c}
+	c.Certificates = &CertificatesServiceOp{client: c}
+	c.Domains = &DomainsServiceOp{client: c}
+	c.Droplets = &DropletsServiceOp{client: c}
+	c.DropletActions = &DropletActionsServiceOp{client: c}
+	c.Firewalls = &FirewallsServiceOp{client: c}
+	c.FloatingIPs = &FloatingIPsServiceOp{client: c}
+	c.FloatingIPActions = &FloatingIPActionsServiceOp{client: c}
+	c.Images = &ImagesServiceOp{client: c}
+	c.ImageActions = &ImageActionsServiceOp{client: c}
+	c.Invoices = &InvoicesServiceOp{client: c}
+	c.Keys = &KeysServiceOp{client: c}
+	c.LoadBalancers = &LoadBalancersServiceOp{client: c}
+	c.Projects = &ProjectsServiceOp{client: c}
+	c.Regions = &RegionsServiceOp{client: c}
+	c.Sizes = &SizesServiceOp{client: c}
+	c.Snapshots = &SnapshotsServiceOp{client: c}
+	c.Storage = &StorageServiceOp{client: c}
+	c.StorageActions = &StorageActionsServiceOp{client: c}
+	c.Tags = &TagsServiceOp{client: c}
+	c.Kubernetes = &KubernetesServiceOp{client: c}
+	c.Registry = &RegistryServiceOp{client: c}
+	c.Databases = &DatabasesServiceOp{client: c}
+	c.VPCs = &VPCsServiceOp{client: c}
+	c.OneClick = &OneClickServiceOp{client: c}
+
+	return c
+}
+
+// ClientOpt are options for New.
+type ClientOpt func(*Client) error
+
+// New returns a new DigitalOcean API client instance.
+func New(httpClient *http.Client, opts ...ClientOpt) (*Client, error) {
+	c := NewClient(httpClient)
+	for _, opt := range opts {
+		if err := opt(c); err != nil {
+			return nil, err
+		}
+	}
+
+	return c, nil
+}
+
+// SetBaseURL is a client option for setting the base URL.
+func SetBaseURL(bu string) ClientOpt {
+	return func(c *Client) error {
+		u, err := url.Parse(bu)
+		if err != nil {
+			return err
+		}
+
+		c.BaseURL = u
+		return nil
+	}
+}
+
+// SetUserAgent is a client option for setting the user agent.
+func SetUserAgent(ua string) ClientOpt {
+	return func(c *Client) error {
+		c.UserAgent = fmt.Sprintf("%s %s", ua, c.UserAgent)
+		return nil
+	}
+}
+
+// NewRequest creates an API request. A relative URL can be provided in urlStr, which will be resolved to the
+// BaseURL of the Client. Relative URLS should always be specified without a preceding slash. If specified, the
+// value pointed to by body is JSON encoded and included in as the request body.
+func (c *Client) NewRequest(ctx context.Context, method, urlStr string, body interface{}) (*http.Request, error) {
+	u, err := c.BaseURL.Parse(urlStr)
+	if err != nil {
+		return nil, err
+	}
+
+	buf := new(bytes.Buffer)
+	if body != nil {
+		err = json.NewEncoder(buf).Encode(body)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	req, err := http.NewRequest(method, u.String(), buf)
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Add("Content-Type", mediaType)
+	req.Header.Add("Accept", mediaType)
+	req.Header.Add("User-Agent", c.UserAgent)
+	return req, nil
+}
+
+// OnRequestCompleted sets the DO API request completion callback
+func (c *Client) OnRequestCompleted(rc RequestCompletionCallback) {
+	c.onRequestCompleted = rc
+}
+
+// newResponse creates a new Response for the provided http.Response
+func newResponse(r *http.Response) *Response {
+	response := Response{Response: r}
+	response.populateRate()
+
+	return &response
+}
+
+// populateRate parses the rate related headers and populates the response Rate.
+func (r *Response) populateRate() {
+	if limit := r.Header.Get(headerRateLimit); limit != "" {
+		r.Rate.Limit, _ = strconv.Atoi(limit)
+	}
+	if remaining := r.Header.Get(headerRateRemaining); remaining != "" {
+		r.Rate.Remaining, _ = strconv.Atoi(remaining)
+	}
+	if reset := r.Header.Get(headerRateReset); reset != "" {
+		if v, _ := strconv.ParseInt(reset, 10, 64); v != 0 {
+			r.Rate.Reset = Timestamp{time.Unix(v, 0)}
+		}
+	}
+}
+
+// Do sends an API request and returns the API response. The API response is JSON decoded and stored in the value
+// pointed to by v, or returned as an error if an API error has occurred. If v implements the io.Writer interface,
+// the raw response will be written to v, without attempting to decode it.
+func (c *Client) Do(ctx context.Context, req *http.Request, v interface{}) (*Response, error) {
+	resp, err := DoRequestWithClient(ctx, c.client, req)
+	if err != nil {
+		return nil, err
+	}
+	if c.onRequestCompleted != nil {
+		c.onRequestCompleted(req, resp)
+	}
+
+	defer func() {
+		if rerr := resp.Body.Close(); err == nil {
+			err = rerr
+		}
+	}()
+
+	response := newResponse(resp)
+	c.Rate = response.Rate
+
+	err = CheckResponse(resp)
+	if err != nil {
+		return response, err
+	}
+
+	if v != nil {
+		if w, ok := v.(io.Writer); ok {
+			_, err = io.Copy(w, resp.Body)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			err = json.NewDecoder(resp.Body).Decode(v)
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	return response, err
+}
+
+// DoRequest submits an HTTP request.
+func DoRequest(ctx context.Context, req *http.Request) (*http.Response, error) {
+	return DoRequestWithClient(ctx, http.DefaultClient, req)
+}
+
+// DoRequestWithClient submits an HTTP request using the specified client.
+func DoRequestWithClient(
+	ctx context.Context,
+	client *http.Client,
+	req *http.Request) (*http.Response, error) {
+	req = req.WithContext(ctx)
+	return client.Do(req)
+}
+
+func (r *ErrorResponse) Error() string {
+	if r.RequestID != "" {
+		return fmt.Sprintf("%v %v: %d (request %q) %v",
+			r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.RequestID, r.Message)
+	}
+	return fmt.Sprintf("%v %v: %d %v",
+		r.Response.Request.Method, r.Response.Request.URL, r.Response.StatusCode, r.Message)
+}
+
+// CheckResponse checks the API response for errors, and returns them if present. A response is considered an
+// error if it has a status code outside the 200 range. API error responses are expected to have either no response
+// body, or a JSON response body that maps to ErrorResponse. Any other response body will be silently ignored.
+func CheckResponse(r *http.Response) error {
+	if c := r.StatusCode; c >= 200 && c <= 299 {
+		return nil
+	}
+
+	errorResponse := &ErrorResponse{Response: r}
+	data, err := ioutil.ReadAll(r.Body)
+	if err == nil && len(data) > 0 {
+		err := json.Unmarshal(data, errorResponse)
+		if err != nil {
+			errorResponse.Message = string(data)
+		}
+	}
+
+	return errorResponse
+}
+
+func (r Rate) String() string {
+	return Stringify(r)
+}
+
+// String is a helper routine that allocates a new string value
+// to store v and returns a pointer to it.
+func String(v string) *string {
+	p := new(string)
+	*p = v
+	return p
+}
+
+// Int is a helper routine that allocates a new int32 value
+// to store v and returns a pointer to it, but unlike Int32
+// its argument value is an int.
+func Int(v int) *int {
+	p := new(int)
+	*p = v
+	return p
+}
+
+// Bool is a helper routine that allocates a new bool value
+// to store v and returns a pointer to it.
+func Bool(v bool) *bool {
+	p := new(bool)
+	*p = v
+	return p
+}
+
+// StreamToString converts a reader to a string
+func StreamToString(stream io.Reader) string {
+	buf := new(bytes.Buffer)
+	_, _ = buf.ReadFrom(stream)
+	return buf.String()
+}

vendor/github.com/digitalocean/godo/image_actions.go

@@ -0,0 +1,102 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+// ImageActionsService is an interface for interfacing with the image actions
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2#image-actions
+type ImageActionsService interface {
+	Get(context.Context, int, int) (*Action, *Response, error)
+	Transfer(context.Context, int, *ActionRequest) (*Action, *Response, error)
+	Convert(context.Context, int) (*Action, *Response, error)
+}
+
+// ImageActionsServiceOp handles communition with the image action related methods of the
+// DigitalOcean API.
+type ImageActionsServiceOp struct {
+	client *Client
+}
+
+var _ ImageActionsService = &ImageActionsServiceOp{}
+
+// Transfer an image
+func (i *ImageActionsServiceOp) Transfer(ctx context.Context, imageID int, transferRequest *ActionRequest) (*Action, *Response, error) {
+	if imageID < 1 {
+		return nil, nil, NewArgError("imageID", "cannot be less than 1")
+	}
+
+	if transferRequest == nil {
+		return nil, nil, NewArgError("transferRequest", "cannot be nil")
+	}
+
+	path := fmt.Sprintf("v2/images/%d/actions", imageID)
+
+	req, err := i.client.NewRequest(ctx, http.MethodPost, path, transferRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionRoot)
+	resp, err := i.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Event, resp, err
+}
+
+// Convert an image to a snapshot
+func (i *ImageActionsServiceOp) Convert(ctx context.Context, imageID int) (*Action, *Response, error) {
+	if imageID < 1 {
+		return nil, nil, NewArgError("imageID", "cannont be less than 1")
+	}
+
+	path := fmt.Sprintf("v2/images/%d/actions", imageID)
+
+	convertRequest := &ActionRequest{
+		"type": "convert",
+	}
+
+	req, err := i.client.NewRequest(ctx, http.MethodPost, path, convertRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionRoot)
+	resp, err := i.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Event, resp, err
+}
+
+// Get an action for a particular image by id.
+func (i *ImageActionsServiceOp) Get(ctx context.Context, imageID, actionID int) (*Action, *Response, error) {
+	if imageID < 1 {
+		return nil, nil, NewArgError("imageID", "cannot be less than 1")
+	}
+
+	if actionID < 1 {
+		return nil, nil, NewArgError("actionID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("v2/images/%d/actions/%d", imageID, actionID)
+
+	req, err := i.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(actionRoot)
+	resp, err := i.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Event, resp, err
+}

vendor/github.com/digitalocean/godo/images.go

@@ -0,0 +1,245 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+const imageBasePath = "v2/images"
+
+// ImagesService is an interface for interfacing with the images
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2#images
+type ImagesService interface {
+	List(context.Context, *ListOptions) ([]Image, *Response, error)
+	ListDistribution(ctx context.Context, opt *ListOptions) ([]Image, *Response, error)
+	ListApplication(ctx context.Context, opt *ListOptions) ([]Image, *Response, error)
+	ListUser(ctx context.Context, opt *ListOptions) ([]Image, *Response, error)
+	ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Image, *Response, error)
+	GetByID(context.Context, int) (*Image, *Response, error)
+	GetBySlug(context.Context, string) (*Image, *Response, error)
+	Create(context.Context, *CustomImageCreateRequest) (*Image, *Response, error)
+	Update(context.Context, int, *ImageUpdateRequest) (*Image, *Response, error)
+	Delete(context.Context, int) (*Response, error)
+}
+
+// ImagesServiceOp handles communication with the image related methods of the
+// DigitalOcean API.
+type ImagesServiceOp struct {
+	client *Client
+}
+
+var _ ImagesService = &ImagesServiceOp{}
+
+// Image represents a DigitalOcean Image
+type Image struct {
+	ID            int      `json:"id,float64,omitempty"`
+	Name          string   `json:"name,omitempty"`
+	Type          string   `json:"type,omitempty"`
+	Distribution  string   `json:"distribution,omitempty"`
+	Slug          string   `json:"slug,omitempty"`
+	Public        bool     `json:"public,omitempty"`
+	Regions       []string `json:"regions,omitempty"`
+	MinDiskSize   int      `json:"min_disk_size,omitempty"`
+	SizeGigaBytes float64  `json:"size_gigabytes,omitempty"`
+	Created       string   `json:"created_at,omitempty"`
+	Description   string   `json:"description,omitempty"`
+	Tags          []string `json:"tags,omitempty"`
+	Status        string   `json:"status,omitempty"`
+	ErrorMessage  string   `json:"error_message,omitempty"`
+}
+
+// ImageUpdateRequest represents a request to update an image.
+type ImageUpdateRequest struct {
+	Name string `json:"name"`
+}
+
+// CustomImageCreateRequest represents a request to create a custom image.
+type CustomImageCreateRequest struct {
+	Name         string   `json:"name"`
+	Url          string   `json:"url"`
+	Region       string   `json:"region"`
+	Distribution string   `json:"distribution,omitempty"`
+	Description  string   `json:"description,omitempty"`
+	Tags         []string `json:"tags,omitempty"`
+}
+
+type imageRoot struct {
+	Image *Image
+}
+
+type imagesRoot struct {
+	Images []Image
+	Links  *Links `json:"links"`
+	Meta   *Meta  `json:"meta"`
+}
+
+type listImageOptions struct {
+	Private bool   `url:"private,omitempty"`
+	Type    string `url:"type,omitempty"`
+	Tag     string `url:"tag_name,omitempty"`
+}
+
+func (i Image) String() string {
+	return Stringify(i)
+}
+
+// List lists all the images available.
+func (s *ImagesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) {
+	return s.list(ctx, opt, nil)
+}
+
+// ListDistribution lists all the distribution images.
+func (s *ImagesServiceOp) ListDistribution(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) {
+	listOpt := listImageOptions{Type: "distribution"}
+	return s.list(ctx, opt, &listOpt)
+}
+
+// ListApplication lists all the application images.
+func (s *ImagesServiceOp) ListApplication(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) {
+	listOpt := listImageOptions{Type: "application"}
+	return s.list(ctx, opt, &listOpt)
+}
+
+// ListUser lists all the user images.
+func (s *ImagesServiceOp) ListUser(ctx context.Context, opt *ListOptions) ([]Image, *Response, error) {
+	listOpt := listImageOptions{Private: true}
+	return s.list(ctx, opt, &listOpt)
+}
+
+// ListByTag lists all images with a specific tag applied.
+func (s *ImagesServiceOp) ListByTag(ctx context.Context, tag string, opt *ListOptions) ([]Image, *Response, error) {
+	listOpt := listImageOptions{Tag: tag}
+	return s.list(ctx, opt, &listOpt)
+}
+
+// GetByID retrieves an image by id.
+func (s *ImagesServiceOp) GetByID(ctx context.Context, imageID int) (*Image, *Response, error) {
+	if imageID < 1 {
+		return nil, nil, NewArgError("imageID", "cannot be less than 1")
+	}
+
+	return s.get(ctx, interface{}(imageID))
+}
+
+// GetBySlug retrieves an image by slug.
+func (s *ImagesServiceOp) GetBySlug(ctx context.Context, slug string) (*Image, *Response, error) {
+	if len(slug) < 1 {
+		return nil, nil, NewArgError("slug", "cannot be blank")
+	}
+
+	return s.get(ctx, interface{}(slug))
+}
+
+func (s *ImagesServiceOp) Create(ctx context.Context, createRequest *CustomImageCreateRequest) (*Image, *Response, error) {
+	if createRequest == nil {
+		return nil, nil, NewArgError("createRequest", "cannot be nil")
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodPost, imageBasePath, createRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(imageRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Image, resp, err
+}
+
+// Update an image name.
+func (s *ImagesServiceOp) Update(ctx context.Context, imageID int, updateRequest *ImageUpdateRequest) (*Image, *Response, error) {
+	if imageID < 1 {
+		return nil, nil, NewArgError("imageID", "cannot be less than 1")
+	}
+
+	if updateRequest == nil {
+		return nil, nil, NewArgError("updateRequest", "cannot be nil")
+	}
+
+	path := fmt.Sprintf("%s/%d", imageBasePath, imageID)
+	req, err := s.client.NewRequest(ctx, http.MethodPut, path, updateRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(imageRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Image, resp, err
+}
+
+// Delete an image.
+func (s *ImagesServiceOp) Delete(ctx context.Context, imageID int) (*Response, error) {
+	if imageID < 1 {
+		return nil, NewArgError("imageID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d", imageBasePath, imageID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := s.client.Do(ctx, req, nil)
+
+	return resp, err
+}
+
+// Helper method for getting an individual image
+func (s *ImagesServiceOp) get(ctx context.Context, ID interface{}) (*Image, *Response, error) {
+	path := fmt.Sprintf("%s/%v", imageBasePath, ID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(imageRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Image, resp, err
+}
+
+// Helper method for listing images
+func (s *ImagesServiceOp) list(ctx context.Context, opt *ListOptions, listOpt *listImageOptions) ([]Image, *Response, error) {
+	path := imageBasePath
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+	path, err = addOptions(path, listOpt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(imagesRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Images, resp, err
+}

vendor/github.com/digitalocean/godo/invoices.go

@@ -0,0 +1,225 @@
+package godo
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+)
+
+const invoicesBasePath = "v2/customers/my/invoices"
+
+// InvoicesService is an interface for interfacing with the Invoice
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2/#invoices
+type InvoicesService interface {
+	Get(context.Context, string, *ListOptions) (*Invoice, *Response, error)
+	GetPDF(context.Context, string) ([]byte, *Response, error)
+	GetCSV(context.Context, string) ([]byte, *Response, error)
+	List(context.Context, *ListOptions) (*InvoiceList, *Response, error)
+	GetSummary(context.Context, string) (*InvoiceSummary, *Response, error)
+}
+
+// InvoicesServiceOp handles communication with the Invoice related methods of
+// the DigitalOcean API.
+type InvoicesServiceOp struct {
+	client *Client
+}
+
+var _ InvoicesService = &InvoicesServiceOp{}
+
+// Invoice represents a DigitalOcean Invoice
+type Invoice struct {
+	InvoiceItems []InvoiceItem `json:"invoice_items"`
+	Links        *Links        `json:"links"`
+	Meta         *Meta         `json:"meta"`
+}
+
+// InvoiceItem represents a line-item on a DigitalOcean Invoice
+type InvoiceItem struct {
+	Product          string    `json:"product"`
+	ResourceID       string    `json:"resource_id"`
+	ResourceUUID     string    `json:"resource_uuid"`
+	GroupDescription string    `json:"group_description"`
+	Description      string    `json:"description"`
+	Amount           string    `json:"amount"`
+	Duration         string    `json:"duration"`
+	DurationUnit     string    `json:"duration_unit"`
+	StartTime        time.Time `json:"start_time"`
+	EndTime          time.Time `json:"end_time"`
+	ProjectName      string    `json:"project_name"`
+}
+
+// InvoiceList contains a paginated list of all of a customer's invoices.
+// The InvoicePreview is the month-to-date usage generated by DigitalOcean.
+type InvoiceList struct {
+	Invoices       []InvoiceListItem `json:"invoices"`
+	InvoicePreview InvoiceListItem   `json:"invoice_preview"`
+	Links          *Links            `json:"links"`
+	Meta           *Meta             `json:"meta"`
+}
+
+// InvoiceListItem contains a small list of information about a customer's invoice.
+// More information can be found in the Invoice or InvoiceSummary
+type InvoiceListItem struct {
+	InvoiceUUID   string    `json:"invoice_uuid"`
+	Amount        string    `json:"amount"`
+	InvoicePeriod string    `json:"invoice_period"`
+	UpdatedAt     time.Time `json:"updated_at"`
+}
+
+// InvoiceSummary contains metadata and summarized usage for an invoice generated by DigitalOcean
+type InvoiceSummary struct {
+	InvoiceUUID           string                  `json:"invoice_uuid"`
+	BillingPeriod         string                  `json:"billing_period"`
+	Amount                string                  `json:"amount"`
+	UserName              string                  `json:"user_name"`
+	UserBillingAddress    Address                 `json:"user_billing_address"`
+	UserCompany           string                  `json:"user_company"`
+	UserEmail             string                  `json:"user_email"`
+	ProductCharges        InvoiceSummaryBreakdown `json:"product_charges"`
+	Overages              InvoiceSummaryBreakdown `json:"overages"`
+	Taxes                 InvoiceSummaryBreakdown `json:"taxes"`
+	CreditsAndAdjustments InvoiceSummaryBreakdown `json:"credits_and_adjustments"`
+}
+
+// Address represents the billing address of a customer
+type Address struct {
+	AddressLine1    string    `json:"address_line1"`
+	AddressLine2    string    `json:"address_line2"`
+	City            string    `json:"city"`
+	Region          string    `json:"region"`
+	PostalCode      string    `json:"postal_code"`
+	CountryISO2Code string    `json:"country_iso2_code"`
+	CreatedAt       time.Time `json:"created_at"`
+	UpdatedAt       time.Time `json:"updated_at"`
+}
+
+// InvoiceSummaryBreakdown is a grouped set of InvoiceItems from an invoice
+type InvoiceSummaryBreakdown struct {
+	Name   string                        `json:"name"`
+	Amount string                        `json:"amount"`
+	Items  []InvoiceSummaryBreakdownItem `json:"items"`
+}
+
+// InvoiceSummaryBreakdownItem further breaks down the InvoiceSummary by product
+type InvoiceSummaryBreakdownItem struct {
+	Name   string `json:"name"`
+	Amount string `json:"amount"`
+	Count  string `json:"count"`
+}
+
+func (i Invoice) String() string {
+	return Stringify(i)
+}
+
+// Get detailed invoice items for an Invoice
+func (s *InvoicesServiceOp) Get(ctx context.Context, invoiceUUID string, opt *ListOptions) (*Invoice, *Response, error) {
+	path := fmt.Sprintf("%s/%s", invoicesBasePath, invoiceUUID)
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(Invoice)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root, resp, err
+}
+
+// List invoices for a customer
+func (s *InvoicesServiceOp) List(ctx context.Context, opt *ListOptions) (*InvoiceList, *Response, error) {
+	path := invoicesBasePath
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(InvoiceList)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root, resp, err
+}
+
+// Get a summary of metadata and summarized usage for an Invoice
+func (s *InvoicesServiceOp) GetSummary(ctx context.Context, invoiceUUID string) (*InvoiceSummary, *Response, error) {
+	path := fmt.Sprintf("%s/%s/summary", invoicesBasePath, invoiceUUID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(InvoiceSummary)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root, resp, err
+}
+
+// Get the pdf for an Invoice
+func (s *InvoicesServiceOp) GetPDF(ctx context.Context, invoiceUUID string) ([]byte, *Response, error) {
+	path := fmt.Sprintf("%s/%s/pdf", invoicesBasePath, invoiceUUID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var root bytes.Buffer
+	resp, err := s.client.Do(ctx, req, &root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Bytes(), resp, err
+}
+
+// Get the csv for an Invoice
+func (s *InvoicesServiceOp) GetCSV(ctx context.Context, invoiceUUID string) ([]byte, *Response, error) {
+	path := fmt.Sprintf("%s/%s/csv", invoicesBasePath, invoiceUUID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var root bytes.Buffer
+	resp, err := s.client.Do(ctx, req, &root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Bytes(), resp, err
+}

vendor/github.com/digitalocean/godo/keys.go

@@ -0,0 +1,230 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+const keysBasePath = "v2/account/keys"
+
+// KeysService is an interface for interfacing with the keys
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2#keys
+type KeysService interface {
+	List(context.Context, *ListOptions) ([]Key, *Response, error)
+	GetByID(context.Context, int) (*Key, *Response, error)
+	GetByFingerprint(context.Context, string) (*Key, *Response, error)
+	Create(context.Context, *KeyCreateRequest) (*Key, *Response, error)
+	UpdateByID(context.Context, int, *KeyUpdateRequest) (*Key, *Response, error)
+	UpdateByFingerprint(context.Context, string, *KeyUpdateRequest) (*Key, *Response, error)
+	DeleteByID(context.Context, int) (*Response, error)
+	DeleteByFingerprint(context.Context, string) (*Response, error)
+}
+
+// KeysServiceOp handles communication with key related method of the
+// DigitalOcean API.
+type KeysServiceOp struct {
+	client *Client
+}
+
+var _ KeysService = &KeysServiceOp{}
+
+// Key represents a DigitalOcean Key.
+type Key struct {
+	ID          int    `json:"id,float64,omitempty"`
+	Name        string `json:"name,omitempty"`
+	Fingerprint string `json:"fingerprint,omitempty"`
+	PublicKey   string `json:"public_key,omitempty"`
+}
+
+// KeyUpdateRequest represents a request to update a DigitalOcean key.
+type KeyUpdateRequest struct {
+	Name string `json:"name"`
+}
+
+type keysRoot struct {
+	SSHKeys []Key  `json:"ssh_keys"`
+	Links   *Links `json:"links"`
+	Meta    *Meta  `json:"meta"`
+}
+
+type keyRoot struct {
+	SSHKey *Key `json:"ssh_key"`
+}
+
+func (s Key) String() string {
+	return Stringify(s)
+}
+
+// KeyCreateRequest represents a request to create a new key.
+type KeyCreateRequest struct {
+	Name      string `json:"name"`
+	PublicKey string `json:"public_key"`
+}
+
+// List all keys
+func (s *KeysServiceOp) List(ctx context.Context, opt *ListOptions) ([]Key, *Response, error) {
+	path := keysBasePath
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(keysRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.SSHKeys, resp, err
+}
+
+// Performs a get given a path
+func (s *KeysServiceOp) get(ctx context.Context, path string) (*Key, *Response, error) {
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(keyRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.SSHKey, resp, err
+}
+
+// GetByID gets a Key by id
+func (s *KeysServiceOp) GetByID(ctx context.Context, keyID int) (*Key, *Response, error) {
+	if keyID < 1 {
+		return nil, nil, NewArgError("keyID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d", keysBasePath, keyID)
+	return s.get(ctx, path)
+}
+
+// GetByFingerprint gets a Key by by fingerprint
+func (s *KeysServiceOp) GetByFingerprint(ctx context.Context, fingerprint string) (*Key, *Response, error) {
+	if len(fingerprint) < 1 {
+		return nil, nil, NewArgError("fingerprint", "cannot not be empty")
+	}
+
+	path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint)
+	return s.get(ctx, path)
+}
+
+// Create a key using a KeyCreateRequest
+func (s *KeysServiceOp) Create(ctx context.Context, createRequest *KeyCreateRequest) (*Key, *Response, error) {
+	if createRequest == nil {
+		return nil, nil, NewArgError("createRequest", "cannot be nil")
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodPost, keysBasePath, createRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(keyRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.SSHKey, resp, err
+}
+
+// UpdateByID updates a key name by ID.
+func (s *KeysServiceOp) UpdateByID(ctx context.Context, keyID int, updateRequest *KeyUpdateRequest) (*Key, *Response, error) {
+	if keyID < 1 {
+		return nil, nil, NewArgError("keyID", "cannot be less than 1")
+	}
+
+	if updateRequest == nil {
+		return nil, nil, NewArgError("updateRequest", "cannot be nil")
+	}
+
+	path := fmt.Sprintf("%s/%d", keysBasePath, keyID)
+	req, err := s.client.NewRequest(ctx, "PUT", path, updateRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(keyRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.SSHKey, resp, err
+}
+
+// UpdateByFingerprint updates a key name by fingerprint.
+func (s *KeysServiceOp) UpdateByFingerprint(ctx context.Context, fingerprint string, updateRequest *KeyUpdateRequest) (*Key, *Response, error) {
+	if len(fingerprint) < 1 {
+		return nil, nil, NewArgError("fingerprint", "cannot be empty")
+	}
+
+	if updateRequest == nil {
+		return nil, nil, NewArgError("updateRequest", "cannot be nil")
+	}
+
+	path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint)
+	req, err := s.client.NewRequest(ctx, "PUT", path, updateRequest)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(keyRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.SSHKey, resp, err
+}
+
+// Delete key using a path
+func (s *KeysServiceOp) delete(ctx context.Context, path string) (*Response, error) {
+	req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := s.client.Do(ctx, req, nil)
+
+	return resp, err
+}
+
+// DeleteByID deletes a key by its id
+func (s *KeysServiceOp) DeleteByID(ctx context.Context, keyID int) (*Response, error) {
+	if keyID < 1 {
+		return nil, NewArgError("keyID", "cannot be less than 1")
+	}
+
+	path := fmt.Sprintf("%s/%d", keysBasePath, keyID)
+	return s.delete(ctx, path)
+}
+
+// DeleteByFingerprint deletes a key by its fingerprint
+func (s *KeysServiceOp) DeleteByFingerprint(ctx context.Context, fingerprint string) (*Response, error) {
+	if len(fingerprint) < 1 {
+		return nil, NewArgError("fingerprint", "cannot be empty")
+	}
+
+	path := fmt.Sprintf("%s/%s", keysBasePath, fingerprint)
+	return s.delete(ctx, path)
+}

vendor/github.com/digitalocean/godo/kubernetes.go

@@ -0,0 +1,697 @@
+package godo
+
+import (
+	"bytes"
+	"context"
+	"encoding"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	kubernetesBasePath     = "/v2/kubernetes"
+	kubernetesClustersPath = kubernetesBasePath + "/clusters"
+	kubernetesOptionsPath  = kubernetesBasePath + "/options"
+)
+
+// KubernetesService is an interface for interfacing with the Kubernetes endpoints
+// of the DigitalOcean API.
+// See: https://developers.digitalocean.com/documentation/v2#kubernetes
+type KubernetesService interface {
+	Create(context.Context, *KubernetesClusterCreateRequest) (*KubernetesCluster, *Response, error)
+	Get(context.Context, string) (*KubernetesCluster, *Response, error)
+	GetUser(context.Context, string) (*KubernetesClusterUser, *Response, error)
+	GetUpgrades(context.Context, string) ([]*KubernetesVersion, *Response, error)
+	GetKubeConfig(context.Context, string) (*KubernetesClusterConfig, *Response, error)
+	GetCredentials(context.Context, string, *KubernetesClusterCredentialsGetRequest) (*KubernetesClusterCredentials, *Response, error)
+	List(context.Context, *ListOptions) ([]*KubernetesCluster, *Response, error)
+	Update(context.Context, string, *KubernetesClusterUpdateRequest) (*KubernetesCluster, *Response, error)
+	Upgrade(context.Context, string, *KubernetesClusterUpgradeRequest) (*Response, error)
+	Delete(context.Context, string) (*Response, error)
+
+	CreateNodePool(ctx context.Context, clusterID string, req *KubernetesNodePoolCreateRequest) (*KubernetesNodePool, *Response, error)
+	GetNodePool(ctx context.Context, clusterID, poolID string) (*KubernetesNodePool, *Response, error)
+	ListNodePools(ctx context.Context, clusterID string, opts *ListOptions) ([]*KubernetesNodePool, *Response, error)
+	UpdateNodePool(ctx context.Context, clusterID, poolID string, req *KubernetesNodePoolUpdateRequest) (*KubernetesNodePool, *Response, error)
+	// RecycleNodePoolNodes is DEPRECATED please use DeleteNode
+	// The method will be removed in godo 2.0.
+	RecycleNodePoolNodes(ctx context.Context, clusterID, poolID string, req *KubernetesNodePoolRecycleNodesRequest) (*Response, error)
+	DeleteNodePool(ctx context.Context, clusterID, poolID string) (*Response, error)
+	DeleteNode(ctx context.Context, clusterID, poolID, nodeID string, req *KubernetesNodeDeleteRequest) (*Response, error)
+
+	GetOptions(context.Context) (*KubernetesOptions, *Response, error)
+}
+
+var _ KubernetesService = &KubernetesServiceOp{}
+
+// KubernetesServiceOp handles communication with Kubernetes methods of the DigitalOcean API.
+type KubernetesServiceOp struct {
+	client *Client
+}
+
+// KubernetesClusterCreateRequest represents a request to create a Kubernetes cluster.
+type KubernetesClusterCreateRequest struct {
+	Name        string   `json:"name,omitempty"`
+	RegionSlug  string   `json:"region,omitempty"`
+	VersionSlug string   `json:"version,omitempty"`
+	Tags        []string `json:"tags,omitempty"`
+	VPCUUID     string   `json:"vpc_uuid,omitempty"`
+
+	NodePools []*KubernetesNodePoolCreateRequest `json:"node_pools,omitempty"`
+
+	MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy"`
+	AutoUpgrade       bool                         `json:"auto_upgrade"`
+}
+
+// KubernetesClusterUpdateRequest represents a request to update a Kubernetes cluster.
+type KubernetesClusterUpdateRequest struct {
+	Name              string                       `json:"name,omitempty"`
+	Tags              []string                     `json:"tags,omitempty"`
+	MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy,omitempty"`
+	AutoUpgrade       *bool                        `json:"auto_upgrade,omitempty"`
+}
+
+// KubernetesClusterUpgradeRequest represents a request to upgrade a Kubernetes cluster.
+type KubernetesClusterUpgradeRequest struct {
+	VersionSlug string `json:"version,omitempty"`
+}
+
+// KubernetesNodePoolCreateRequest represents a request to create a node pool for a
+// Kubernetes cluster.
+type KubernetesNodePoolCreateRequest struct {
+	Name      string            `json:"name,omitempty"`
+	Size      string            `json:"size,omitempty"`
+	Count     int               `json:"count,omitempty"`
+	Tags      []string          `json:"tags,omitempty"`
+	Labels    map[string]string `json:"labels,omitempty"`
+	AutoScale bool              `json:"auto_scale,omitempty"`
+	MinNodes  int               `json:"min_nodes,omitempty"`
+	MaxNodes  int               `json:"max_nodes,omitempty"`
+}
+
+// KubernetesNodePoolUpdateRequest represents a request to update a node pool in a
+// Kubernetes cluster.
+type KubernetesNodePoolUpdateRequest struct {
+	Name      string            `json:"name,omitempty"`
+	Count     *int              `json:"count,omitempty"`
+	Tags      []string          `json:"tags,omitempty"`
+	Labels    map[string]string `json:"labels,omitempty"`
+	AutoScale *bool             `json:"auto_scale,omitempty"`
+	MinNodes  *int              `json:"min_nodes,omitempty"`
+	MaxNodes  *int              `json:"max_nodes,omitempty"`
+}
+
+// KubernetesNodePoolRecycleNodesRequest is DEPRECATED please use DeleteNode
+// The type will be removed in godo 2.0.
+type KubernetesNodePoolRecycleNodesRequest struct {
+	Nodes []string `json:"nodes,omitempty"`
+}
+
+// KubernetesNodeDeleteRequest is a request to delete a specific node in a node pool.
+type KubernetesNodeDeleteRequest struct {
+	// Replace will cause a new node to be created to replace the deleted node.
+	Replace bool `json:"replace,omitempty"`
+
+	// SkipDrain skips draining the node before deleting it.
+	SkipDrain bool `json:"skip_drain,omitempty"`
+}
+
+// KubernetesClusterCredentialsGetRequest is a request to get cluster credentials.
+type KubernetesClusterCredentialsGetRequest struct {
+	ExpirySeconds *int `json:"expiry_seconds,omitempty"`
+}
+
+// KubernetesCluster represents a Kubernetes cluster.
+type KubernetesCluster struct {
+	ID            string   `json:"id,omitempty"`
+	Name          string   `json:"name,omitempty"`
+	RegionSlug    string   `json:"region,omitempty"`
+	VersionSlug   string   `json:"version,omitempty"`
+	ClusterSubnet string   `json:"cluster_subnet,omitempty"`
+	ServiceSubnet string   `json:"service_subnet,omitempty"`
+	IPv4          string   `json:"ipv4,omitempty"`
+	Endpoint      string   `json:"endpoint,omitempty"`
+	Tags          []string `json:"tags,omitempty"`
+	VPCUUID       string   `json:"vpc_uuid,omitempty"`
+
+	NodePools []*KubernetesNodePool `json:"node_pools,omitempty"`
+
+	MaintenancePolicy *KubernetesMaintenancePolicy `json:"maintenance_policy,omitempty"`
+	AutoUpgrade       bool                         `json:"auto_upgrade,omitempty"`
+
+	Status    *KubernetesClusterStatus `json:"status,omitempty"`
+	CreatedAt time.Time                `json:"created_at,omitempty"`
+	UpdatedAt time.Time                `json:"updated_at,omitempty"`
+}
+
+// KubernetesClusterUser represents a Kubernetes cluster user.
+type KubernetesClusterUser struct {
+	Username string   `json:"username,omitempty"`
+	Groups   []string `json:"groups,omitempty"`
+}
+
+// KubernetesClusterCredentials represents Kubernetes cluster credentials.
+type KubernetesClusterCredentials struct {
+	Server                   string    `json:"server"`
+	CertificateAuthorityData []byte    `json:"certificate_authority_data"`
+	ClientCertificateData    []byte    `json:"client_certificate_data"`
+	ClientKeyData            []byte    `json:"client_key_data"`
+	Token                    string    `json:"token"`
+	ExpiresAt                time.Time `json:"expires_at"`
+}
+
+// KubernetesMaintenancePolicy is a configuration to set the maintenance window
+// of a cluster
+type KubernetesMaintenancePolicy struct {
+	StartTime string                         `json:"start_time"`
+	Duration  string                         `json:"duration"`
+	Day       KubernetesMaintenancePolicyDay `json:"day"`
+}
+
+// KubernetesMaintenancePolicyDay represents the possible days of a maintenance
+// window
+type KubernetesMaintenancePolicyDay int
+
+const (
+	KubernetesMaintenanceDayAny KubernetesMaintenancePolicyDay = iota
+	KubernetesMaintenanceDayMonday
+	KubernetesMaintenanceDayTuesday
+	KubernetesMaintenanceDayWednesday
+	KubernetesMaintenanceDayThursday
+	KubernetesMaintenanceDayFriday
+	KubernetesMaintenanceDaySaturday
+	KubernetesMaintenanceDaySunday
+)
+
+var (
+	days = [...]string{
+		"any",
+		"monday",
+		"tuesday",
+		"wednesday",
+		"thursday",
+		"friday",
+		"saturday",
+		"sunday",
+	}
+
+	toDay = map[string]KubernetesMaintenancePolicyDay{
+		"any":       KubernetesMaintenanceDayAny,
+		"monday":    KubernetesMaintenanceDayMonday,
+		"tuesday":   KubernetesMaintenanceDayTuesday,
+		"wednesday": KubernetesMaintenanceDayWednesday,
+		"thursday":  KubernetesMaintenanceDayThursday,
+		"friday":    KubernetesMaintenanceDayFriday,
+		"saturday":  KubernetesMaintenanceDaySaturday,
+		"sunday":    KubernetesMaintenanceDaySunday,
+	}
+)
+
+// KubernetesMaintenanceToDay returns the appropriate KubernetesMaintenancePolicyDay for the given string.
+func KubernetesMaintenanceToDay(day string) (KubernetesMaintenancePolicyDay, error) {
+	d, ok := toDay[day]
+	if !ok {
+		return 0, fmt.Errorf("unknown day: %q", day)
+	}
+
+	return d, nil
+}
+
+func (k KubernetesMaintenancePolicyDay) String() string {
+	if KubernetesMaintenanceDayAny <= k && k <= KubernetesMaintenanceDaySunday {
+		return days[k]
+	}
+	return fmt.Sprintf("%d !Weekday", k)
+
+}
+
+func (k *KubernetesMaintenancePolicyDay) UnmarshalJSON(data []byte) error {
+	var val string
+	if err := json.Unmarshal(data, &val); err != nil {
+		return err
+	}
+
+	parsed, err := KubernetesMaintenanceToDay(val)
+	if err != nil {
+		return err
+	}
+	*k = parsed
+	return nil
+}
+
+func (k KubernetesMaintenancePolicyDay) MarshalJSON() ([]byte, error) {
+	if KubernetesMaintenanceDayAny <= k && k <= KubernetesMaintenanceDaySunday {
+		return json.Marshal(days[k])
+	}
+
+	return nil, fmt.Errorf("invalid day: %d", k)
+}
+
+// Possible states for a cluster.
+const (
+	KubernetesClusterStatusProvisioning = KubernetesClusterStatusState("provisioning")
+	KubernetesClusterStatusRunning      = KubernetesClusterStatusState("running")
+	KubernetesClusterStatusDegraded     = KubernetesClusterStatusState("degraded")
+	KubernetesClusterStatusError        = KubernetesClusterStatusState("error")
+	KubernetesClusterStatusDeleted      = KubernetesClusterStatusState("deleted")
+	KubernetesClusterStatusUpgrading    = KubernetesClusterStatusState("upgrading")
+	KubernetesClusterStatusInvalid      = KubernetesClusterStatusState("invalid")
+)
+
+// KubernetesClusterStatusState represents states for a cluster.
+type KubernetesClusterStatusState string
+
+var _ encoding.TextUnmarshaler = (*KubernetesClusterStatusState)(nil)
+
+// UnmarshalText unmarshals the state.
+func (s *KubernetesClusterStatusState) UnmarshalText(text []byte) error {
+	switch KubernetesClusterStatusState(strings.ToLower(string(text))) {
+	case KubernetesClusterStatusProvisioning:
+		*s = KubernetesClusterStatusProvisioning
+	case KubernetesClusterStatusRunning:
+		*s = KubernetesClusterStatusRunning
+	case KubernetesClusterStatusDegraded:
+		*s = KubernetesClusterStatusDegraded
+	case KubernetesClusterStatusError:
+		*s = KubernetesClusterStatusError
+	case KubernetesClusterStatusDeleted:
+		*s = KubernetesClusterStatusDeleted
+	case KubernetesClusterStatusUpgrading:
+		*s = KubernetesClusterStatusUpgrading
+	case "", KubernetesClusterStatusInvalid:
+		*s = KubernetesClusterStatusInvalid
+	default:
+		return fmt.Errorf("unknown cluster state %q", string(text))
+	}
+	return nil
+}
+
+// KubernetesClusterStatus describes the status of a cluster.
+type KubernetesClusterStatus struct {
+	State   KubernetesClusterStatusState `json:"state,omitempty"`
+	Message string                       `json:"message,omitempty"`
+}
+
+// KubernetesNodePool represents a node pool in a Kubernetes cluster.
+type KubernetesNodePool struct {
+	ID        string            `json:"id,omitempty"`
+	Name      string            `json:"name,omitempty"`
+	Size      string            `json:"size,omitempty"`
+	Count     int               `json:"count,omitempty"`
+	Tags      []string          `json:"tags,omitempty"`
+	Labels    map[string]string `json:"labels,omitempty"`
+	AutoScale bool              `json:"auto_scale,omitempty"`
+	MinNodes  int               `json:"min_nodes,omitempty"`
+	MaxNodes  int               `json:"max_nodes,omitempty"`
+
+	Nodes []*KubernetesNode `json:"nodes,omitempty"`
+}
+
+// KubernetesNode represents a Node in a node pool in a Kubernetes cluster.
+type KubernetesNode struct {
+	ID        string                `json:"id,omitempty"`
+	Name      string                `json:"name,omitempty"`
+	Status    *KubernetesNodeStatus `json:"status,omitempty"`
+	DropletID string                `json:"droplet_id,omitempty"`
+
+	CreatedAt time.Time `json:"created_at,omitempty"`
+	UpdatedAt time.Time `json:"updated_at,omitempty"`
+}
+
+// KubernetesNodeStatus represents the status of a particular Node in a Kubernetes cluster.
+type KubernetesNodeStatus struct {
+	State   string `json:"state,omitempty"`
+	Message string `json:"message,omitempty"`
+}
+
+// KubernetesOptions represents options available for creating Kubernetes clusters.
+type KubernetesOptions struct {
+	Versions []*KubernetesVersion  `json:"versions,omitempty"`
+	Regions  []*KubernetesRegion   `json:"regions,omitempty"`
+	Sizes    []*KubernetesNodeSize `json:"sizes,omitempty"`
+}
+
+// KubernetesVersion is a DigitalOcean Kubernetes release.
+type KubernetesVersion struct {
+	Slug              string `json:"slug,omitempty"`
+	KubernetesVersion string `json:"kubernetes_version,omitempty"`
+}
+
+// KubernetesNodeSize is a node sizes supported for Kubernetes clusters.
+type KubernetesNodeSize struct {
+	Name string `json:"name"`
+	Slug string `json:"slug"`
+}
+
+// KubernetesRegion is a region usable by Kubernetes clusters.
+type KubernetesRegion struct {
+	Name string `json:"name"`
+	Slug string `json:"slug"`
+}
+
+type kubernetesClustersRoot struct {
+	Clusters []*KubernetesCluster `json:"kubernetes_clusters,omitempty"`
+	Links    *Links               `json:"links,omitempty"`
+	Meta     *Meta                `json:"meta"`
+}
+
+type kubernetesClusterRoot struct {
+	Cluster *KubernetesCluster `json:"kubernetes_cluster,omitempty"`
+}
+
+type kubernetesClusterUserRoot struct {
+	User *KubernetesClusterUser `json:"kubernetes_cluster_user,omitempty"`
+}
+
+type kubernetesNodePoolRoot struct {
+	NodePool *KubernetesNodePool `json:"node_pool,omitempty"`
+}
+
+type kubernetesNodePoolsRoot struct {
+	NodePools []*KubernetesNodePool `json:"node_pools,omitempty"`
+	Links     *Links                `json:"links,omitempty"`
+}
+
+type kubernetesUpgradesRoot struct {
+	AvailableUpgradeVersions []*KubernetesVersion `json:"available_upgrade_versions,omitempty"`
+}
+
+// Get retrieves the details of a Kubernetes cluster.
+func (svc *KubernetesServiceOp) Get(ctx context.Context, clusterID string) (*KubernetesCluster, *Response, error) {
+	path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesClusterRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Cluster, resp, nil
+}
+
+// GetUser retrieves the details of a Kubernetes cluster user.
+func (svc *KubernetesServiceOp) GetUser(ctx context.Context, clusterID string) (*KubernetesClusterUser, *Response, error) {
+	path := fmt.Sprintf("%s/%s/user", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesClusterUserRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.User, resp, nil
+}
+
+// GetUpgrades retrieves versions a Kubernetes cluster can be upgraded to. An
+// upgrade can be requested using `Upgrade`.
+func (svc *KubernetesServiceOp) GetUpgrades(ctx context.Context, clusterID string) ([]*KubernetesVersion, *Response, error) {
+	path := fmt.Sprintf("%s/%s/upgrades", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesUpgradesRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, nil, err
+	}
+	return root.AvailableUpgradeVersions, resp, nil
+}
+
+// Create creates a Kubernetes cluster.
+func (svc *KubernetesServiceOp) Create(ctx context.Context, create *KubernetesClusterCreateRequest) (*KubernetesCluster, *Response, error) {
+	path := kubernetesClustersPath
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesClusterRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Cluster, resp, nil
+}
+
+// Delete deletes a Kubernetes cluster. There is no way to recover a cluster
+// once it has been destroyed.
+func (svc *KubernetesServiceOp) Delete(ctx context.Context, clusterID string) (*Response, error) {
+	path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// List returns a list of the Kubernetes clusters visible with the caller's API token.
+func (svc *KubernetesServiceOp) List(ctx context.Context, opts *ListOptions) ([]*KubernetesCluster, *Response, error) {
+	path := kubernetesClustersPath
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesClustersRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Clusters, resp, nil
+}
+
+// KubernetesClusterConfig is the content of a Kubernetes config file, which can be
+// used to interact with your Kubernetes cluster using `kubectl`.
+// See: https://kubernetes.io/docs/tasks/tools/install-kubectl/
+type KubernetesClusterConfig struct {
+	KubeconfigYAML []byte
+}
+
+// GetKubeConfig returns a Kubernetes config file for the specified cluster.
+func (svc *KubernetesServiceOp) GetKubeConfig(ctx context.Context, clusterID string) (*KubernetesClusterConfig, *Response, error) {
+	path := fmt.Sprintf("%s/%s/kubeconfig", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	configBytes := bytes.NewBuffer(nil)
+	resp, err := svc.client.Do(ctx, req, configBytes)
+	if err != nil {
+		return nil, resp, err
+	}
+	res := &KubernetesClusterConfig{
+		KubeconfigYAML: configBytes.Bytes(),
+	}
+	return res, resp, nil
+}
+
+// GetCredentials returns a Kubernetes API server credentials for the specified cluster.
+func (svc *KubernetesServiceOp) GetCredentials(ctx context.Context, clusterID string, get *KubernetesClusterCredentialsGetRequest) (*KubernetesClusterCredentials, *Response, error) {
+	path := fmt.Sprintf("%s/%s/credentials", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	q := req.URL.Query()
+	if get.ExpirySeconds != nil {
+		q.Add("expiry_seconds", strconv.Itoa(*get.ExpirySeconds))
+	}
+	req.URL.RawQuery = q.Encode()
+	credentials := new(KubernetesClusterCredentials)
+	resp, err := svc.client.Do(ctx, req, credentials)
+	if err != nil {
+		return nil, nil, err
+	}
+	return credentials, resp, nil
+}
+
+// Update updates a Kubernetes cluster's properties.
+func (svc *KubernetesServiceOp) Update(ctx context.Context, clusterID string, update *KubernetesClusterUpdateRequest) (*KubernetesCluster, *Response, error) {
+	path := fmt.Sprintf("%s/%s", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPut, path, update)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesClusterRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Cluster, resp, nil
+}
+
+// Upgrade upgrades a Kubernetes cluster to a new version. Valid upgrade
+// versions for a given cluster can be retrieved with `GetUpgrades`.
+func (svc *KubernetesServiceOp) Upgrade(ctx context.Context, clusterID string, upgrade *KubernetesClusterUpgradeRequest) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/upgrade", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, upgrade)
+	if err != nil {
+		return nil, err
+	}
+	return svc.client.Do(ctx, req, nil)
+}
+
+// CreateNodePool creates a new node pool in an existing Kubernetes cluster.
+func (svc *KubernetesServiceOp) CreateNodePool(ctx context.Context, clusterID string, create *KubernetesNodePoolCreateRequest) (*KubernetesNodePool, *Response, error) {
+	path := fmt.Sprintf("%s/%s/node_pools", kubernetesClustersPath, clusterID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, create)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesNodePoolRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.NodePool, resp, nil
+}
+
+// GetNodePool retrieves an existing node pool in a Kubernetes cluster.
+func (svc *KubernetesServiceOp) GetNodePool(ctx context.Context, clusterID, poolID string) (*KubernetesNodePool, *Response, error) {
+	path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID)
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesNodePoolRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.NodePool, resp, nil
+}
+
+// ListNodePools lists all the node pools found in a Kubernetes cluster.
+func (svc *KubernetesServiceOp) ListNodePools(ctx context.Context, clusterID string, opts *ListOptions) ([]*KubernetesNodePool, *Response, error) {
+	path := fmt.Sprintf("%s/%s/node_pools", kubernetesClustersPath, clusterID)
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesNodePoolsRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.NodePools, resp, nil
+}
+
+// UpdateNodePool updates the details of an existing node pool.
+func (svc *KubernetesServiceOp) UpdateNodePool(ctx context.Context, clusterID, poolID string, update *KubernetesNodePoolUpdateRequest) (*KubernetesNodePool, *Response, error) {
+	path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPut, path, update)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesNodePoolRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.NodePool, resp, nil
+}
+
+// RecycleNodePoolNodes is DEPRECATED please use DeleteNode
+// The method will be removed in godo 2.0.
+func (svc *KubernetesServiceOp) RecycleNodePoolNodes(ctx context.Context, clusterID, poolID string, recycle *KubernetesNodePoolRecycleNodesRequest) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/node_pools/%s/recycle", kubernetesClustersPath, clusterID, poolID)
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, path, recycle)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// DeleteNodePool deletes a node pool, and subsequently all the nodes in that pool.
+func (svc *KubernetesServiceOp) DeleteNodePool(ctx context.Context, clusterID, poolID string) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/node_pools/%s", kubernetesClustersPath, clusterID, poolID)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// DeleteNode deletes a specific node in a node pool.
+func (svc *KubernetesServiceOp) DeleteNode(ctx context.Context, clusterID, poolID, nodeID string, deleteReq *KubernetesNodeDeleteRequest) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/node_pools/%s/nodes/%s", kubernetesClustersPath, clusterID, poolID, nodeID)
+	if deleteReq != nil {
+		v := make(url.Values)
+		if deleteReq.SkipDrain {
+			v.Set("skip_drain", "1")
+		}
+		if deleteReq.Replace {
+			v.Set("replace", "1")
+		}
+		if query := v.Encode(); query != "" {
+			path = path + "?" + query
+		}
+	}
+
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+type kubernetesOptionsRoot struct {
+	Options *KubernetesOptions `json:"options,omitempty"`
+	Links   *Links             `json:"links,omitempty"`
+}
+
+// GetOptions returns options about the Kubernetes service, such as the versions available for
+// cluster creation.
+func (svc *KubernetesServiceOp) GetOptions(ctx context.Context) (*KubernetesOptions, *Response, error) {
+	path := kubernetesOptionsPath
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(kubernetesOptionsRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Options, resp, nil
+}

vendor/github.com/digitalocean/godo/links.go

@@ -0,0 +1,83 @@
+package godo
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+)
+
+// Links manages links that are returned along with a List
+type Links struct {
+	Pages   *Pages       `json:"pages,omitempty"`
+	Actions []LinkAction `json:"actions,omitempty"`
+}
+
+// Pages are pages specified in Links
+type Pages struct {
+	First string `json:"first,omitempty"`
+	Prev  string `json:"prev,omitempty"`
+	Last  string `json:"last,omitempty"`
+	Next  string `json:"next,omitempty"`
+}
+
+// LinkAction is a pointer to an action
+type LinkAction struct {
+	ID   int    `json:"id,omitempty"`
+	Rel  string `json:"rel,omitempty"`
+	HREF string `json:"href,omitempty"`
+}
+
+// CurrentPage is current page of the list
+func (l *Links) CurrentPage() (int, error) {
+	return l.Pages.current()
+}
+
+func (p *Pages) current() (int, error) {
+	switch {
+	case p == nil:
+		return 1, nil
+	case p.Prev == "" && p.Next != "":
+		return 1, nil
+	case p.Prev != "":
+		prevPage, err := pageForURL(p.Prev)
+		if err != nil {
+			return 0, err
+		}
+
+		return prevPage + 1, nil
+	}
+
+	return 0, nil
+}
+
+// IsLastPage returns true if the current page is the last
+func (l *Links) IsLastPage() bool {
+	if l.Pages == nil {
+		return true
+	}
+	return l.Pages.isLast()
+}
+
+func (p *Pages) isLast() bool {
+	return p.Next == ""
+}
+
+func pageForURL(urlText string) (int, error) {
+	u, err := url.ParseRequestURI(urlText)
+	if err != nil {
+		return 0, err
+	}
+
+	pageStr := u.Query().Get("page")
+	page, err := strconv.Atoi(pageStr)
+	if err != nil {
+		return 0, err
+	}
+
+	return page, nil
+}
+
+// Get a link action by id.
+func (la *LinkAction) Get(ctx context.Context, client *Client) (*Action, *Response, error) {
+	return client.Actions.Get(ctx, la.ID)
+}

vendor/github.com/digitalocean/godo/load_balancers.go

@@ -0,0 +1,324 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+const loadBalancersBasePath = "/v2/load_balancers"
+const forwardingRulesPath = "forwarding_rules"
+
+const dropletsPath = "droplets"
+
+// LoadBalancersService is an interface for managing load balancers with the DigitalOcean API.
+// See: https://developers.digitalocean.com/documentation/v2#load-balancers
+type LoadBalancersService interface {
+	Get(context.Context, string) (*LoadBalancer, *Response, error)
+	List(context.Context, *ListOptions) ([]LoadBalancer, *Response, error)
+	Create(context.Context, *LoadBalancerRequest) (*LoadBalancer, *Response, error)
+	Update(ctx context.Context, lbID string, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error)
+	Delete(ctx context.Context, lbID string) (*Response, error)
+	AddDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error)
+	RemoveDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error)
+	AddForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error)
+	RemoveForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error)
+}
+
+// LoadBalancer represents a DigitalOcean load balancer configuration.
+// Tags can only be provided upon the creation of a Load Balancer.
+type LoadBalancer struct {
+	ID                     string           `json:"id,omitempty"`
+	Name                   string           `json:"name,omitempty"`
+	IP                     string           `json:"ip,omitempty"`
+	Algorithm              string           `json:"algorithm,omitempty"`
+	Status                 string           `json:"status,omitempty"`
+	Created                string           `json:"created_at,omitempty"`
+	ForwardingRules        []ForwardingRule `json:"forwarding_rules,omitempty"`
+	HealthCheck            *HealthCheck     `json:"health_check,omitempty"`
+	StickySessions         *StickySessions  `json:"sticky_sessions,omitempty"`
+	Region                 *Region          `json:"region,omitempty"`
+	DropletIDs             []int            `json:"droplet_ids,omitempty"`
+	Tag                    string           `json:"tag,omitempty"`
+	Tags                   []string         `json:"tags,omitempty"`
+	RedirectHttpToHttps    bool             `json:"redirect_http_to_https,omitempty"`
+	EnableProxyProtocol    bool             `json:"enable_proxy_protocol,omitempty"`
+	EnableBackendKeepalive bool             `json:"enable_backend_keepalive,omitempty"`
+	VPCUUID                string           `json:"vpc_uuid,omitempty"`
+}
+
+// String creates a human-readable description of a LoadBalancer.
+func (l LoadBalancer) String() string {
+	return Stringify(l)
+}
+
+func (l LoadBalancer) URN() string {
+	return ToURN("LoadBalancer", l.ID)
+}
+
+// AsRequest creates a LoadBalancerRequest that can be submitted to Update with the current values of the LoadBalancer.
+// Modifying the returned LoadBalancerRequest will not modify the original LoadBalancer.
+func (l LoadBalancer) AsRequest() *LoadBalancerRequest {
+	r := LoadBalancerRequest{
+		Name:                   l.Name,
+		Algorithm:              l.Algorithm,
+		ForwardingRules:        append([]ForwardingRule(nil), l.ForwardingRules...),
+		DropletIDs:             append([]int(nil), l.DropletIDs...),
+		Tag:                    l.Tag,
+		RedirectHttpToHttps:    l.RedirectHttpToHttps,
+		EnableProxyProtocol:    l.EnableProxyProtocol,
+		EnableBackendKeepalive: l.EnableBackendKeepalive,
+		HealthCheck:            l.HealthCheck,
+		VPCUUID:                l.VPCUUID,
+	}
+
+	if l.HealthCheck != nil {
+		r.HealthCheck = &HealthCheck{}
+		*r.HealthCheck = *l.HealthCheck
+	}
+	if l.StickySessions != nil {
+		r.StickySessions = &StickySessions{}
+		*r.StickySessions = *l.StickySessions
+	}
+	if l.Region != nil {
+		r.Region = l.Region.Slug
+	}
+	return &r
+}
+
+// ForwardingRule represents load balancer forwarding rules.
+type ForwardingRule struct {
+	EntryProtocol  string `json:"entry_protocol,omitempty"`
+	EntryPort      int    `json:"entry_port,omitempty"`
+	TargetProtocol string `json:"target_protocol,omitempty"`
+	TargetPort     int    `json:"target_port,omitempty"`
+	CertificateID  string `json:"certificate_id,omitempty"`
+	TlsPassthrough bool   `json:"tls_passthrough,omitempty"`
+}
+
+// String creates a human-readable description of a ForwardingRule.
+func (f ForwardingRule) String() string {
+	return Stringify(f)
+}
+
+// HealthCheck represents optional load balancer health check rules.
+type HealthCheck struct {
+	Protocol               string `json:"protocol,omitempty"`
+	Port                   int    `json:"port,omitempty"`
+	Path                   string `json:"path,omitempty"`
+	CheckIntervalSeconds   int    `json:"check_interval_seconds,omitempty"`
+	ResponseTimeoutSeconds int    `json:"response_timeout_seconds,omitempty"`
+	HealthyThreshold       int    `json:"healthy_threshold,omitempty"`
+	UnhealthyThreshold     int    `json:"unhealthy_threshold,omitempty"`
+}
+
+// String creates a human-readable description of a HealthCheck.
+func (h HealthCheck) String() string {
+	return Stringify(h)
+}
+
+// StickySessions represents optional load balancer session affinity rules.
+type StickySessions struct {
+	Type             string `json:"type,omitempty"`
+	CookieName       string `json:"cookie_name,omitempty"`
+	CookieTtlSeconds int    `json:"cookie_ttl_seconds,omitempty"`
+}
+
+// String creates a human-readable description of a StickySessions instance.
+func (s StickySessions) String() string {
+	return Stringify(s)
+}
+
+// LoadBalancerRequest represents the configuration to be applied to an existing or a new load balancer.
+type LoadBalancerRequest struct {
+	Name                   string           `json:"name,omitempty"`
+	Algorithm              string           `json:"algorithm,omitempty"`
+	Region                 string           `json:"region,omitempty"`
+	ForwardingRules        []ForwardingRule `json:"forwarding_rules,omitempty"`
+	HealthCheck            *HealthCheck     `json:"health_check,omitempty"`
+	StickySessions         *StickySessions  `json:"sticky_sessions,omitempty"`
+	DropletIDs             []int            `json:"droplet_ids,omitempty"`
+	Tag                    string           `json:"tag,omitempty"`
+	Tags                   []string         `json:"tags,omitempty"`
+	RedirectHttpToHttps    bool             `json:"redirect_http_to_https,omitempty"`
+	EnableProxyProtocol    bool             `json:"enable_proxy_protocol,omitempty"`
+	EnableBackendKeepalive bool             `json:"enable_backend_keepalive,omitempty"`
+	VPCUUID                string           `json:"vpc_uuid,omitempty"`
+}
+
+// String creates a human-readable description of a LoadBalancerRequest.
+func (l LoadBalancerRequest) String() string {
+	return Stringify(l)
+}
+
+type forwardingRulesRequest struct {
+	Rules []ForwardingRule `json:"forwarding_rules,omitempty"`
+}
+
+func (l forwardingRulesRequest) String() string {
+	return Stringify(l)
+}
+
+type dropletIDsRequest struct {
+	IDs []int `json:"droplet_ids,omitempty"`
+}
+
+func (l dropletIDsRequest) String() string {
+	return Stringify(l)
+}
+
+type loadBalancersRoot struct {
+	LoadBalancers []LoadBalancer `json:"load_balancers"`
+	Links         *Links         `json:"links"`
+	Meta          *Meta          `json:"meta"`
+}
+
+type loadBalancerRoot struct {
+	LoadBalancer *LoadBalancer `json:"load_balancer"`
+}
+
+// LoadBalancersServiceOp handles communication with load balancer-related methods of the DigitalOcean API.
+type LoadBalancersServiceOp struct {
+	client *Client
+}
+
+var _ LoadBalancersService = &LoadBalancersServiceOp{}
+
+// Get an existing load balancer by its identifier.
+func (l *LoadBalancersServiceOp) Get(ctx context.Context, lbID string) (*LoadBalancer, *Response, error) {
+	path := fmt.Sprintf("%s/%s", loadBalancersBasePath, lbID)
+
+	req, err := l.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(loadBalancerRoot)
+	resp, err := l.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.LoadBalancer, resp, err
+}
+
+// List load balancers, with optional pagination.
+func (l *LoadBalancersServiceOp) List(ctx context.Context, opt *ListOptions) ([]LoadBalancer, *Response, error) {
+	path, err := addOptions(loadBalancersBasePath, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := l.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(loadBalancersRoot)
+	resp, err := l.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.LoadBalancers, resp, err
+}
+
+// Create a new load balancer with a given configuration.
+func (l *LoadBalancersServiceOp) Create(ctx context.Context, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error) {
+	req, err := l.client.NewRequest(ctx, http.MethodPost, loadBalancersBasePath, lbr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(loadBalancerRoot)
+	resp, err := l.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.LoadBalancer, resp, err
+}
+
+// Update an existing load balancer with new configuration.
+func (l *LoadBalancersServiceOp) Update(ctx context.Context, lbID string, lbr *LoadBalancerRequest) (*LoadBalancer, *Response, error) {
+	path := fmt.Sprintf("%s/%s", loadBalancersBasePath, lbID)
+
+	req, err := l.client.NewRequest(ctx, "PUT", path, lbr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(loadBalancerRoot)
+	resp, err := l.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.LoadBalancer, resp, err
+}
+
+// Delete a load balancer by its identifier.
+func (l *LoadBalancersServiceOp) Delete(ctx context.Context, ldID string) (*Response, error) {
+	path := fmt.Sprintf("%s/%s", loadBalancersBasePath, ldID)
+
+	req, err := l.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return l.client.Do(ctx, req, nil)
+}
+
+// AddDroplets adds droplets to a load balancer.
+func (l *LoadBalancersServiceOp) AddDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, dropletsPath)
+
+	req, err := l.client.NewRequest(ctx, http.MethodPost, path, &dropletIDsRequest{IDs: dropletIDs})
+	if err != nil {
+		return nil, err
+	}
+
+	return l.client.Do(ctx, req, nil)
+}
+
+// RemoveDroplets removes droplets from a load balancer.
+func (l *LoadBalancersServiceOp) RemoveDroplets(ctx context.Context, lbID string, dropletIDs ...int) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, dropletsPath)
+
+	req, err := l.client.NewRequest(ctx, http.MethodDelete, path, &dropletIDsRequest{IDs: dropletIDs})
+	if err != nil {
+		return nil, err
+	}
+
+	return l.client.Do(ctx, req, nil)
+}
+
+// AddForwardingRules adds forwarding rules to a load balancer.
+func (l *LoadBalancersServiceOp) AddForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, forwardingRulesPath)
+
+	req, err := l.client.NewRequest(ctx, http.MethodPost, path, &forwardingRulesRequest{Rules: rules})
+	if err != nil {
+		return nil, err
+	}
+
+	return l.client.Do(ctx, req, nil)
+}
+
+// RemoveForwardingRules removes forwarding rules from a load balancer.
+func (l *LoadBalancersServiceOp) RemoveForwardingRules(ctx context.Context, lbID string, rules ...ForwardingRule) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/%s", loadBalancersBasePath, lbID, forwardingRulesPath)
+
+	req, err := l.client.NewRequest(ctx, http.MethodDelete, path, &forwardingRulesRequest{Rules: rules})
+	if err != nil {
+		return nil, err
+	}
+
+	return l.client.Do(ctx, req, nil)
+}

vendor/github.com/digitalocean/godo/meta.go

@@ -0,0 +1,6 @@
+package godo
+
+// Meta describes generic information about a response.
+type Meta struct {
+	Total int `json:"total"`
+}

vendor/github.com/digitalocean/godo/projects.go

@@ -0,0 +1,310 @@
+package godo
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"path"
+)
+
+const (
+	// DefaultProject is the ID you should use if you are working with your
+	// default project.
+	DefaultProject = "default"
+
+	projectsBasePath = "/v2/projects"
+)
+
+// ProjectsService is an interface for creating and managing Projects with the DigitalOcean API.
+// See: https://developers.digitalocean.com/documentation/v2/#projects
+type ProjectsService interface {
+	List(context.Context, *ListOptions) ([]Project, *Response, error)
+	GetDefault(context.Context) (*Project, *Response, error)
+	Get(context.Context, string) (*Project, *Response, error)
+	Create(context.Context, *CreateProjectRequest) (*Project, *Response, error)
+	Update(context.Context, string, *UpdateProjectRequest) (*Project, *Response, error)
+	Delete(context.Context, string) (*Response, error)
+
+	ListResources(context.Context, string, *ListOptions) ([]ProjectResource, *Response, error)
+	AssignResources(context.Context, string, ...interface{}) ([]ProjectResource, *Response, error)
+}
+
+// ProjectsServiceOp handles communication with Projects methods of the DigitalOcean API.
+type ProjectsServiceOp struct {
+	client *Client
+}
+
+// Project represents a DigitalOcean Project configuration.
+type Project struct {
+	ID          string `json:"id"`
+	OwnerUUID   string `json:"owner_uuid"`
+	OwnerID     uint64 `json:"owner_id"`
+	Name        string `json:"name"`
+	Description string `json:"description"`
+	Purpose     string `json:"purpose"`
+	Environment string `json:"environment"`
+	IsDefault   bool   `json:"is_default"`
+	CreatedAt   string `json:"created_at"`
+	UpdatedAt   string `json:"updated_at"`
+}
+
+// String creates a human-readable description of a Project.
+func (p Project) String() string {
+	return Stringify(p)
+}
+
+// CreateProjectRequest represents the request to create a new project.
+type CreateProjectRequest struct {
+	Name        string `json:"name"`
+	Description string `json:"description"`
+	Purpose     string `json:"purpose"`
+	Environment string `json:"environment"`
+}
+
+// UpdateProjectRequest represents the request to update project information.
+// This type expects certain attribute types, but is built this way to allow
+// nil values as well. See `updateProjectRequest` for the "real" types.
+type UpdateProjectRequest struct {
+	Name        interface{}
+	Description interface{}
+	Purpose     interface{}
+	Environment interface{}
+	IsDefault   interface{}
+}
+
+type updateProjectRequest struct {
+	Name        *string `json:"name"`
+	Description *string `json:"description"`
+	Purpose     *string `json:"purpose"`
+	Environment *string `json:"environment"`
+	IsDefault   *bool   `json:"is_default"`
+}
+
+// MarshalJSON takes an UpdateRequest and converts it to the "typed" request
+// which is sent to the projects API. This is a PATCH request, which allows
+// partial attributes, so `null` values are OK.
+func (upr *UpdateProjectRequest) MarshalJSON() ([]byte, error) {
+	d := &updateProjectRequest{}
+	if str, ok := upr.Name.(string); ok {
+		d.Name = &str
+	}
+	if str, ok := upr.Description.(string); ok {
+		d.Description = &str
+	}
+	if str, ok := upr.Purpose.(string); ok {
+		d.Purpose = &str
+	}
+	if str, ok := upr.Environment.(string); ok {
+		d.Environment = &str
+	}
+	if val, ok := upr.IsDefault.(bool); ok {
+		d.IsDefault = &val
+	}
+
+	return json.Marshal(d)
+}
+
+type assignResourcesRequest struct {
+	Resources []string `json:"resources"`
+}
+
+// ProjectResource is the projects API's representation of a resource.
+type ProjectResource struct {
+	URN        string                `json:"urn"`
+	AssignedAt string                `json:"assigned_at"`
+	Links      *ProjectResourceLinks `json:"links"`
+	Status     string                `json:"status,omitempty"`
+}
+
+// ProjetResourceLinks specify the link for more information about the resource.
+type ProjectResourceLinks struct {
+	Self string `json:"self"`
+}
+
+type projectsRoot struct {
+	Projects []Project `json:"projects"`
+	Links    *Links    `json:"links"`
+	Meta     *Meta     `json:"meta"`
+}
+
+type projectRoot struct {
+	Project *Project `json:"project"`
+}
+
+type projectResourcesRoot struct {
+	Resources []ProjectResource `json:"resources"`
+	Links     *Links            `json:"links,omitempty"`
+	Meta      *Meta             `json:"meta"`
+}
+
+var _ ProjectsService = &ProjectsServiceOp{}
+
+// List Projects.
+func (p *ProjectsServiceOp) List(ctx context.Context, opts *ListOptions) ([]Project, *Response, error) {
+	path, err := addOptions(projectsBasePath, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(projectsRoot)
+	resp, err := p.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Projects, resp, err
+}
+
+// GetDefault project.
+func (p *ProjectsServiceOp) GetDefault(ctx context.Context) (*Project, *Response, error) {
+	return p.getHelper(ctx, "default")
+}
+
+// Get retrieves a single project by its ID.
+func (p *ProjectsServiceOp) Get(ctx context.Context, projectID string) (*Project, *Response, error) {
+	return p.getHelper(ctx, projectID)
+}
+
+// Create a new project.
+func (p *ProjectsServiceOp) Create(ctx context.Context, cr *CreateProjectRequest) (*Project, *Response, error) {
+	req, err := p.client.NewRequest(ctx, http.MethodPost, projectsBasePath, cr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(projectRoot)
+	resp, err := p.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Project, resp, err
+}
+
+// Update an existing project.
+func (p *ProjectsServiceOp) Update(ctx context.Context, projectID string, ur *UpdateProjectRequest) (*Project, *Response, error) {
+	path := path.Join(projectsBasePath, projectID)
+	req, err := p.client.NewRequest(ctx, http.MethodPatch, path, ur)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(projectRoot)
+	resp, err := p.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Project, resp, err
+}
+
+// Delete an existing project. You cannot have any resources in a project
+// before deleting it. See the API documentation for more details.
+func (p *ProjectsServiceOp) Delete(ctx context.Context, projectID string) (*Response, error) {
+	path := path.Join(projectsBasePath, projectID)
+	req, err := p.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	return p.client.Do(ctx, req, nil)
+}
+
+// ListResources lists all resources in a project.
+func (p *ProjectsServiceOp) ListResources(ctx context.Context, projectID string, opts *ListOptions) ([]ProjectResource, *Response, error) {
+	basePath := path.Join(projectsBasePath, projectID, "resources")
+	path, err := addOptions(basePath, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(projectResourcesRoot)
+	resp, err := p.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Resources, resp, err
+}
+
+// AssignResources assigns one or more resources to a project. AssignResources
+// accepts resources in two possible formats:
+
+//  1. The resource type, like `&Droplet{ID: 1}` or `&FloatingIP{IP: "1.2.3.4"}`
+//  2. A valid DO URN as a string, like "do:droplet:1234"
+//
+// There is no unassign. To move a resource to another project, just assign
+// it to that other project.
+func (p *ProjectsServiceOp) AssignResources(ctx context.Context, projectID string, resources ...interface{}) ([]ProjectResource, *Response, error) {
+	path := path.Join(projectsBasePath, projectID, "resources")
+
+	ar := &assignResourcesRequest{
+		Resources: make([]string, len(resources)),
+	}
+
+	for i, resource := range resources {
+		switch resource := resource.(type) {
+		case ResourceWithURN:
+			ar.Resources[i] = resource.URN()
+		case string:
+			ar.Resources[i] = resource
+		default:
+			return nil, nil, fmt.Errorf("%T must either be a string or have a valid URN method", resource)
+		}
+	}
+	req, err := p.client.NewRequest(ctx, http.MethodPost, path, ar)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(projectResourcesRoot)
+	resp, err := p.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+
+	return root.Resources, resp, err
+}
+
+func (p *ProjectsServiceOp) getHelper(ctx context.Context, projectID string) (*Project, *Response, error) {
+	path := path.Join(projectsBasePath, projectID)
+
+	req, err := p.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(projectRoot)
+	resp, err := p.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Project, resp, err
+}

vendor/github.com/digitalocean/godo/regions.go

@@ -0,0 +1,68 @@
+package godo
+
+import (
+	"context"
+	"net/http"
+)
+
+// RegionsService is an interface for interfacing with the regions
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2#regions
+type RegionsService interface {
+	List(context.Context, *ListOptions) ([]Region, *Response, error)
+}
+
+// RegionsServiceOp handles communication with the region related methods of the
+// DigitalOcean API.
+type RegionsServiceOp struct {
+	client *Client
+}
+
+var _ RegionsService = &RegionsServiceOp{}
+
+// Region represents a DigitalOcean Region
+type Region struct {
+	Slug      string   `json:"slug,omitempty"`
+	Name      string   `json:"name,omitempty"`
+	Sizes     []string `json:"sizes,omitempty"`
+	Available bool     `json:"available,omitempty"`
+	Features  []string `json:"features,omitempty"`
+}
+
+type regionsRoot struct {
+	Regions []Region
+	Links   *Links `json:"links"`
+	Meta    *Meta  `json:"meta"`
+}
+
+func (r Region) String() string {
+	return Stringify(r)
+}
+
+// List all regions
+func (s *RegionsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Region, *Response, error) {
+	path := "v2/regions"
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(regionsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Regions, resp, err
+}

vendor/github.com/digitalocean/godo/registry.go

@@ -0,0 +1,253 @@
+package godo
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+)
+
+const (
+	registryPath = "/v2/registry"
+	// RegistryServer is the hostname of the DigitalOcean registry service
+	RegistryServer = "registry.digitalocean.com"
+)
+
+// RegistryService is an interface for interfacing with the Registry endpoints
+// of the DigitalOcean API.
+// See: https://developers.digitalocean.com/documentation/v2#registry
+type RegistryService interface {
+	Create(context.Context, *RegistryCreateRequest) (*Registry, *Response, error)
+	Get(context.Context) (*Registry, *Response, error)
+	Delete(context.Context) (*Response, error)
+	DockerCredentials(context.Context, *RegistryDockerCredentialsRequest) (*DockerCredentials, *Response, error)
+	ListRepositories(context.Context, string, *ListOptions) ([]*Repository, *Response, error)
+	ListRepositoryTags(context.Context, string, string, *ListOptions) ([]*RepositoryTag, *Response, error)
+	DeleteTag(context.Context, string, string, string) (*Response, error)
+	DeleteManifest(context.Context, string, string, string) (*Response, error)
+}
+
+var _ RegistryService = &RegistryServiceOp{}
+
+// RegistryServiceOp handles communication with Registry methods of the DigitalOcean API.
+type RegistryServiceOp struct {
+	client *Client
+}
+
+// RegistryCreateRequest represents a request to create a registry.
+type RegistryCreateRequest struct {
+	Name string `json:"name,omitempty"`
+}
+
+// RegistryDockerCredentialsRequest represents a request to retrieve docker
+// credentials for a registry.
+type RegistryDockerCredentialsRequest struct {
+	ReadWrite     bool `json:"read_write"`
+	ExpirySeconds *int `json:"expiry_seconds,omitempty"`
+}
+
+// Registry represents a registry.
+type Registry struct {
+	Name      string    `json:"name,omitempty"`
+	CreatedAt time.Time `json:"created_at,omitempty"`
+}
+
+// Repository represents a repository
+type Repository struct {
+	RegistryName string         `json:"registry_name,omitempty"`
+	Name         string         `json:"name,omitempty"`
+	LatestTag    *RepositoryTag `json:"latest_tag,omitempty"`
+	TagCount     uint64         `json:"tag_count,omitempty"`
+}
+
+// RepositoryTag represents a repository tag
+type RepositoryTag struct {
+	RegistryName        string    `json:"registry_name,omitempty"`
+	Repository          string    `json:"repository,omitempty"`
+	Tag                 string    `json:"tag,omitempty"`
+	ManifestDigest      string    `json:"manifest_digest,omitempty"`
+	CompressedSizeBytes uint64    `json:"compressed_size_bytes,omitempty"`
+	SizeBytes           uint64    `json:"size_bytes,omitempty"`
+	UpdatedAt           time.Time `json:"updated_at,omitempty"`
+}
+
+type registryRoot struct {
+	Registry *Registry `json:"registry,omitempty"`
+}
+
+type repositoriesRoot struct {
+	Repositories []*Repository `json:"repositories,omitempty"`
+	Links        *Links        `json:"links,omitempty"`
+	Meta         *Meta         `json:"meta"`
+}
+
+type repositoryTagsRoot struct {
+	Tags  []*RepositoryTag `json:"tags,omitempty"`
+	Links *Links           `json:"links,omitempty"`
+	Meta  *Meta            `json:"meta"`
+}
+
+// Get retrieves the details of a Registry.
+func (svc *RegistryServiceOp) Get(ctx context.Context) (*Registry, *Response, error) {
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, registryPath, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(registryRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Registry, resp, nil
+}
+
+// Create creates a registry.
+func (svc *RegistryServiceOp) Create(ctx context.Context, create *RegistryCreateRequest) (*Registry, *Response, error) {
+	req, err := svc.client.NewRequest(ctx, http.MethodPost, registryPath, create)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(registryRoot)
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	return root.Registry, resp, nil
+}
+
+// Delete deletes a registry. There is no way to recover a registry once it has
+// been destroyed.
+func (svc *RegistryServiceOp) Delete(ctx context.Context) (*Response, error) {
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, registryPath, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+	return resp, nil
+}
+
+// DockerCredentials is the content of a Docker config file
+// that is used by the docker CLI
+// See: https://docs.docker.com/engine/reference/commandline/cli/#configjson-properties
+type DockerCredentials struct {
+	DockerConfigJSON []byte
+}
+
+// DockerCredentials retrieves a Docker config file containing the registry's credentials.
+func (svc *RegistryServiceOp) DockerCredentials(ctx context.Context, request *RegistryDockerCredentialsRequest) (*DockerCredentials, *Response, error) {
+	path := fmt.Sprintf("%s/%s", registryPath, "docker-credentials")
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	q := req.URL.Query()
+	q.Add("read_write", strconv.FormatBool(request.ReadWrite))
+	if request.ExpirySeconds != nil {
+		q.Add("expiry_seconds", strconv.Itoa(*request.ExpirySeconds))
+	}
+	req.URL.RawQuery = q.Encode()
+
+	var buf bytes.Buffer
+	resp, err := svc.client.Do(ctx, req, &buf)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	dc := &DockerCredentials{
+		DockerConfigJSON: buf.Bytes(),
+	}
+	return dc, resp, nil
+}
+
+// ListRepositories returns a list of the Repositories visible with the registry's credentials.
+func (svc *RegistryServiceOp) ListRepositories(ctx context.Context, registry string, opts *ListOptions) ([]*Repository, *Response, error) {
+	path := fmt.Sprintf("%s/%s/repositories", registryPath, registry)
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(repositoriesRoot)
+
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Repositories, resp, nil
+}
+
+// ListRepositoryTags returns a list of the RepositoryTags available within the given repository.
+func (svc *RegistryServiceOp) ListRepositoryTags(ctx context.Context, registry, repository string, opts *ListOptions) ([]*RepositoryTag, *Response, error) {
+	path := fmt.Sprintf("%s/%s/repositories/%s/tags", registryPath, registry, url.PathEscape(repository))
+	path, err := addOptions(path, opts)
+	if err != nil {
+		return nil, nil, err
+	}
+	req, err := svc.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+	root := new(repositoryTagsRoot)
+
+	resp, err := svc.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Tags, resp, nil
+}
+
+// DeleteTag deletes a tag within a given repository.
+func (svc *RegistryServiceOp) DeleteTag(ctx context.Context, registry, repository, tag string) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/repositories/%s/tags/%s", registryPath, registry, url.PathEscape(repository), tag)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+
+	return resp, nil
+}
+
+// DeleteManifest deletes a manifest by its digest within a given repository.
+func (svc *RegistryServiceOp) DeleteManifest(ctx context.Context, registry, repository, digest string) (*Response, error) {
+	path := fmt.Sprintf("%s/%s/repositories/%s/digests/%s", registryPath, registry, url.PathEscape(repository), digest)
+	req, err := svc.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err := svc.client.Do(ctx, req, nil)
+	if err != nil {
+		return resp, err
+	}
+
+	return resp, nil
+}

vendor/github.com/digitalocean/godo/sizes.go

@@ -0,0 +1,72 @@
+package godo
+
+import (
+	"context"
+	"net/http"
+)
+
+// SizesService is an interface for interfacing with the size
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2#sizes
+type SizesService interface {
+	List(context.Context, *ListOptions) ([]Size, *Response, error)
+}
+
+// SizesServiceOp handles communication with the size related methods of the
+// DigitalOcean API.
+type SizesServiceOp struct {
+	client *Client
+}
+
+var _ SizesService = &SizesServiceOp{}
+
+// Size represents a DigitalOcean Size
+type Size struct {
+	Slug         string   `json:"slug,omitempty"`
+	Memory       int      `json:"memory,omitempty"`
+	Vcpus        int      `json:"vcpus,omitempty"`
+	Disk         int      `json:"disk,omitempty"`
+	PriceMonthly float64  `json:"price_monthly,omitempty"`
+	PriceHourly  float64  `json:"price_hourly,omitempty"`
+	Regions      []string `json:"regions,omitempty"`
+	Available    bool     `json:"available,omitempty"`
+	Transfer     float64  `json:"transfer,omitempty"`
+}
+
+func (s Size) String() string {
+	return Stringify(s)
+}
+
+type sizesRoot struct {
+	Sizes []Size
+	Links *Links `json:"links"`
+	Meta  *Meta  `json:"meta"`
+}
+
+// List all images
+func (s *SizesServiceOp) List(ctx context.Context, opt *ListOptions) ([]Size, *Response, error) {
+	path := "v2/sizes"
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(sizesRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+	if m := root.Meta; m != nil {
+		resp.Meta = m
+	}
+
+	return root.Sizes, resp, err
+}

vendor/github.com/digitalocean/godo/snapshots.go

@@ -0,0 +1,142 @@
+package godo
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+)
+
+const snapshotBasePath = "v2/snapshots"
+
+// SnapshotsService is an interface for interfacing with the snapshots
+// endpoints of the DigitalOcean API
+// See: https://developers.digitalocean.com/documentation/v2#snapshots
+type SnapshotsService interface {
+	List(context.Context, *ListOptions) ([]Snapshot, *Response, error)
+	ListVolume(context.Context, *ListOptions) ([]Snapshot, *Response, error)
+	ListDroplet(context.Context, *ListOptions) ([]Snapshot, *Response, error)
+	Get(context.Context, string) (*Snapshot, *Response, error)
+	Delete(context.Context, string) (*Response, error)
+}
+
+// SnapshotsServiceOp handles communication with the snapshot related methods of the
+// DigitalOcean API.
+type SnapshotsServiceOp struct {
+	client *Client
+}
+
+var _ SnapshotsService = &SnapshotsServiceOp{}
+
+// Snapshot represents a DigitalOcean Snapshot
+type Snapshot struct {
+	ID            string   `json:"id,omitempty"`
+	Name          string   `json:"name,omitempty"`
+	ResourceID    string   `json:"resource_id,omitempty"`
+	ResourceType  string   `json:"resource_type,omitempty"`
+	Regions       []string `json:"regions,omitempty"`
+	MinDiskSize   int      `json:"min_disk_size,omitempty"`
+	SizeGigaBytes float64  `json:"size_gigabytes,omitempty"`
+	Created       string   `json:"created_at,omitempty"`
+	Tags          []string `json:"tags,omitempty"`
+}
+
+type snapshotRoot struct {
+	Snapshot *Snapshot `json:"snapshot"`
+}
+
+type snapshotsRoot struct {
+	Snapshots []Snapshot `json:"snapshots"`
+	Links     *Links     `json:"links,omitempty"`
+	Meta      *Meta      `json:"meta,omitempty"`
+}
+
+type listSnapshotOptions struct {
+	ResourceType string `url:"resource_type,omitempty"`
+}
+
+func (s Snapshot) String() string {
+	return Stringify(s)
+}
+
+// List lists all the snapshots available.
+func (s *SnapshotsServiceOp) List(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) {
+	return s.list(ctx, opt, nil)
+}
+
+// ListDroplet lists all the Droplet snapshots.
+func (s *SnapshotsServiceOp) ListDroplet(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) {
+	listOpt := listSnapshotOptions{ResourceType: "droplet"}
+	return s.list(ctx, opt, &listOpt)
+}
+
+// ListVolume lists all the volume snapshots.
+func (s *SnapshotsServiceOp) ListVolume(ctx context.Context, opt *ListOptions) ([]Snapshot, *Response, error) {
+	listOpt := listSnapshotOptions{ResourceType: "volume"}
+	return s.list(ctx, opt, &listOpt)
+}
+
+// Get retrieves an snapshot by id.
+func (s *SnapshotsServiceOp) Get(ctx context.Context, snapshotID string) (*Snapshot, *Response, error) {
+	return s.get(ctx, snapshotID)
+}
+
+// Delete an snapshot.
+func (s *SnapshotsServiceOp) Delete(ctx context.Context, snapshotID string) (*Response, error) {
+	path := fmt.Sprintf("%s/%s", snapshotBasePath, snapshotID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodDelete, path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := s.client.Do(ctx, req, nil)
+
+	return resp, err
+}
+
+// Helper method for getting an individual snapshot
+func (s *SnapshotsServiceOp) get(ctx context.Context, ID string) (*Snapshot, *Response, error) {
+	path := fmt.Sprintf("%s/%s", snapshotBasePath, ID)
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(snapshotRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+
+	return root.Snapshot, resp, err
+}
+
+// Helper method for listing snapshots
+func (s *SnapshotsServiceOp) list(ctx context.Context, opt *ListOptions, listOpt *listSnapshotOptions) ([]Snapshot, *Response, error) {
+	path := snapshotBasePath
+	path, err := addOptions(path, opt)
+	if err != nil {
+		return nil, nil, err
+	}
+	path, err = addOptions(path, listOpt)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	req, err := s.client.NewRequest(ctx, http.MethodGet, path, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	root := new(snapshotsRoot)
+	resp, err := s.client.Do(ctx, req, root)
+	if err != nil {
+		return nil, resp, err
+	}
+	if l := root.Links; l != nil {
+		resp.Links = l
+	}
+
+	return root.Snapshots, resp, err
+}