prometheus/web/api/v1/api.go

// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	"context"
	"errors"
	"fmt"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"slices"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/go-kit/log"
	"github.com/go-kit/log/level"
	"github.com/grafana/regexp"
	jsoniter "github.com/json-iterator/go"
	"github.com/munnerz/goautoneg"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"
	"github.com/prometheus/common/route"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/model/labels"
	"github.com/prometheus/prometheus/model/metadata"
	"github.com/prometheus/prometheus/model/timestamp"
	"github.com/prometheus/prometheus/promql"
	"github.com/prometheus/prometheus/promql/parser"
	"github.com/prometheus/prometheus/rules"
	"github.com/prometheus/prometheus/scrape"
	"github.com/prometheus/prometheus/storage"
	"github.com/prometheus/prometheus/storage/remote"
	"github.com/prometheus/prometheus/tsdb"
	"github.com/prometheus/prometheus/tsdb/index"
	"github.com/prometheus/prometheus/util/annotations"
	"github.com/prometheus/prometheus/util/httputil"
	"github.com/prometheus/prometheus/util/stats"
)

type status string

const (
	statusSuccess status = "success"
	statusError   status = "error"

	// Non-standard status code (originally introduced by nginx) for the case when a client closes
	// the connection while the server is still processing the request.
	statusClientClosedConnection = 499
)

type errorType string

const (
	errorNone          errorType = ""
	errorTimeout       errorType = "timeout"
	errorCanceled      errorType = "canceled"
	errorExec          errorType = "execution"
	errorBadData       errorType = "bad_data"
	errorInternal      errorType = "internal"
	errorUnavailable   errorType = "unavailable"
	errorNotFound      errorType = "not_found"
	errorNotAcceptable errorType = "not_acceptable"
)

var LocalhostRepresentations = []string{"127.0.0.1", "localhost", "::1"}

type apiError struct {
	typ errorType
	err error
}

func (e *apiError) Error() string {
	return fmt.Sprintf("%s: %s", e.typ, e.err)
}

// ScrapePoolsRetriever provide the list of all scrape pools.
type ScrapePoolsRetriever interface {
	ScrapePools() []string
}

// TargetRetriever provides the list of active/dropped targets to scrape or not.
type TargetRetriever interface {
	TargetsActive() map[string][]*scrape.Target
	TargetsDropped() map[string][]*scrape.Target
	TargetsDroppedCounts() map[string]int
}

// AlertmanagerRetriever provides a list of all/dropped AlertManager URLs.
type AlertmanagerRetriever interface {
	Alertmanagers() []*url.URL
	DroppedAlertmanagers() []*url.URL
}

// RulesRetriever provides a list of active rules and alerts.
type RulesRetriever interface {
	RuleGroups() []*rules.Group
	AlertingRules() []*rules.AlertingRule
}

type StatsRenderer func(context.Context, *stats.Statistics, string) stats.QueryStats

func defaultStatsRenderer(_ context.Context, s *stats.Statistics, param string) stats.QueryStats {
	if param != "" {
		return stats.NewQueryStats(s)
	}
	return nil
}

// PrometheusVersion contains build information about Prometheus.
type PrometheusVersion struct {
	Version   string `json:"version"`
	Revision  string `json:"revision"`
	Branch    string `json:"branch"`
	BuildUser string `json:"buildUser"`
	BuildDate string `json:"buildDate"`
	GoVersion string `json:"goVersion"`
}

// RuntimeInfo contains runtime information about Prometheus.
type RuntimeInfo struct {
	StartTime           time.Time `json:"startTime"`
	CWD                 string    `json:"CWD"`
	ReloadConfigSuccess bool      `json:"reloadConfigSuccess"`
	LastConfigTime      time.Time `json:"lastConfigTime"`
	CorruptionCount     int64     `json:"corruptionCount"`
	GoroutineCount      int       `json:"goroutineCount"`
	GOMAXPROCS          int       `json:"GOMAXPROCS"`
	GOMEMLIMIT          int64     `json:"GOMEMLIMIT"`
	GOGC                string    `json:"GOGC"`
	GODEBUG             string    `json:"GODEBUG"`
	StorageRetention    string    `json:"storageRetention"`
}

// Response contains a response to a HTTP API request.
type Response struct {
	Status    status      `json:"status"`
	Data      interface{} `json:"data,omitempty"`
	ErrorType errorType   `json:"errorType,omitempty"`
	Error     string      `json:"error,omitempty"`
	Warnings  []string    `json:"warnings,omitempty"`
}

type apiFuncResult struct {
	data      interface{}
	err       *apiError
	warnings  annotations.Annotations
	finalizer func()
}

type apiFunc func(r *http.Request) apiFuncResult

// TSDBAdminStats defines the tsdb interfaces used by the v1 API for admin operations as well as statistics.
type TSDBAdminStats interface {
	CleanTombstones() error
	Delete(ctx context.Context, mint, maxt int64, ms ...*labels.Matcher) error
	Snapshot(dir string, withHead bool) error
	Stats(statsByLabelName string, limit int) (*tsdb.Stats, error)
	WALReplayStatus() (tsdb.WALReplayStatus, error)
}

// QueryEngine defines the interface for the *promql.Engine, so it can be replaced, wrapped or mocked.
type QueryEngine interface {
	SetQueryLogger(l promql.QueryLogger)
	NewInstantQuery(ctx context.Context, q storage.Queryable, opts promql.QueryOpts, qs string, ts time.Time) (promql.Query, error)
	NewRangeQuery(ctx context.Context, q storage.Queryable, opts promql.QueryOpts, qs string, start, end time.Time, interval time.Duration) (promql.Query, error)
}

type QueryOpts interface {
	EnablePerStepStats() bool
	LookbackDelta() time.Duration
}

// API can register a set of endpoints in a router and handle
// them using the provided storage and query engine.
type API struct {
	Queryable         storage.SampleAndChunkQueryable
	QueryEngine       QueryEngine
	ExemplarQueryable storage.ExemplarQueryable

	scrapePoolsRetriever  func(context.Context) ScrapePoolsRetriever
	targetRetriever       func(context.Context) TargetRetriever
	alertmanagerRetriever func(context.Context) AlertmanagerRetriever
	rulesRetriever        func(context.Context) RulesRetriever
	now                   func() time.Time
	config                func() config.Config
	flagsMap              map[string]string
	ready                 func(http.HandlerFunc) http.HandlerFunc
	globalURLOptions      GlobalURLOptions

	db            TSDBAdminStats
	dbDir         string
	enableAdmin   bool
	logger        log.Logger
	CORSOrigin    *regexp.Regexp
	buildInfo     *PrometheusVersion
	runtimeInfo   func() (RuntimeInfo, error)
	gatherer      prometheus.Gatherer
	isAgent       bool
	statsRenderer StatsRenderer

	remoteWriteHandler http.Handler
	remoteReadHandler  http.Handler
	otlpWriteHandler   http.Handler

	codecs []Codec
}

// NewAPI returns an initialized API type.
func NewAPI(
	qe QueryEngine,
	q storage.SampleAndChunkQueryable,
	ap storage.Appendable,
	eq storage.ExemplarQueryable,
	spsr func(context.Context) ScrapePoolsRetriever,
	tr func(context.Context) TargetRetriever,
	ar func(context.Context) AlertmanagerRetriever,
	configFunc func() config.Config,
	flagsMap map[string]string,
	globalURLOptions GlobalURLOptions,
	readyFunc func(http.HandlerFunc) http.HandlerFunc,
	db TSDBAdminStats,
	dbDir string,
	enableAdmin bool,
	logger log.Logger,
	rr func(context.Context) RulesRetriever,
	remoteReadSampleLimit int,
	remoteReadConcurrencyLimit int,
	remoteReadMaxBytesInFrame int,
	isAgent bool,
	corsOrigin *regexp.Regexp,
	runtimeInfo func() (RuntimeInfo, error),
	buildInfo *PrometheusVersion,
	gatherer prometheus.Gatherer,
	registerer prometheus.Registerer,
	statsRenderer StatsRenderer,
	rwEnabled bool,
	otlpEnabled bool,
) *API {
	a := &API{
		QueryEngine:       qe,
		Queryable:         q,
		ExemplarQueryable: eq,

		scrapePoolsRetriever:  spsr,
		targetRetriever:       tr,
		alertmanagerRetriever: ar,

		now:              time.Now,
		config:           configFunc,
		flagsMap:         flagsMap,
		ready:            readyFunc,
		globalURLOptions: globalURLOptions,
		db:               db,
		dbDir:            dbDir,
		enableAdmin:      enableAdmin,
		rulesRetriever:   rr,
		logger:           logger,
		CORSOrigin:       corsOrigin,
		runtimeInfo:      runtimeInfo,
		buildInfo:        buildInfo,
		gatherer:         gatherer,
		isAgent:          isAgent,
		statsRenderer:    defaultStatsRenderer,

		remoteReadHandler: remote.NewReadHandler(logger, registerer, q, configFunc, remoteReadSampleLimit, remoteReadConcurrencyLimit, remoteReadMaxBytesInFrame),
	}

	a.InstallCodec(JSONCodec{})

	if statsRenderer != nil {
		a.statsRenderer = statsRenderer
	}

	if ap == nil && (rwEnabled || otlpEnabled) {
		panic("remote write or otlp write enabled, but no appender passed in.")
	}

	if rwEnabled {
		a.remoteWriteHandler = remote.NewWriteHandler(logger, registerer, ap)
	}
	if otlpEnabled {
		a.otlpWriteHandler = remote.NewOTLPWriteHandler(logger, ap)
	}

	return a
}

// InstallCodec adds codec to this API's available codecs.
// Codecs installed first take precedence over codecs installed later when evaluating wildcards in Accept headers.
// The first installed codec is used as a fallback when the Accept header cannot be satisfied or if there is no Accept header.
func (api *API) InstallCodec(codec Codec) {
	api.codecs = append(api.codecs, codec)
}

// ClearCodecs removes all available codecs from this API, including the default codec installed by NewAPI.
func (api *API) ClearCodecs() {
	api.codecs = nil
}

func setUnavailStatusOnTSDBNotReady(r apiFuncResult) apiFuncResult {
	if r.err != nil && errors.Is(r.err.err, tsdb.ErrNotReady) {
		r.err.typ = errorUnavailable
	}
	return r
}

// Register the API's endpoints in the given router.
func (api *API) Register(r *route.Router) {
	wrap := func(f apiFunc) http.HandlerFunc {
		hf := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			httputil.SetCORS(w, api.CORSOrigin, r)
			result := setUnavailStatusOnTSDBNotReady(f(r))
			if result.finalizer != nil {
				defer result.finalizer()
			}
			if result.err != nil {
				api.respondError(w, result.err, result.data)
				return
			}

			if result.data != nil {
				api.respond(w, r, result.data, result.warnings, r.FormValue("query"))
				return
			}
			w.WriteHeader(http.StatusNoContent)
		})
		return api.ready(httputil.CompressionHandler{
			Handler: hf,
		}.ServeHTTP)
	}

	wrapAgent := func(f apiFunc) http.HandlerFunc {
		return wrap(func(r *http.Request) apiFuncResult {
			if api.isAgent {
				return apiFuncResult{nil, &apiError{errorExec, errors.New("unavailable with Prometheus Agent")}, nil, nil}
			}
			return f(r)
		})
	}

	r.Options("/*path", wrap(api.options))

	r.Get("/query", wrapAgent(api.query))
	r.Post("/query", wrapAgent(api.query))
	r.Get("/query_range", wrapAgent(api.queryRange))
	r.Post("/query_range", wrapAgent(api.queryRange))
	r.Get("/query_exemplars", wrapAgent(api.queryExemplars))
	r.Post("/query_exemplars", wrapAgent(api.queryExemplars))

	r.Get("/format_query", wrapAgent(api.formatQuery))
	r.Post("/format_query", wrapAgent(api.formatQuery))

	r.Get("/labels", wrapAgent(api.labelNames))
	r.Post("/labels", wrapAgent(api.labelNames))
	r.Get("/label/:name/values", wrapAgent(api.labelValues))

	r.Get("/series", wrapAgent(api.series))
	r.Post("/series", wrapAgent(api.series))
	r.Del("/series", wrapAgent(api.dropSeries))

	r.Get("/scrape_pools", wrap(api.scrapePools))
	r.Get("/targets", wrap(api.targets))
	r.Get("/targets/metadata", wrap(api.targetMetadata))
	r.Get("/alertmanagers", wrapAgent(api.alertmanagers))

	r.Get("/metadata", wrap(api.metricMetadata))

	r.Get("/status/config", wrap(api.serveConfig))
	r.Get("/status/runtimeinfo", wrap(api.serveRuntimeInfo))
	r.Get("/status/buildinfo", wrap(api.serveBuildInfo))
	r.Get("/status/flags", wrap(api.serveFlags))
	r.Get("/status/tsdb", wrapAgent(api.serveTSDBStatus))
	r.Get("/status/walreplay", api.serveWALReplayStatus)
	r.Post("/read", api.ready(api.remoteRead))
	r.Post("/write", api.ready(api.remoteWrite))
	r.Post("/otlp/v1/metrics", api.ready(api.otlpWrite))

	r.Get("/alerts", wrapAgent(api.alerts))
	r.Get("/rules", wrapAgent(api.rules))

	// Admin APIs
	r.Post("/admin/tsdb/delete_series", wrapAgent(api.deleteSeries))
	r.Post("/admin/tsdb/clean_tombstones", wrapAgent(api.cleanTombstones))
	r.Post("/admin/tsdb/snapshot", wrapAgent(api.snapshot))

	r.Put("/admin/tsdb/delete_series", wrapAgent(api.deleteSeries))
	r.Put("/admin/tsdb/clean_tombstones", wrapAgent(api.cleanTombstones))
	r.Put("/admin/tsdb/snapshot", wrapAgent(api.snapshot))
}

type QueryData struct {
	ResultType parser.ValueType `json:"resultType"`
	Result     parser.Value     `json:"result"`
	Stats      stats.QueryStats `json:"stats,omitempty"`
}

func invalidParamError(err error, parameter string) apiFuncResult {
	return apiFuncResult{nil, &apiError{
		errorBadData, fmt.Errorf("invalid parameter %q: %w", parameter, err),
	}, nil, nil}
}

func (api *API) options(*http.Request) apiFuncResult {
	return apiFuncResult{nil, nil, nil, nil}
}

func (api *API) query(r *http.Request) (result apiFuncResult) {
	ts, err := parseTimeParam(r, "time", api.now())
	if err != nil {
		return invalidParamError(err, "time")
	}
	ctx := r.Context()
	if to := r.FormValue("timeout"); to != "" {
		var cancel context.CancelFunc
		timeout, err := parseDuration(to)
		if err != nil {
			return invalidParamError(err, "timeout")
		}

		ctx, cancel = context.WithDeadline(ctx, api.now().Add(timeout))
		defer cancel()
	}

	opts, err := extractQueryOpts(r)
	if err != nil {
		return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
	}
	qry, err := api.QueryEngine.NewInstantQuery(ctx, api.Queryable, opts, r.FormValue("query"), ts)
	if err != nil {
		return invalidParamError(err, "query")
	}

	// From now on, we must only return with a finalizer in the result (to
	// be called by the caller) or call qry.Close ourselves (which is
	// required in the case of a panic).
	defer func() {
		if result.finalizer == nil {
			qry.Close()
		}
	}()

	ctx = httputil.ContextFromRequest(ctx, r)

	res := qry.Exec(ctx)
	if res.Err != nil {
		return apiFuncResult{nil, returnAPIError(res.Err), res.Warnings, qry.Close}
	}

	// Optional stats field in response if parameter "stats" is not empty.
	sr := api.statsRenderer
	if sr == nil {
		sr = defaultStatsRenderer
	}
	qs := sr(ctx, qry.Stats(), r.FormValue("stats"))

	return apiFuncResult{&QueryData{
		ResultType: res.Value.Type(),
		Result:     res.Value,
		Stats:      qs,
	}, nil, res.Warnings, qry.Close}
}

func (api *API) formatQuery(r *http.Request) (result apiFuncResult) {
	expr, err := parser.ParseExpr(r.FormValue("query"))
	if err != nil {
		return invalidParamError(err, "query")
	}

	return apiFuncResult{expr.Pretty(0), nil, nil, nil}
}

func extractQueryOpts(r *http.Request) (promql.QueryOpts, error) {
	var duration time.Duration

	if strDuration := r.FormValue("lookback_delta"); strDuration != "" {
		parsedDuration, err := parseDuration(strDuration)
		if err != nil {
			return nil, fmt.Errorf("error parsing lookback delta duration: %w", err)
		}
		duration = parsedDuration
	}

	return promql.NewPrometheusQueryOpts(r.FormValue("stats") == "all", duration), nil
}

func (api *API) queryRange(r *http.Request) (result apiFuncResult) {
	start, err := parseTime(r.FormValue("start"))
	if err != nil {
		return invalidParamError(err, "start")
	}
	end, err := parseTime(r.FormValue("end"))
	if err != nil {
		return invalidParamError(err, "end")
	}
	if end.Before(start) {
		return invalidParamError(errors.New("end timestamp must not be before start time"), "end")
	}

	step, err := parseDuration(r.FormValue("step"))
	if err != nil {
		return invalidParamError(err, "step")
	}

	if step <= 0 {
		return invalidParamError(errors.New("zero or negative query resolution step widths are not accepted. Try a positive integer"), "step")
	}

	// For safety, limit the number of returned points per timeseries.
	// This is sufficient for 60s resolution for a week or 1h resolution for a year.
	if end.Sub(start)/step > 11000 {
		err := errors.New("exceeded maximum resolution of 11,000 points per timeseries. Try decreasing the query resolution (?step=XX)")
		return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
	}

	ctx := r.Context()
	if to := r.FormValue("timeout"); to != "" {
		var cancel context.CancelFunc
		timeout, err := parseDuration(to)
		if err != nil {
			return invalidParamError(err, "timeout")
		}

		ctx, cancel = context.WithTimeout(ctx, timeout)
		defer cancel()
	}

	opts, err := extractQueryOpts(r)
	if err != nil {
		return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
	}
	qry, err := api.QueryEngine.NewRangeQuery(ctx, api.Queryable, opts, r.FormValue("query"), start, end, step)
	if err != nil {
		return invalidParamError(err, "query")
	}
	// From now on, we must only return with a finalizer in the result (to
	// be called by the caller) or call qry.Close ourselves (which is
	// required in the case of a panic).
	defer func() {
		if result.finalizer == nil {
			qry.Close()
		}
	}()

	ctx = httputil.ContextFromRequest(ctx, r)

	res := qry.Exec(ctx)
	if res.Err != nil {
		return apiFuncResult{nil, returnAPIError(res.Err), res.Warnings, qry.Close}
	}

	// Optional stats field in response if parameter "stats" is not empty.
	sr := api.statsRenderer
	if sr == nil {
		sr = defaultStatsRenderer
	}
	qs := sr(ctx, qry.Stats(), r.FormValue("stats"))

	return apiFuncResult{&QueryData{
		ResultType: res.Value.Type(),
		Result:     res.Value,
		Stats:      qs,
	}, nil, res.Warnings, qry.Close}
}

func (api *API) queryExemplars(r *http.Request) apiFuncResult {
	start, err := parseTimeParam(r, "start", MinTime)
	if err != nil {
		return invalidParamError(err, "start")
	}
	end, err := parseTimeParam(r, "end", MaxTime)
	if err != nil {
		return invalidParamError(err, "end")
	}
	if end.Before(start) {
		err := errors.New("end timestamp must not be before start timestamp")
		return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
	}

	expr, err := parser.ParseExpr(r.FormValue("query"))
	if err != nil {
		return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
	}

	selectors := parser.ExtractSelectors(expr)
	if len(selectors) < 1 {
		return apiFuncResult{nil, nil, nil, nil}
	}

	ctx := r.Context()
	eq, err := api.ExemplarQueryable.ExemplarQuerier(ctx)
	if err != nil {
		return apiFuncResult{nil, returnAPIError(err), nil, nil}
	}

	res, err := eq.Select(timestamp.FromTime(start), timestamp.FromTime(end), selectors...)
	if err != nil {
		return apiFuncResult{nil, returnAPIError(err), nil, nil}
	}

	return apiFuncResult{res, nil, nil, nil}
}

func returnAPIError(err error) *apiError {
	if err == nil {
		return nil
	}

	var eqc promql.ErrQueryCanceled
	var eqt promql.ErrQueryTimeout
	var es promql.ErrStorage
	switch {
	case errors.As(err, &eqc):
		return &apiError{errorCanceled, err}
	case errors.As(err, &eqt):
		return &apiError{errorTimeout, err}
	case errors.As(err, &es):
		return &apiError{errorInternal, err}
	}

	if errors.Is(err, context.Canceled) {
		return &apiError{errorCanceled, err}
	}

	return &apiError{errorExec, err}
}

func (api *API) labelNames(r *http.Request) apiFuncResult {
	limit, err := parseLimitParam(r.FormValue("limit"))
	if err != nil {
		return invalidParamError(err, "limit")
	}

	start, err := parseTimeParam(r, "start", MinTime)
	if err != nil {
		return invalidParamError(err, "start")
	}
	end, err := parseTimeParam(r, "end", MaxTime)
	if err != nil {
		return invalidParamError(err, "end")
	}

	matcherSets, err := parseMatchersParam(r.Form["match[]"])
	if err != nil {
		return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
	}

	q, err := api.Queryable.Querier(timestamp.FromTime(start), timestamp.FromTime(end))
	if err != nil {
		return apiFuncResult{nil, returnAPIError(err), nil, nil}
	}
	defer q.Close()

	var (
		names    []string
		warnings annotations.Annotations
	)
	if len(matcherSets) > 1 {
		labelNamesSet := make(map[string]struct{})

		for _, matchers := range matcherSets {
			vals, callWarnings, err := q.LabelNames(r.Context(), matchers...)
			if err != nil {
				return apiFuncResult{nil, returnAPIError(err), warnings, nil}
			}

			warnings.Merge(callWarnings)
			for _, val := range vals {
				labelNamesSet[val] = struct{}{}
			}
		}

		// Convert the map to an array.
		names = make([]string, 0, len(labelNamesSet))
		for key := range labelNamesSet {
			names = append(names, key)
		}
		slices.Sort(names)
	} else {
		var matchers []*labels.Matcher
		if len(matcherSets) == 1 {
			matchers = matcherSets[0]
		}
		names, warnings, err = q.LabelNames(r.Context(), matchers...)
		if err != nil {
			return apiFuncResult{nil, &apiError{errorExec, err}, warnings, nil}
		}
	}

	if names == nil {
		names = []string{}
	}

	if len(names) >= limit {
		names = names[:limit]
		warnings = warnings.Add(errors.New("results truncated due to limit"))
	}
	return apiFuncResult{names, nil, warnings, nil}
}

func (api *API) labelValues(r *http.Request) (result apiFuncResult) {
	ctx := r.Context()
	name := route.Param(ctx, "name")

	if !model.LabelNameRE.MatchString(name) {
		return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("invalid label name: %q", name)}, nil, nil}
	}

	limit, err := parseLimitParam(r.FormValue("limit"))
	if err != nil {
		return invalidParamError(err, "limit")
	}

	start, err := parseTimeParam(r, "start", MinTime)
	if err != nil {
		return invalidParamError(err, "start")
	}
	end, err := parseTimeParam(r, "end", MaxTime)
	if err != nil {
		return invalidParamError(err, "end")
	}

	matcherSets, err := parseMatchersParam(r.Form["match[]"])
	if err != nil {
		return apiFuncResult{nil, &apiError{errorBadData, err}, nil, nil}
	}

	q, err := api.Queryable.Querier(timestamp.FromTime(start), timestamp.FromTime(end))
	if err != nil {
		return apiFuncResult{nil, &apiError{errorExec, err}, nil, nil}
	}
	// From now on, we must only return with a finalizer in the result (to
	// be called by the caller) or call q.Close ourselves (which is required
	// in the case of a panic).
	defer func() {
		if result.finalizer == nil {
			q.Close()
		}
	}()
	closer := func() {
		q.Close()
	}

	var (
		vals     []string
		warnings annotations.Annotations
	)
	if len(matcherSets) > 1 {
		var callWarnings annotations.Annotations
		labelValuesSet := make(map[string]struct{})
		for _, matchers := range matcherSets {
			vals, callWarnings, err = q.LabelValues(ctx, name, matchers...)
			if err != nil {
				return apiFuncResult{nil, &apiError{errorExec, err}, warnings, closer}
			}
			warnings.Merge(callWarnings)
			for _, val := range vals {
				labelValuesSet[val] = struct{}{}
			}
		}

		vals = make([]string, 0, len(labelValuesSet))
		for val := range labelValuesSet {
			vals = append(vals, val)
		}
	} else {
		var matchers []*labels.Matcher
		if len(matcherSets) == 1 {
			matchers = matcherSets[0]
		}
		vals, warnings, err = q.LabelValues(ctx, name, matchers...)
		if err != nil {
			return apiFuncResult{nil, &apiError{errorExec, err}, warnings, closer}
		}

		if vals == nil {
			vals = []string{}
		}
	}

	slices.Sort(vals)

	if len(vals) >= limit {
		vals = vals[:limit]
		warnings = warnings.Add(errors.New("results truncated due to limit"))
	}

	return apiFuncResult{vals, nil, warnings, closer}
}

var (
	// MinTime is the default timestamp used for the begin of optional time ranges.
	// Exposed to let downstream projects to reference it.
	MinTime = time.Unix(math.MinInt64/1000+62135596801, 0).UTC()

	// MaxTime is the default timestamp used for the end of optional time ranges.
	// Exposed to let downstream projects to reference it.
	MaxTime = time.Unix(math.MaxInt64/1000-62135596801, 999999999).UTC()

	minTimeFormatted = MinTime.Format(time.RFC3339Nano)
	maxTimeFormatted = MaxTime.Format(time.RFC3339Nano)
)

func (api *API) series(r *http.Request) (result apiFuncResult) {
	ctx := r.Context()

	if err := r.ParseForm(); err != nil {
		return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error parsing form values: %w", err)}, nil, nil}
	}
	if len(r.Form["match[]"]) == 0 {
		return apiFuncResult{nil, &apiError{errorBadData, errors.New("no match[] parameter provided")}, nil, nil}
	}

	limit, err := parseLimitParam(r.FormValue("limit"))
	if err != nil {
		return invalidParamError(err, "limit")
	}

	start, err := parseTimeParam(r, "start", MinTime)
	if err != nil {
		return invalidParamError(err, "start")
	}
	end, err := parseTimeParam(r, "end", MaxTime)
	if err != nil {
		return invalidParamError(err, "end")
	}

	matcherSets, err := parseMatchersParam(r.Form["match[]"])
	if err != nil {
		return invalidParamError(err, "match[]")
	}

	q, err := api.Queryable.Querier(timestamp.FromTime(start), timestamp.FromTime(end))
	if err != nil {
		return apiFuncResult{nil, returnAPIError(err), nil, nil}
	}
	// From now on, we must only return with a finalizer in the result (to
	// be called by the caller) or call q.Close ourselves (which is required
	// in the case of a panic).
	defer func() {
		if result.finalizer == nil {
			q.Close()
		}
	}()
	closer := func() {
		q.Close()
	}

	hints := &storage.SelectHints{
		Start: timestamp.FromTime(start),
		End:   timestamp.FromTime(end),
		Func:  "series", // There is no series function, this token is used for lookups that don't need samples.
	}
	var set storage.SeriesSet

	if len(matcherSets) > 1 {
		var sets []storage.SeriesSet
		for _, mset := range matcherSets {
			// We need to sort this select results to merge (deduplicate) the series sets later.
			s := q.Select(ctx, true, hints, mset...)
			sets = append(sets, s)
		}
		set = storage.NewMergeSeriesSet(sets, storage.ChainedSeriesMerge)
	} else {
		// At this point at least one match exists.
		set = q.Select(ctx, false, hints, matcherSets[0]...)
	}

	metrics := []labels.Labels{}

	warnings := set.Warnings()

	for set.Next() {
		metrics = append(metrics, set.At().Labels())

		if len(metrics) >= limit {
			warnings.Add(errors.New("results truncated due to limit"))
			return apiFuncResult{metrics, nil, warnings, closer}
		}
	}
	if set.Err() != nil {
		return apiFuncResult{nil, returnAPIError(set.Err()), warnings, closer}
	}

	return apiFuncResult{metrics, nil, warnings, closer}
}

func (api *API) dropSeries(_ *http.Request) apiFuncResult {
	return apiFuncResult{nil, &apiError{errorInternal, errors.New("not implemented")}, nil, nil}
}

// Target has the information for one target.
type Target struct {
	// Labels before any processing.
	DiscoveredLabels labels.Labels `json:"discoveredLabels"`
	// Any labels that are added to this target and its metrics.
	Labels labels.Labels `json:"labels"`

	ScrapePool string `json:"scrapePool"`
	ScrapeURL  string `json:"scrapeUrl"`
	GlobalURL  string `json:"globalUrl"`

	LastError          string              `json:"lastError"`
	LastScrape         time.Time           `json:"lastScrape"`
	LastScrapeDuration float64             `json:"lastScrapeDuration"`
	Health             scrape.TargetHealth `json:"health"`

	ScrapeInterval string `json:"scrapeInterval"`
	ScrapeTimeout  string `json:"scrapeTimeout"`
}

type ScrapePoolsDiscovery struct {
	ScrapePools []string `json:"scrapePools"`
}

// DroppedTarget has the information for one target that was dropped during relabelling.
type DroppedTarget struct {
	// Labels before any processing.
	DiscoveredLabels labels.Labels `json:"discoveredLabels"`
}

// TargetDiscovery has all the active targets.
type TargetDiscovery struct {
	ActiveTargets       []*Target        `json:"activeTargets"`
	DroppedTargets      []*DroppedTarget `json:"droppedTargets"`
	DroppedTargetCounts map[string]int   `json:"droppedTargetCounts"`
}

// GlobalURLOptions contains fields used for deriving the global URL for local targets.
type GlobalURLOptions struct {
	ListenAddress string
	Host          string
	Scheme        string
}

// sanitizeSplitHostPort acts like net.SplitHostPort.
// Additionally, if there is no port in the host passed as input, we return the
// original host, making sure that IPv6 addresses are not surrounded by square
// brackets.
func sanitizeSplitHostPort(input string) (string, string, error) {
	host, port, err := net.SplitHostPort(input)
	if err != nil && strings.HasSuffix(err.Error(), "missing port in address") {
		var errWithPort error
		host, _, errWithPort = net.SplitHostPort(input + ":80")
		if errWithPort == nil {
			err = nil
		}
	}
	return host, port, err
}

func getGlobalURL(u *url.URL, opts GlobalURLOptions) (*url.URL, error) {
	host, port, err := sanitizeSplitHostPort(u.Host)
	if err != nil {
		return u, err
	}

	for _, lhr := range LocalhostRepresentations {
		if host == lhr {
			_, ownPort, err := net.SplitHostPort(opts.ListenAddress)
			if err != nil {
				return u, err
			}

			if port == ownPort {
				// Only in the case where the target is on localhost and its port is
				// the same as the one we're listening on, we know for sure that
				// we're monitoring our own process and that we need to change the
				// scheme, hostname, and port to the externally reachable ones as
				// well. We shouldn't need to touch the path at all, since if a
				// path prefix is defined, the path under which we scrape ourselves
				// should already contain the prefix.
				u.Scheme = opts.Scheme
				u.Host = opts.Host
			} else {
				// Otherwise, we only know that localhost is not reachable
				// externally, so we replace only the hostname by the one in the
				// external URL. It could be the wrong hostname for the service on
				// this port, but it's still the best possible guess.
				host, _, err := sanitizeSplitHostPort(opts.Host)
				if err != nil {
					return u, err
				}
				u.Host = host
				if port != "" {
					u.Host = net.JoinHostPort(u.Host, port)
				}
			}
			break
		}
	}

	return u, nil
}

func (api *API) scrapePools(r *http.Request) apiFuncResult {
	names := api.scrapePoolsRetriever(r.Context()).ScrapePools()
	sort.Strings(names)
	res := &ScrapePoolsDiscovery{ScrapePools: names}
	return apiFuncResult{data: res, err: nil, warnings: nil, finalizer: nil}
}

func (api *API) targets(r *http.Request) apiFuncResult {
	sortKeys := func(targets map[string][]*scrape.Target) ([]string, int) {
		var n int
		keys := make([]string, 0, len(targets))
		for k := range targets {
			keys = append(keys, k)
			n += len(targets[k])
		}
		slices.Sort(keys)
		return keys, n
	}

	scrapePool := r.URL.Query().Get("scrapePool")
	state := strings.ToLower(r.URL.Query().Get("state"))
	showActive := state == "" || state == "any" || state == "active"
	showDropped := state == "" || state == "any" || state == "dropped"
	res := &TargetDiscovery{}

	if showActive {
		targetsActive := api.targetRetriever(r.Context()).TargetsActive()
		activeKeys, numTargets := sortKeys(targetsActive)
		res.ActiveTargets = make([]*Target, 0, numTargets)
		builder := labels.NewScratchBuilder(0)

		for _, key := range activeKeys {
			if scrapePool != "" && key != scrapePool {
				continue
			}
			for _, target := range targetsActive[key] {
				lastErrStr := ""
				lastErr := target.LastError()
				if lastErr != nil {
					lastErrStr = lastErr.Error()
				}

				globalURL, err := getGlobalURL(target.URL(), api.globalURLOptions)

				res.ActiveTargets = append(res.ActiveTargets, &Target{
					DiscoveredLabels: target.DiscoveredLabels(),
					Labels:           target.Labels(&builder),
					ScrapePool:       key,
					ScrapeURL:        target.URL().String(),
					GlobalURL:        globalURL.String(),
					LastError: func() string {
						switch {
						case err == nil && lastErrStr == "":
							return ""
						case err != nil:
							return fmt.Errorf("%s: %w", lastErrStr, err).Error()
						default:
							return lastErrStr
						}
					}(),
					LastScrape:         target.LastScrape(),
					LastScrapeDuration: target.LastScrapeDuration().Seconds(),
					Health:             target.Health(),
					ScrapeInterval:     target.GetValue(model.ScrapeIntervalLabel),
					ScrapeTimeout:      target.GetValue(model.ScrapeTimeoutLabel),
				})
			}
		}
	} else {
		res.ActiveTargets = []*Target{}
	}
	if showDropped {
		res.DroppedTargetCounts = api.targetRetriever(r.Context()).TargetsDroppedCounts()
	}
	if showDropped {
		targetsDropped := api.targetRetriever(r.Context()).TargetsDropped()
		droppedKeys, numTargets := sortKeys(targetsDropped)
		res.DroppedTargets = make([]*DroppedTarget, 0, numTargets)
		for _, key := range droppedKeys {
			if scrapePool != "" && key != scrapePool {
				continue
			}
			for _, target := range targetsDropped[key] {
				res.DroppedTargets = append(res.DroppedTargets, &DroppedTarget{
					DiscoveredLabels: target.DiscoveredLabels(),
				})
			}
		}
	} else {
		res.DroppedTargets = []*DroppedTarget{}
	}
	return apiFuncResult{res, nil, nil, nil}
}

func matchLabels(lset labels.Labels, matchers []*labels.Matcher) bool {
	for _, m := range matchers {
		if !m.Matches(lset.Get(m.Name)) {
			return false
		}
	}
	return true
}

func (api *API) targetMetadata(r *http.Request) apiFuncResult {
	limit := -1
	if s := r.FormValue("limit"); s != "" {
		var err error
		if limit, err = strconv.Atoi(s); err != nil {
			return apiFuncResult{nil, &apiError{errorBadData, errors.New("limit must be a number")}, nil, nil}
		}
	}

	matchTarget := r.FormValue("match_target")
	var matchers []*labels.Matcher
	var err error
	if matchTarget != "" {
		matchers, err = parser.ParseMetricSelector(matchTarget)
		if err != nil {
			return invalidParamError(err, "match_target")
		}
	}

	builder := labels.NewScratchBuilder(0)
	metric := r.FormValue("metric")
	res := []metricMetadata{}
	for _, tt := range api.targetRetriever(r.Context()).TargetsActive() {
		for _, t := range tt {
			if limit >= 0 && len(res) >= limit {
				break
			}
			targetLabels := t.Labels(&builder)
			// Filter targets that don't satisfy the label matchers.
			if matchTarget != "" && !matchLabels(targetLabels, matchers) {
				continue
			}
			// If no metric is specified, get the full list for the target.
			if metric == "" {
				for _, md := range t.ListMetadata() {
					res = append(res, metricMetadata{
						Target: targetLabels,
						Metric: md.Metric,
						Type:   md.Type,
						Help:   md.Help,
						Unit:   md.Unit,
					})
				}
				continue
			}
			// Get metadata for the specified metric.
			if md, ok := t.GetMetadata(metric); ok {
				res = append(res, metricMetadata{
					Target: targetLabels,
					Type:   md.Type,
					Help:   md.Help,
					Unit:   md.Unit,
				})
			}
		}
	}

	return apiFuncResult{res, nil, nil, nil}
}

type metricMetadata struct {
	Target labels.Labels    `json:"target"`
	Metric string           `json:"metric,omitempty"`
	Type   model.MetricType `json:"type"`
	Help   string           `json:"help"`
	Unit   string           `json:"unit"`
}

// AlertmanagerDiscovery has all the active Alertmanagers.
type AlertmanagerDiscovery struct {
	ActiveAlertmanagers  []*AlertmanagerTarget `json:"activeAlertmanagers"`
	DroppedAlertmanagers []*AlertmanagerTarget `json:"droppedAlertmanagers"`
}

// AlertmanagerTarget has info on one AM.
type AlertmanagerTarget struct {
	URL string `json:"url"`
}

func (api *API) alertmanagers(r *http.Request) apiFuncResult {
	urls := api.alertmanagerRetriever(r.Context()).Alertmanagers()
	droppedURLS := api.alertmanagerRetriever(r.Context()).DroppedAlertmanagers()
	ams := &AlertmanagerDiscovery{ActiveAlertmanagers: make([]*AlertmanagerTarget, len(urls)), DroppedAlertmanagers: make([]*AlertmanagerTarget, len(droppedURLS))}
	for i, url := range urls {
		ams.ActiveAlertmanagers[i] = &AlertmanagerTarget{URL: url.String()}
	}
	for i, url := range droppedURLS {
		ams.DroppedAlertmanagers[i] = &AlertmanagerTarget{URL: url.String()}
	}
	return apiFuncResult{ams, nil, nil, nil}
}

// AlertDiscovery has info for all active alerts.
type AlertDiscovery struct {
	Alerts []*Alert `json:"alerts"`
}

// Alert has info for an alert.
type Alert struct {
	Labels          labels.Labels `json:"labels"`
	Annotations     labels.Labels `json:"annotations"`
	State           string        `json:"state"`
	ActiveAt        *time.Time    `json:"activeAt,omitempty"`
	KeepFiringSince *time.Time    `json:"keepFiringSince,omitempty"`
	Value           string        `json:"value"`
}

func (api *API) alerts(r *http.Request) apiFuncResult {
	alertingRules := api.rulesRetriever(r.Context()).AlertingRules()
	alerts := []*Alert{}

	for _, alertingRule := range alertingRules {
		alerts = append(
			alerts,
			rulesAlertsToAPIAlerts(alertingRule.ActiveAlerts())...,
		)
	}

	res := &AlertDiscovery{Alerts: alerts}

	return apiFuncResult{res, nil, nil, nil}
}

func rulesAlertsToAPIAlerts(rulesAlerts []*rules.Alert) []*Alert {
	apiAlerts := make([]*Alert, len(rulesAlerts))
	for i, ruleAlert := range rulesAlerts {
		apiAlerts[i] = &Alert{
			Labels:      ruleAlert.Labels,
			Annotations: ruleAlert.Annotations,
			State:       ruleAlert.State.String(),
			ActiveAt:    &ruleAlert.ActiveAt,
			Value:       strconv.FormatFloat(ruleAlert.Value, 'e', -1, 64),
		}
		if !ruleAlert.KeepFiringSince.IsZero() {
			apiAlerts[i].KeepFiringSince = &ruleAlert.KeepFiringSince
		}
	}

	return apiAlerts
}

func (api *API) metricMetadata(r *http.Request) apiFuncResult {
	metrics := map[string]map[metadata.Metadata]struct{}{}

	limit := -1
	if s := r.FormValue("limit"); s != "" {
		var err error
		if limit, err = strconv.Atoi(s); err != nil {
			return apiFuncResult{nil, &apiError{errorBadData, errors.New("limit must be a number")}, nil, nil}
		}
	}
	limitPerMetric := -1
	if s := r.FormValue("limit_per_metric"); s != "" {
		var err error
		if limitPerMetric, err = strconv.Atoi(s); err != nil {
			return apiFuncResult{nil, &apiError{errorBadData, errors.New("limit_per_metric must be a number")}, nil, nil}
		}
	}

	metric := r.FormValue("metric")
	for _, tt := range api.targetRetriever(r.Context()).TargetsActive() {
		for _, t := range tt {
			if metric == "" {
				for _, mm := range t.ListMetadata() {
					m := metadata.Metadata{Type: mm.Type, Help: mm.Help, Unit: mm.Unit}
					ms, ok := metrics[mm.Metric]

					if limitPerMetric > 0 && len(ms) >= limitPerMetric {
						continue
					}

					if !ok {
						ms = map[metadata.Metadata]struct{}{}
						metrics[mm.Metric] = ms
					}
					ms[m] = struct{}{}
				}
				continue
			}

			if md, ok := t.GetMetadata(metric); ok {
				m := metadata.Metadata{Type: md.Type, Help: md.Help, Unit: md.Unit}
				ms, ok := metrics[md.Metric]

				if limitPerMetric > 0 && len(ms) >= limitPerMetric {
					continue
				}

				if !ok {
					ms = map[metadata.Metadata]struct{}{}
					metrics[md.Metric] = ms
				}
				ms[m] = struct{}{}
			}
		}
	}

	// Put the elements from the pseudo-set into a slice for marshaling.
	res := map[string][]metadata.Metadata{}
	for name, set := range metrics {
		if limit >= 0 && len(res) >= limit {
			break
		}

		s := []metadata.Metadata{}
		for metadata := range set {
			s = append(s, metadata)
		}
		res[name] = s
	}

	return apiFuncResult{res, nil, nil, nil}
}

// RuleDiscovery has info for all rules.
type RuleDiscovery struct {
	RuleGroups []*RuleGroup `json:"groups"`
}

// RuleGroup has info for rules which are part of a group.
type RuleGroup struct {
	Name string `json:"name"`
	File string `json:"file"`
	// In order to preserve rule ordering, while exposing type (alerting or recording)
	// specific properties, both alerting and recording rules are exposed in the
	// same array.
	Rules          []Rule    `json:"rules"`
	Interval       float64   `json:"interval"`
	Limit          int       `json:"limit"`
	EvaluationTime float64   `json:"evaluationTime"`
	LastEvaluation time.Time `json:"lastEvaluation"`
}

type Rule interface{}

type AlertingRule struct {
	// State can be "pending", "firing", "inactive".
	State          string           `json:"state"`
	Name           string           `json:"name"`
	Query          string           `json:"query"`
	Duration       float64          `json:"duration"`
	KeepFiringFor  float64          `json:"keepFiringFor"`
	Labels         labels.Labels    `json:"labels"`
	Annotations    labels.Labels    `json:"annotations"`
	Alerts         []*Alert         `json:"alerts"`
	Health         rules.RuleHealth `json:"health"`
	LastError      string           `json:"lastError,omitempty"`
	EvaluationTime float64          `json:"evaluationTime"`
	LastEvaluation time.Time        `json:"lastEvaluation"`
	// Type of an alertingRule is always "alerting".
	Type string `json:"type"`
}

type RecordingRule struct {
	Name           string           `json:"name"`
	Query          string           `json:"query"`
	Labels         labels.Labels    `json:"labels,omitempty"`
	Health         rules.RuleHealth `json:"health"`
	LastError      string           `json:"lastError,omitempty"`
	EvaluationTime float64          `json:"evaluationTime"`
	LastEvaluation time.Time        `json:"lastEvaluation"`
	// Type of a recordingRule is always "recording".
	Type string `json:"type"`
}

func (api *API) rules(r *http.Request) apiFuncResult {
	if err := r.ParseForm(); err != nil {
		return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error parsing form values: %w", err)}, nil, nil}
	}

	queryFormToSet := func(values []string) map[string]struct{} {
		set := make(map[string]struct{}, len(values))
		for _, v := range values {
			set[v] = struct{}{}
		}
		return set
	}

	rnSet := queryFormToSet(r.Form["rule_name[]"])
	rgSet := queryFormToSet(r.Form["rule_group[]"])
	fSet := queryFormToSet(r.Form["file[]"])

	ruleGroups := api.rulesRetriever(r.Context()).RuleGroups()
	res := &RuleDiscovery{RuleGroups: make([]*RuleGroup, 0, len(ruleGroups))}
	typ := strings.ToLower(r.URL.Query().Get("type"))

	if typ != "" && typ != "alert" && typ != "record" {
		return invalidParamError(fmt.Errorf("not supported value %q", typ), "type")
	}

	returnAlerts := typ == "" || typ == "alert"
	returnRecording := typ == "" || typ == "record"

	excludeAlerts, err := parseExcludeAlerts(r)
	if err != nil {
		return invalidParamError(err, "exclude_alerts")
	}

	rgs := make([]*RuleGroup, 0, len(ruleGroups))
	for _, grp := range ruleGroups {
		if len(rgSet) > 0 {
			if _, ok := rgSet[grp.Name()]; !ok {
				continue
			}
		}

		if len(fSet) > 0 {
			if _, ok := fSet[grp.File()]; !ok {
				continue
			}
		}

		apiRuleGroup := &RuleGroup{
			Name:           grp.Name(),
			File:           grp.File(),
			Interval:       grp.Interval().Seconds(),
			Limit:          grp.Limit(),
			Rules:          []Rule{},
			EvaluationTime: grp.GetEvaluationTime().Seconds(),
			LastEvaluation: grp.GetLastEvaluation(),
		}
		for _, rr := range grp.Rules() {
			var enrichedRule Rule

			if len(rnSet) > 0 {
				if _, ok := rnSet[rr.Name()]; !ok {
					continue
				}
			}

			lastError := ""
			if rr.LastError() != nil {
				lastError = rr.LastError().Error()
			}
			switch rule := rr.(type) {
			case *rules.AlertingRule:
				if !returnAlerts {
					break
				}
				var activeAlerts []*Alert
				if !excludeAlerts {
					activeAlerts = rulesAlertsToAPIAlerts(rule.ActiveAlerts())
				}
				enrichedRule = AlertingRule{
					State:          rule.State().String(),
					Name:           rule.Name(),
					Query:          rule.Query().String(),
					Duration:       rule.HoldDuration().Seconds(),
					KeepFiringFor:  rule.KeepFiringFor().Seconds(),
					Labels:         rule.Labels(),
					Annotations:    rule.Annotations(),
					Alerts:         activeAlerts,
					Health:         rule.Health(),
					LastError:      lastError,
					EvaluationTime: rule.GetEvaluationDuration().Seconds(),
					LastEvaluation: rule.GetEvaluationTimestamp(),
					Type:           "alerting",
				}
			case *rules.RecordingRule:
				if !returnRecording {
					break
				}
				enrichedRule = RecordingRule{
					Name:           rule.Name(),
					Query:          rule.Query().String(),
					Labels:         rule.Labels(),
					Health:         rule.Health(),
					LastError:      lastError,
					EvaluationTime: rule.GetEvaluationDuration().Seconds(),
					LastEvaluation: rule.GetEvaluationTimestamp(),
					Type:           "recording",
				}
			default:
				err := fmt.Errorf("failed to assert type of rule '%v'", rule.Name())
				return apiFuncResult{nil, &apiError{errorInternal, err}, nil, nil}
			}

			if enrichedRule != nil {
				apiRuleGroup.Rules = append(apiRuleGroup.Rules, enrichedRule)
			}
		}

		// If the rule group response has no rules, skip it - this means we filtered all the rules of this group.
		if len(apiRuleGroup.Rules) > 0 {
			rgs = append(rgs, apiRuleGroup)
		}
	}
	res.RuleGroups = rgs
	return apiFuncResult{res, nil, nil, nil}
}

func parseExcludeAlerts(r *http.Request) (bool, error) {
	excludeAlertsParam := strings.ToLower(r.URL.Query().Get("exclude_alerts"))

	if excludeAlertsParam == "" {
		return false, nil
	}

	excludeAlerts, err := strconv.ParseBool(excludeAlertsParam)
	if err != nil {
		return false, fmt.Errorf("error converting exclude_alerts: %w", err)
	}
	return excludeAlerts, nil
}

type prometheusConfig struct {
	YAML string `json:"yaml"`
}

func (api *API) serveRuntimeInfo(_ *http.Request) apiFuncResult {
	status, err := api.runtimeInfo()
	if err != nil {
		return apiFuncResult{status, &apiError{errorInternal, err}, nil, nil}
	}
	return apiFuncResult{status, nil, nil, nil}
}

func (api *API) serveBuildInfo(_ *http.Request) apiFuncResult {
	return apiFuncResult{api.buildInfo, nil, nil, nil}
}

func (api *API) serveConfig(_ *http.Request) apiFuncResult {
	cfg := &prometheusConfig{
		YAML: api.config().String(),
	}
	return apiFuncResult{cfg, nil, nil, nil}
}

func (api *API) serveFlags(_ *http.Request) apiFuncResult {
	return apiFuncResult{api.flagsMap, nil, nil, nil}
}

// TSDBStat holds the information about individual cardinality.
type TSDBStat struct {
	Name  string `json:"name"`
	Value uint64 `json:"value"`
}

// HeadStats has information about the TSDB head.
type HeadStats struct {
	NumSeries     uint64 `json:"numSeries"`
	NumLabelPairs int    `json:"numLabelPairs"`
	ChunkCount    int64  `json:"chunkCount"`
	MinTime       int64  `json:"minTime"`
	MaxTime       int64  `json:"maxTime"`
}

// TSDBStatus has information of cardinality statistics from postings.
type TSDBStatus struct {
	HeadStats                   HeadStats  `json:"headStats"`
	SeriesCountByMetricName     []TSDBStat `json:"seriesCountByMetricName"`
	LabelValueCountByLabelName  []TSDBStat `json:"labelValueCountByLabelName"`
	MemoryInBytesByLabelName    []TSDBStat `json:"memoryInBytesByLabelName"`
	SeriesCountByLabelValuePair []TSDBStat `json:"seriesCountByLabelValuePair"`
}

// TSDBStatsFromIndexStats converts a index.Stat slice to a TSDBStat slice.
func TSDBStatsFromIndexStats(stats []index.Stat) []TSDBStat {
	result := make([]TSDBStat, 0, len(stats))
	for _, item := range stats {
		item := TSDBStat{Name: item.Name, Value: item.Count}
		result = append(result, item)
	}
	return result
}

func (api *API) serveTSDBStatus(r *http.Request) apiFuncResult {
	limit := 10
	if s := r.FormValue("limit"); s != "" {
		var err error
		if limit, err = strconv.Atoi(s); err != nil || limit < 1 {
			return apiFuncResult{nil, &apiError{errorBadData, errors.New("limit must be a positive number")}, nil, nil}
		}
	}
	s, err := api.db.Stats(labels.MetricName, limit)
	if err != nil {
		return apiFuncResult{nil, &apiError{errorInternal, err}, nil, nil}
	}
	metrics, err := api.gatherer.Gather()
	if err != nil {
		return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("error gathering runtime status: %w", err)}, nil, nil}
	}
	chunkCount := int64(math.NaN())
	for _, mF := range metrics {
		if *mF.Name == "prometheus_tsdb_head_chunks" {
			m := mF.Metric[0]
			if m.Gauge != nil {
				chunkCount = int64(m.Gauge.GetValue())
				break
			}
		}
	}
	return apiFuncResult{TSDBStatus{
		HeadStats: HeadStats{
			NumSeries:     s.NumSeries,
			ChunkCount:    chunkCount,
			MinTime:       s.MinTime,
			MaxTime:       s.MaxTime,
			NumLabelPairs: s.IndexPostingStats.NumLabelPairs,
		},
		SeriesCountByMetricName:     TSDBStatsFromIndexStats(s.IndexPostingStats.CardinalityMetricsStats),
		LabelValueCountByLabelName:  TSDBStatsFromIndexStats(s.IndexPostingStats.CardinalityLabelStats),
		MemoryInBytesByLabelName:    TSDBStatsFromIndexStats(s.IndexPostingStats.LabelValueStats),
		SeriesCountByLabelValuePair: TSDBStatsFromIndexStats(s.IndexPostingStats.LabelValuePairsStats),
	}, nil, nil, nil}
}

type walReplayStatus struct {
	Min     int `json:"min"`
	Max     int `json:"max"`
	Current int `json:"current"`
}

func (api *API) serveWALReplayStatus(w http.ResponseWriter, r *http.Request) {
	httputil.SetCORS(w, api.CORSOrigin, r)
	status, err := api.db.WALReplayStatus()
	if err != nil {
		api.respondError(w, &apiError{errorInternal, err}, nil)
	}
	api.respond(w, r, walReplayStatus{
		Min:     status.Min,
		Max:     status.Max,
		Current: status.Current,
	}, nil, "")
}

func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) {
	// This is only really for tests - this will never be nil IRL.
	if api.remoteReadHandler != nil {
		api.remoteReadHandler.ServeHTTP(w, r)
	} else {
		http.Error(w, "not found", http.StatusNotFound)
	}
}

func (api *API) remoteWrite(w http.ResponseWriter, r *http.Request) {
	if api.remoteWriteHandler != nil {
		api.remoteWriteHandler.ServeHTTP(w, r)
	} else {
		http.Error(w, "remote write receiver needs to be enabled with --web.enable-remote-write-receiver", http.StatusNotFound)
	}
}

func (api *API) otlpWrite(w http.ResponseWriter, r *http.Request) {
	if api.otlpWriteHandler != nil {
		api.otlpWriteHandler.ServeHTTP(w, r)
	} else {
		http.Error(w, "otlp write receiver needs to be enabled with --enable-feature=otlp-write-receiver", http.StatusNotFound)
	}
}

func (api *API) deleteSeries(r *http.Request) apiFuncResult {
	if !api.enableAdmin {
		return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("admin APIs disabled")}, nil, nil}
	}
	if err := r.ParseForm(); err != nil {
		return apiFuncResult{nil, &apiError{errorBadData, fmt.Errorf("error parsing form values: %w", err)}, nil, nil}
	}
	if len(r.Form["match[]"]) == 0 {
		return apiFuncResult{nil, &apiError{errorBadData, errors.New("no match[] parameter provided")}, nil, nil}
	}

	start, err := parseTimeParam(r, "start", MinTime)
	if err != nil {
		return invalidParamError(err, "start")
	}
	end, err := parseTimeParam(r, "end", MaxTime)
	if err != nil {
		return invalidParamError(err, "end")
	}

	for _, s := range r.Form["match[]"] {
		matchers, err := parser.ParseMetricSelector(s)
		if err != nil {
			return invalidParamError(err, "match[]")
		}
		if err := api.db.Delete(r.Context(), timestamp.FromTime(start), timestamp.FromTime(end), matchers...); err != nil {
			return apiFuncResult{nil, &apiError{errorInternal, err}, nil, nil}
		}
	}

	return apiFuncResult{nil, nil, nil, nil}
}

func (api *API) snapshot(r *http.Request) apiFuncResult {
	if !api.enableAdmin {
		return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("admin APIs disabled")}, nil, nil}
	}
	var (
		skipHead bool
		err      error
	)
	if r.FormValue("skip_head") != "" {
		skipHead, err = strconv.ParseBool(r.FormValue("skip_head"))
		if err != nil {
			return invalidParamError(fmt.Errorf("unable to parse boolean: %w", err), "skip_head")
		}
	}

	var (
		snapdir = filepath.Join(api.dbDir, "snapshots")
		name    = fmt.Sprintf("%s-%016x",
			time.Now().UTC().Format("20060102T150405Z0700"),
			rand.Int63())
		dir = filepath.Join(snapdir, name)
	)
	if err := os.MkdirAll(dir, 0o777); err != nil {
		return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("create snapshot directory: %w", err)}, nil, nil}
	}
	if err := api.db.Snapshot(dir, !skipHead); err != nil {
		return apiFuncResult{nil, &apiError{errorInternal, fmt.Errorf("create snapshot: %w", err)}, nil, nil}
	}

	return apiFuncResult{struct {
		Name string `json:"name"`
	}{name}, nil, nil, nil}
}

func (api *API) cleanTombstones(*http.Request) apiFuncResult {
	if !api.enableAdmin {
		return apiFuncResult{nil, &apiError{errorUnavailable, errors.New("admin APIs disabled")}, nil, nil}
	}
	if err := api.db.CleanTombstones(); err != nil {
		return apiFuncResult{nil, &apiError{errorInternal, err}, nil, nil}
	}

	return apiFuncResult{nil, nil, nil, nil}
}

// Query string is needed to get the position information for the annotations, and it
// can be empty if the position information isn't needed.
func (api *API) respond(w http.ResponseWriter, req *http.Request, data interface{}, warnings annotations.Annotations, query string) {
	statusMessage := statusSuccess

	resp := &Response{
		Status:   statusMessage,
		Data:     data,
		Warnings: warnings.AsStrings(query, 10),
	}

	codec, err := api.negotiateCodec(req, resp)
	if err != nil {
		api.respondError(w, &apiError{errorNotAcceptable, err}, nil)
		return
	}

	b, err := codec.Encode(resp)
	if err != nil {
		level.Error(api.logger).Log("msg", "error marshaling response", "err", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Set("Content-Type", codec.ContentType().String())
	w.WriteHeader(http.StatusOK)
	if n, err := w.Write(b); err != nil {
		level.Error(api.logger).Log("msg", "error writing response", "bytesWritten", n, "err", err)
	}
}

func (api *API) negotiateCodec(req *http.Request, resp *Response) (Codec, error) {
	for _, clause := range goautoneg.ParseAccept(req.Header.Get("Accept")) {
		for _, codec := range api.codecs {
			if codec.ContentType().Satisfies(clause) && codec.CanEncode(resp) {
				return codec, nil
			}
		}
	}

	defaultCodec := api.codecs[0]
	if !defaultCodec.CanEncode(resp) {
		return nil, fmt.Errorf("cannot encode response as %s", defaultCodec.ContentType())
	}

	return defaultCodec, nil
}

func (api *API) respondError(w http.ResponseWriter, apiErr *apiError, data interface{}) {
	json := jsoniter.ConfigCompatibleWithStandardLibrary
	b, err := json.Marshal(&Response{
		Status:    statusError,
		ErrorType: apiErr.typ,
		Error:     apiErr.err.Error(),
		Data:      data,
	})
	if err != nil {
		level.Error(api.logger).Log("msg", "error marshaling json response", "err", err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	var code int
	switch apiErr.typ {
	case errorBadData:
		code = http.StatusBadRequest
	case errorExec:
		code = http.StatusUnprocessableEntity
	case errorCanceled:
		code = statusClientClosedConnection
	case errorTimeout:
		code = http.StatusServiceUnavailable
	case errorInternal:
		code = http.StatusInternalServerError
	case errorNotFound:
		code = http.StatusNotFound
	case errorNotAcceptable:
		code = http.StatusNotAcceptable
	default:
		code = http.StatusInternalServerError
	}

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(code)
	if n, err := w.Write(b); err != nil {
		level.Error(api.logger).Log("msg", "error writing response", "bytesWritten", n, "err", err)
	}
}

func parseTimeParam(r *http.Request, paramName string, defaultValue time.Time) (time.Time, error) {
	val := r.FormValue(paramName)
	if val == "" {
		return defaultValue, nil
	}
	result, err := parseTime(val)
	if err != nil {
		return time.Time{}, fmt.Errorf("Invalid time value for '%s': %w", paramName, err)
	}
	return result, nil
}

func parseTime(s string) (time.Time, error) {
	if t, err := strconv.ParseFloat(s, 64); err == nil {
		s, ns := math.Modf(t)
		ns = math.Round(ns*1000) / 1000
		return time.Unix(int64(s), int64(ns*float64(time.Second))).UTC(), nil
	}
	if t, err := time.Parse(time.RFC3339Nano, s); err == nil {
		return t, nil
	}

	// Stdlib's time parser can only handle 4 digit years. As a workaround until
	// that is fixed we want to at least support our own boundary times.
	// Context: https://github.com/prometheus/client_golang/issues/614
	// Upstream issue: https://github.com/golang/go/issues/20555
	switch s {
	case minTimeFormatted:
		return MinTime, nil
	case maxTimeFormatted:
		return MaxTime, nil
	}
	return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
}

func parseDuration(s string) (time.Duration, error) {
	if d, err := strconv.ParseFloat(s, 64); err == nil {
		ts := d * float64(time.Second)
		if ts > float64(math.MaxInt64) || ts < float64(math.MinInt64) {
			return 0, fmt.Errorf("cannot parse %q to a valid duration. It overflows int64", s)
		}
		return time.Duration(ts), nil
	}
	if d, err := model.ParseDuration(s); err == nil {
		return time.Duration(d), nil
	}
	return 0, fmt.Errorf("cannot parse %q to a valid duration", s)
}

func parseMatchersParam(matchers []string) ([][]*labels.Matcher, error) {
	matcherSets, err := parser.ParseMetricSelectors(matchers)
	if err != nil {
		return nil, err
	}

OUTER:
	for _, ms := range matcherSets {
		for _, lm := range ms {
			if lm != nil && !lm.Matches("") {
				continue OUTER
			}
		}
		return nil, errors.New("match[] must contain at least one non-empty matcher")
	}
	return matcherSets, nil
}

func parseLimitParam(limitStr string) (limit int, err error) {
	limit = math.MaxInt
	if limitStr == "" {
		return limit, nil
	}

	limit, err = strconv.Atoi(limitStr)
	if err != nil {
		return limit, err
	}
	if limit <= 0 {
		return limit, errors.New("limit must be positive")
	}

	return limit, nil
}