mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-24 13:14:05 -08:00
mantine UI: Distinguish between Not Ready and Stopping
Signed-off-by: Julien <roidelapluie@o11y.eu>
This commit is contained in:
parent
f007659f03
commit
ac5377873f
|
@ -980,7 +980,7 @@ func main() {
|
||||||
},
|
},
|
||||||
func(err error) {
|
func(err error) {
|
||||||
close(cancel)
|
close(cancel)
|
||||||
webHandler.SetReady(false)
|
webHandler.SetReady(web.Stopping)
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
@ -1159,7 +1159,7 @@ func main() {
|
||||||
|
|
||||||
reloadReady.Close()
|
reloadReady.Close()
|
||||||
|
|
||||||
webHandler.SetReady(true)
|
webHandler.SetReady(web.Ready)
|
||||||
level.Info(logger).Log("msg", "Server is ready to receive web requests.")
|
level.Info(logger).Log("msg", "Server is ready to receive web requests.")
|
||||||
<-cancel
|
<-cancel
|
||||||
return nil
|
return nil
|
||||||
|
|
|
@ -1,17 +1,23 @@
|
||||||
import { FC, PropsWithChildren, useEffect, useState } from "react";
|
import { FC, PropsWithChildren, useEffect, useState } from "react";
|
||||||
|
import { IconAlertTriangle } from "@tabler/icons-react";
|
||||||
import { useAppDispatch } from "../state/hooks";
|
import { useAppDispatch } from "../state/hooks";
|
||||||
import { updateSettings, useSettings } from "../state/settingsSlice";
|
import { updateSettings, useSettings } from "../state/settingsSlice";
|
||||||
import { useSuspenseAPIQuery } from "../api/api";
|
import { useSuspenseAPIQuery } from "../api/api";
|
||||||
import { WALReplayStatus } from "../api/responseTypes/walreplay";
|
import { WALReplayStatus } from "../api/responseTypes/walreplay";
|
||||||
import { Progress, Stack, Title } from "@mantine/core";
|
import { Progress, Alert } from "@mantine/core";
|
||||||
import { useSuspenseQuery } from "@tanstack/react-query";
|
import { useSuspenseQuery } from "@tanstack/react-query";
|
||||||
|
|
||||||
|
const STATUS_STARTING = "is starting up...";
|
||||||
|
const STATUS_STOPPING = "is shutting down...";
|
||||||
|
const STATUS_LOADING = "is not ready...";
|
||||||
|
|
||||||
const ReadinessLoader: FC = () => {
|
const ReadinessLoader: FC = () => {
|
||||||
const { pathPrefix } = useSettings();
|
const { pathPrefix, agentMode } = useSettings();
|
||||||
const dispatch = useAppDispatch();
|
const dispatch = useAppDispatch();
|
||||||
|
|
||||||
// Query key is incremented every second to retrigger the status fetching.
|
// Query key is incremented every second to retrigger the status fetching.
|
||||||
const [queryKey, setQueryKey] = useState(0);
|
const [queryKey, setQueryKey] = useState(0);
|
||||||
|
const [statusMessage, setStatusMessage] = useState("");
|
||||||
|
|
||||||
// Query readiness status.
|
// Query readiness status.
|
||||||
const { data: ready } = useSuspenseQuery<boolean>({
|
const { data: ready } = useSuspenseQuery<boolean>({
|
||||||
|
@ -28,8 +34,16 @@ const ReadinessLoader: FC = () => {
|
||||||
});
|
});
|
||||||
switch (res.status) {
|
switch (res.status) {
|
||||||
case 200:
|
case 200:
|
||||||
|
setStatusMessage(""); // Clear any status message when ready.
|
||||||
return true;
|
return true;
|
||||||
case 503:
|
case 503:
|
||||||
|
// Check the custom header `X-Prometheus-Stopping` for stopping information.
|
||||||
|
if (res.headers.get("X-Prometheus-Stopping") === "true") {
|
||||||
|
setStatusMessage(STATUS_STOPPING);
|
||||||
|
} else {
|
||||||
|
setStatusMessage(STATUS_STARTING);
|
||||||
|
}
|
||||||
|
|
||||||
return false;
|
return false;
|
||||||
default:
|
default:
|
||||||
throw new Error(res.statusText);
|
throw new Error(res.statusText);
|
||||||
|
@ -40,14 +54,16 @@ const ReadinessLoader: FC = () => {
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
// Query WAL replay status.
|
// Only call WAL replay status API if the service is starting up.
|
||||||
|
const shouldQueryWALReplay = statusMessage === STATUS_STARTING;
|
||||||
|
|
||||||
const {
|
const {
|
||||||
data: {
|
data: walData,
|
||||||
data: { min, max, current },
|
isSuccess: walSuccess,
|
||||||
},
|
|
||||||
} = useSuspenseAPIQuery<WALReplayStatus>({
|
} = useSuspenseAPIQuery<WALReplayStatus>({
|
||||||
path: "/status/walreplay",
|
path: "/status/walreplay",
|
||||||
key: ["walreplay", queryKey],
|
key: ["walreplay", queryKey],
|
||||||
|
enabled: shouldQueryWALReplay, // Only enabled when service is starting up.
|
||||||
});
|
});
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
@ -62,21 +78,28 @@ const ReadinessLoader: FC = () => {
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Stack gap="lg" maw={1000} mx="auto" mt="xs">
|
<Alert
|
||||||
<Title order={2}>Starting up...</Title>
|
color="yellow"
|
||||||
{max > 0 && (
|
title={"Prometheus " + (agentMode && "Agent "||"") + (statusMessage || STATUS_LOADING)}
|
||||||
|
icon={<IconAlertTriangle/>}
|
||||||
|
maw={500}
|
||||||
|
mx="auto"
|
||||||
|
mt="lg"
|
||||||
|
>
|
||||||
|
{shouldQueryWALReplay && walSuccess && walData && (
|
||||||
<>
|
<>
|
||||||
<p>
|
<strong>
|
||||||
Replaying WAL ({current}/{max})
|
Replaying WAL ({walData.data.current}/{walData.data.max})
|
||||||
</p>
|
</strong>
|
||||||
<Progress
|
<Progress
|
||||||
size="xl"
|
size="xl"
|
||||||
animated
|
animated
|
||||||
value={((current - min + 1) / (max - min + 1)) * 100}
|
color="yellow"
|
||||||
|
value={((walData.data.current - walData.data.min + 1) / (walData.data.max - walData.data.min + 1)) * 100}
|
||||||
/>
|
/>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</Stack>
|
</Alert>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
33
web/web.go
33
web/web.go
|
@ -102,6 +102,14 @@ var newUIReactRouterServerPaths = []string{
|
||||||
"/tsdb-status",
|
"/tsdb-status",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ReadyStatus uint32
|
||||||
|
|
||||||
|
const (
|
||||||
|
NotReady ReadyStatus = iota
|
||||||
|
Ready
|
||||||
|
Stopping
|
||||||
|
)
|
||||||
|
|
||||||
// withStackTrace logs the stack trace in case the request panics. The function
|
// withStackTrace logs the stack trace in case the request panics. The function
|
||||||
// will re-raise the error which will then be handled by the net/http package.
|
// will re-raise the error which will then be handled by the net/http package.
|
||||||
// It is needed because the go-kit log package doesn't manage properly the
|
// It is needed because the go-kit log package doesn't manage properly the
|
||||||
|
@ -331,7 +339,7 @@ func New(logger log.Logger, o *Options) *Handler {
|
||||||
|
|
||||||
now: model.Now,
|
now: model.Now,
|
||||||
}
|
}
|
||||||
h.SetReady(false)
|
h.SetReady(NotReady)
|
||||||
|
|
||||||
factorySPr := func(_ context.Context) api_v1.ScrapePoolsRetriever { return h.scrapeManager }
|
factorySPr := func(_ context.Context) api_v1.ScrapePoolsRetriever { return h.scrapeManager }
|
||||||
factoryTr := func(_ context.Context) api_v1.TargetRetriever { return h.scrapeManager }
|
factoryTr := func(_ context.Context) api_v1.TargetRetriever { return h.scrapeManager }
|
||||||
|
@ -572,30 +580,39 @@ func serveDebug(w http.ResponseWriter, req *http.Request) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetReady sets the ready status of our web Handler.
|
// SetReady sets the ready status of our web Handler.
|
||||||
func (h *Handler) SetReady(v bool) {
|
func (h *Handler) SetReady(v ReadyStatus) {
|
||||||
if v {
|
if v == Ready {
|
||||||
h.ready.Store(1)
|
h.ready.Store(uint32(Ready))
|
||||||
h.metrics.readyStatus.Set(1)
|
h.metrics.readyStatus.Set(1)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
h.ready.Store(0)
|
h.ready.Store(uint32(v))
|
||||||
h.metrics.readyStatus.Set(0)
|
h.metrics.readyStatus.Set(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Verifies whether the server is ready or not.
|
// Verifies whether the server is ready or not.
|
||||||
func (h *Handler) isReady() bool {
|
func (h *Handler) isReady() bool {
|
||||||
return h.ready.Load() > 0
|
return ReadyStatus(h.ready.Load()) == Ready
|
||||||
}
|
}
|
||||||
|
|
||||||
// Checks if server is ready, calls f if it is, returns 503 if it is not.
|
// Checks if server is ready, calls f if it is, returns 503 if it is not.
|
||||||
func (h *Handler) testReady(f http.HandlerFunc) http.HandlerFunc {
|
func (h *Handler) testReady(f http.HandlerFunc) http.HandlerFunc {
|
||||||
return func(w http.ResponseWriter, r *http.Request) {
|
return func(w http.ResponseWriter, r *http.Request) {
|
||||||
if h.isReady() {
|
switch ReadyStatus(h.ready.Load()) {
|
||||||
|
case Ready:
|
||||||
f(w, r)
|
f(w, r)
|
||||||
} else {
|
case NotReady:
|
||||||
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
|
w.Header().Set("X-Prometheus-Stopping", "false")
|
||||||
|
fmt.Fprintf(w, "Service Unavailable")
|
||||||
|
case Stopping:
|
||||||
|
w.Header().Set("X-Prometheus-Stopping", "true")
|
||||||
w.WriteHeader(http.StatusServiceUnavailable)
|
w.WriteHeader(http.StatusServiceUnavailable)
|
||||||
fmt.Fprintf(w, "Service Unavailable")
|
fmt.Fprintf(w, "Service Unavailable")
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusInternalServerError)
|
||||||
|
fmt.Fprintf(w, "Unknown state")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -156,7 +156,7 @@ func TestReadyAndHealthy(t *testing.T) {
|
||||||
cleanupTestResponse(t, resp)
|
cleanupTestResponse(t, resp)
|
||||||
|
|
||||||
// Set to ready.
|
// Set to ready.
|
||||||
webHandler.SetReady(true)
|
webHandler.SetReady(Ready)
|
||||||
|
|
||||||
for _, u := range []string{
|
for _, u := range []string{
|
||||||
baseURL + "/-/healthy",
|
baseURL + "/-/healthy",
|
||||||
|
@ -260,7 +260,7 @@ func TestRoutePrefix(t *testing.T) {
|
||||||
cleanupTestResponse(t, resp)
|
cleanupTestResponse(t, resp)
|
||||||
|
|
||||||
// Set to ready.
|
// Set to ready.
|
||||||
webHandler.SetReady(true)
|
webHandler.SetReady(Ready)
|
||||||
|
|
||||||
resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/healthy")
|
resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/healthy")
|
||||||
require.NoError(t, err)
|
require.NoError(t, err)
|
||||||
|
@ -307,7 +307,7 @@ func TestDebugHandler(t *testing.T) {
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
handler := New(nil, opts)
|
handler := New(nil, opts)
|
||||||
handler.SetReady(true)
|
handler.SetReady(Ready)
|
||||||
|
|
||||||
w := httptest.NewRecorder()
|
w := httptest.NewRecorder()
|
||||||
|
|
||||||
|
@ -349,7 +349,7 @@ func TestHTTPMetrics(t *testing.T) {
|
||||||
counter := handler.metrics.requestCounter
|
counter := handler.metrics.requestCounter
|
||||||
require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
||||||
|
|
||||||
handler.SetReady(true)
|
handler.SetReady(Ready)
|
||||||
for range [2]int{} {
|
for range [2]int{} {
|
||||||
code = getReady()
|
code = getReady()
|
||||||
require.Equal(t, http.StatusOK, code)
|
require.Equal(t, http.StatusOK, code)
|
||||||
|
@ -358,7 +358,7 @@ func TestHTTPMetrics(t *testing.T) {
|
||||||
require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
|
require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
|
||||||
require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
|
||||||
|
|
||||||
handler.SetReady(false)
|
handler.SetReady(NotReady)
|
||||||
for range [2]int{} {
|
for range [2]int{} {
|
||||||
code = getReady()
|
code = getReady()
|
||||||
require.Equal(t, http.StatusServiceUnavailable, code)
|
require.Equal(t, http.StatusServiceUnavailable, code)
|
||||||
|
@ -537,7 +537,7 @@ func TestAgentAPIEndPoints(t *testing.T) {
|
||||||
opts.Flags = map[string]string{}
|
opts.Flags = map[string]string{}
|
||||||
|
|
||||||
webHandler := New(nil, opts)
|
webHandler := New(nil, opts)
|
||||||
webHandler.SetReady(true)
|
webHandler.SetReady(Ready)
|
||||||
webHandler.config = &config.Config{}
|
webHandler.config = &config.Config{}
|
||||||
webHandler.notifier = ¬ifier.Manager{}
|
webHandler.notifier = ¬ifier.Manager{}
|
||||||
l, err := webHandler.Listeners()
|
l, err := webHandler.Listeners()
|
||||||
|
@ -692,7 +692,7 @@ func TestMultipleListenAddresses(t *testing.T) {
|
||||||
time.Sleep(5 * time.Second)
|
time.Sleep(5 * time.Second)
|
||||||
|
|
||||||
// Set to ready.
|
// Set to ready.
|
||||||
webHandler.SetReady(true)
|
webHandler.SetReady(Ready)
|
||||||
|
|
||||||
for _, port := range []string{port1, port2} {
|
for _, port := range []string{port1, port2} {
|
||||||
baseURL := "http://localhost" + port
|
baseURL := "http://localhost" + port
|
||||||
|
|
Loading…
Reference in a new issue