mantine UI: Distinguish between Not Ready and Stopping

Signed-off-by: Julien <roidelapluie@o11y.eu>
This commit is contained in:
Julien 2024-09-16 17:17:50 +02:00
parent f007659f03
commit ac5377873f
4 changed files with 71 additions and 31 deletions

View file

@ -980,7 +980,7 @@ func main() {
}, },
func(err error) { func(err error) {
close(cancel) close(cancel)
webHandler.SetReady(false) webHandler.SetReady(web.Stopping)
}, },
) )
} }
@ -1159,7 +1159,7 @@ func main() {
reloadReady.Close() reloadReady.Close()
webHandler.SetReady(true) webHandler.SetReady(web.Ready)
level.Info(logger).Log("msg", "Server is ready to receive web requests.") level.Info(logger).Log("msg", "Server is ready to receive web requests.")
<-cancel <-cancel
return nil return nil

View file

@ -1,17 +1,23 @@
import { FC, PropsWithChildren, useEffect, useState } from "react"; import { FC, PropsWithChildren, useEffect, useState } from "react";
import { IconAlertTriangle } from "@tabler/icons-react";
import { useAppDispatch } from "../state/hooks"; import { useAppDispatch } from "../state/hooks";
import { updateSettings, useSettings } from "../state/settingsSlice"; import { updateSettings, useSettings } from "../state/settingsSlice";
import { useSuspenseAPIQuery } from "../api/api"; import { useSuspenseAPIQuery } from "../api/api";
import { WALReplayStatus } from "../api/responseTypes/walreplay"; import { WALReplayStatus } from "../api/responseTypes/walreplay";
import { Progress, Stack, Title } from "@mantine/core"; import { Progress, Alert } from "@mantine/core";
import { useSuspenseQuery } from "@tanstack/react-query"; import { useSuspenseQuery } from "@tanstack/react-query";
const STATUS_STARTING = "is starting up...";
const STATUS_STOPPING = "is shutting down...";
const STATUS_LOADING = "is not ready...";
const ReadinessLoader: FC = () => { const ReadinessLoader: FC = () => {
const { pathPrefix } = useSettings(); const { pathPrefix, agentMode } = useSettings();
const dispatch = useAppDispatch(); const dispatch = useAppDispatch();
// Query key is incremented every second to retrigger the status fetching. // Query key is incremented every second to retrigger the status fetching.
const [queryKey, setQueryKey] = useState(0); const [queryKey, setQueryKey] = useState(0);
const [statusMessage, setStatusMessage] = useState("");
// Query readiness status. // Query readiness status.
const { data: ready } = useSuspenseQuery<boolean>({ const { data: ready } = useSuspenseQuery<boolean>({
@ -28,8 +34,16 @@ const ReadinessLoader: FC = () => {
}); });
switch (res.status) { switch (res.status) {
case 200: case 200:
setStatusMessage(""); // Clear any status message when ready.
return true; return true;
case 503: case 503:
// Check the custom header `X-Prometheus-Stopping` for stopping information.
if (res.headers.get("X-Prometheus-Stopping") === "true") {
setStatusMessage(STATUS_STOPPING);
} else {
setStatusMessage(STATUS_STARTING);
}
return false; return false;
default: default:
throw new Error(res.statusText); throw new Error(res.statusText);
@ -40,14 +54,16 @@ const ReadinessLoader: FC = () => {
}, },
}); });
// Query WAL replay status. // Only call WAL replay status API if the service is starting up.
const shouldQueryWALReplay = statusMessage === STATUS_STARTING;
const { const {
data: { data: walData,
data: { min, max, current }, isSuccess: walSuccess,
},
} = useSuspenseAPIQuery<WALReplayStatus>({ } = useSuspenseAPIQuery<WALReplayStatus>({
path: "/status/walreplay", path: "/status/walreplay",
key: ["walreplay", queryKey], key: ["walreplay", queryKey],
enabled: shouldQueryWALReplay, // Only enabled when service is starting up.
}); });
useEffect(() => { useEffect(() => {
@ -62,21 +78,28 @@ const ReadinessLoader: FC = () => {
}, []); }, []);
return ( return (
<Stack gap="lg" maw={1000} mx="auto" mt="xs"> <Alert
<Title order={2}>Starting up...</Title> color="yellow"
{max > 0 && ( title={"Prometheus " + (agentMode && "Agent "||"") + (statusMessage || STATUS_LOADING)}
icon={<IconAlertTriangle/>}
maw={500}
mx="auto"
mt="lg"
>
{shouldQueryWALReplay && walSuccess && walData && (
<> <>
<p> <strong>
Replaying WAL ({current}/{max}) Replaying WAL ({walData.data.current}/{walData.data.max})
</p> </strong>
<Progress <Progress
size="xl" size="xl"
animated animated
value={((current - min + 1) / (max - min + 1)) * 100} color="yellow"
value={((walData.data.current - walData.data.min + 1) / (walData.data.max - walData.data.min + 1)) * 100}
/> />
</> </>
)} )}
</Stack> </Alert>
); );
}; };

View file

@ -102,6 +102,14 @@ var newUIReactRouterServerPaths = []string{
"/tsdb-status", "/tsdb-status",
} }
type ReadyStatus uint32
const (
NotReady ReadyStatus = iota
Ready
Stopping
)
// withStackTrace logs the stack trace in case the request panics. The function // withStackTrace logs the stack trace in case the request panics. The function
// will re-raise the error which will then be handled by the net/http package. // will re-raise the error which will then be handled by the net/http package.
// It is needed because the go-kit log package doesn't manage properly the // It is needed because the go-kit log package doesn't manage properly the
@ -331,7 +339,7 @@ func New(logger log.Logger, o *Options) *Handler {
now: model.Now, now: model.Now,
} }
h.SetReady(false) h.SetReady(NotReady)
factorySPr := func(_ context.Context) api_v1.ScrapePoolsRetriever { return h.scrapeManager } factorySPr := func(_ context.Context) api_v1.ScrapePoolsRetriever { return h.scrapeManager }
factoryTr := func(_ context.Context) api_v1.TargetRetriever { return h.scrapeManager } factoryTr := func(_ context.Context) api_v1.TargetRetriever { return h.scrapeManager }
@ -572,30 +580,39 @@ func serveDebug(w http.ResponseWriter, req *http.Request) {
} }
// SetReady sets the ready status of our web Handler. // SetReady sets the ready status of our web Handler.
func (h *Handler) SetReady(v bool) { func (h *Handler) SetReady(v ReadyStatus) {
if v { if v == Ready {
h.ready.Store(1) h.ready.Store(uint32(Ready))
h.metrics.readyStatus.Set(1) h.metrics.readyStatus.Set(1)
return return
} }
h.ready.Store(0) h.ready.Store(uint32(v))
h.metrics.readyStatus.Set(0) h.metrics.readyStatus.Set(0)
} }
// Verifies whether the server is ready or not. // Verifies whether the server is ready or not.
func (h *Handler) isReady() bool { func (h *Handler) isReady() bool {
return h.ready.Load() > 0 return ReadyStatus(h.ready.Load()) == Ready
} }
// Checks if server is ready, calls f if it is, returns 503 if it is not. // Checks if server is ready, calls f if it is, returns 503 if it is not.
func (h *Handler) testReady(f http.HandlerFunc) http.HandlerFunc { func (h *Handler) testReady(f http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
if h.isReady() { switch ReadyStatus(h.ready.Load()) {
case Ready:
f(w, r) f(w, r)
} else { case NotReady:
w.WriteHeader(http.StatusServiceUnavailable)
w.Header().Set("X-Prometheus-Stopping", "false")
fmt.Fprintf(w, "Service Unavailable")
case Stopping:
w.Header().Set("X-Prometheus-Stopping", "true")
w.WriteHeader(http.StatusServiceUnavailable) w.WriteHeader(http.StatusServiceUnavailable)
fmt.Fprintf(w, "Service Unavailable") fmt.Fprintf(w, "Service Unavailable")
default:
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprintf(w, "Unknown state")
} }
} }
} }

View file

@ -156,7 +156,7 @@ func TestReadyAndHealthy(t *testing.T) {
cleanupTestResponse(t, resp) cleanupTestResponse(t, resp)
// Set to ready. // Set to ready.
webHandler.SetReady(true) webHandler.SetReady(Ready)
for _, u := range []string{ for _, u := range []string{
baseURL + "/-/healthy", baseURL + "/-/healthy",
@ -260,7 +260,7 @@ func TestRoutePrefix(t *testing.T) {
cleanupTestResponse(t, resp) cleanupTestResponse(t, resp)
// Set to ready. // Set to ready.
webHandler.SetReady(true) webHandler.SetReady(Ready)
resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/healthy") resp, err = http.Get(baseURL + opts.RoutePrefix + "/-/healthy")
require.NoError(t, err) require.NoError(t, err)
@ -307,7 +307,7 @@ func TestDebugHandler(t *testing.T) {
}, },
} }
handler := New(nil, opts) handler := New(nil, opts)
handler.SetReady(true) handler.SetReady(Ready)
w := httptest.NewRecorder() w := httptest.NewRecorder()
@ -349,7 +349,7 @@ func TestHTTPMetrics(t *testing.T) {
counter := handler.metrics.requestCounter counter := handler.metrics.requestCounter
require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
handler.SetReady(true) handler.SetReady(Ready)
for range [2]int{} { for range [2]int{} {
code = getReady() code = getReady()
require.Equal(t, http.StatusOK, code) require.Equal(t, http.StatusOK, code)
@ -358,7 +358,7 @@ func TestHTTPMetrics(t *testing.T) {
require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK))))) require.Equal(t, 2, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusOK)))))
require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable))))) require.Equal(t, 1, int(prom_testutil.ToFloat64(counter.WithLabelValues("/-/ready", strconv.Itoa(http.StatusServiceUnavailable)))))
handler.SetReady(false) handler.SetReady(NotReady)
for range [2]int{} { for range [2]int{} {
code = getReady() code = getReady()
require.Equal(t, http.StatusServiceUnavailable, code) require.Equal(t, http.StatusServiceUnavailable, code)
@ -537,7 +537,7 @@ func TestAgentAPIEndPoints(t *testing.T) {
opts.Flags = map[string]string{} opts.Flags = map[string]string{}
webHandler := New(nil, opts) webHandler := New(nil, opts)
webHandler.SetReady(true) webHandler.SetReady(Ready)
webHandler.config = &config.Config{} webHandler.config = &config.Config{}
webHandler.notifier = &notifier.Manager{} webHandler.notifier = &notifier.Manager{}
l, err := webHandler.Listeners() l, err := webHandler.Listeners()
@ -692,7 +692,7 @@ func TestMultipleListenAddresses(t *testing.T) {
time.Sleep(5 * time.Second) time.Sleep(5 * time.Second)
// Set to ready. // Set to ready.
webHandler.SetReady(true) webHandler.SetReady(Ready)
for _, port := range []string{port1, port2} { for _, port := range []string{port1, port2} {
baseURL := "http://localhost" + port baseURL := "http://localhost" + port