mirror of
https://github.com/prometheus/prometheus.git
synced 2024-11-09 23:24:05 -08:00
Merge pull request #1761 from prometheus/fabxc-legacyapi
api: drop legacy API
This commit is contained in:
commit
62af249d06
|
@ -1,53 +0,0 @@
|
|||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/prometheus/common/route"
|
||||
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/storage/local"
|
||||
"github.com/prometheus/prometheus/util/httputil"
|
||||
)
|
||||
|
||||
// API manages the /api HTTP endpoint.
|
||||
type API struct {
|
||||
Now func() model.Time
|
||||
Storage local.Storage
|
||||
QueryEngine *promql.Engine
|
||||
}
|
||||
|
||||
// Register registers the handler for the various endpoints below /api.
|
||||
func (api *API) Register(router *route.Router) {
|
||||
// List all the endpoints here instead of using a wildcard route because we
|
||||
// would otherwise handle /api/v1 as well.
|
||||
router.Options("/query", handle("options", api.Options))
|
||||
router.Options("/query_range", handle("options", api.Options))
|
||||
router.Options("/metrics", handle("options", api.Options))
|
||||
|
||||
router.Get("/query", handle("query", api.Query))
|
||||
router.Get("/query_range", handle("query_range", api.QueryRange))
|
||||
router.Get("/metrics", handle("metrics", api.Metrics))
|
||||
}
|
||||
|
||||
func handle(name string, f http.HandlerFunc) http.HandlerFunc {
|
||||
h := httputil.CompressionHandler{
|
||||
Handler: f,
|
||||
}
|
||||
return prometheus.InstrumentHandler(name, h)
|
||||
}
|
|
@ -1,143 +0,0 @@
|
|||
// Copyright 2015 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"regexp"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/prometheus/common/route"
|
||||
|
||||
"github.com/prometheus/prometheus/promql"
|
||||
"github.com/prometheus/prometheus/storage/local"
|
||||
)
|
||||
|
||||
// This is a bit annoying. On one hand, we have to choose a current timestamp
|
||||
// because the storage doesn't have a mocked-out time yet and would otherwise
|
||||
// immediately throw away "old" samples. On the other hand, we have to make
|
||||
// sure that the float value survives the parsing and re-formatting in the
|
||||
// query layer precisely without any change. Thus we round to seconds and then
|
||||
// add known-good digits after the decimal point which behave well in
|
||||
// parsing/re-formatting.
|
||||
var testTimestamp = model.TimeFromUnix(time.Now().Round(time.Second).Unix()).Add(124 * time.Millisecond)
|
||||
|
||||
func testNow() model.Time {
|
||||
return testTimestamp
|
||||
}
|
||||
|
||||
func TestQuery(t *testing.T) {
|
||||
scenarios := []struct {
|
||||
// URL query string.
|
||||
queryStr string
|
||||
// Expected HTTP response status code.
|
||||
status int
|
||||
// Regex to match against response body.
|
||||
bodyRe string
|
||||
}{
|
||||
{
|
||||
queryStr: "",
|
||||
status: http.StatusOK,
|
||||
bodyRe: `{"type":"error","value":"parse error at char 1: no expression found in input","version":1}`,
|
||||
},
|
||||
{
|
||||
queryStr: "expr=1.4",
|
||||
status: http.StatusOK,
|
||||
bodyRe: `{"type":"scalar","value":"1.4","version":1}`,
|
||||
},
|
||||
{
|
||||
queryStr: "expr=testmetric",
|
||||
status: http.StatusOK,
|
||||
bodyRe: `{"type":"vector","value":\[{"metric":{"__name__":"testmetric"},"value":"0","timestamp":\d+\.\d+}\],"version":1}`,
|
||||
},
|
||||
{
|
||||
queryStr: "expr=testmetric×tamp=" + testTimestamp.String(),
|
||||
status: http.StatusOK,
|
||||
bodyRe: `{"type":"vector","value":\[{"metric":{"__name__":"testmetric"},"value":"0","timestamp":` + testTimestamp.String() + `}\],"version":1}`,
|
||||
},
|
||||
{
|
||||
queryStr: "expr=testmetric×tamp=" + testTimestamp.Add(-time.Hour).String(),
|
||||
status: http.StatusOK,
|
||||
bodyRe: `{"type":"vector","value":\[\],"version":1}`,
|
||||
},
|
||||
{
|
||||
queryStr: "timestamp=invalid",
|
||||
status: http.StatusBadRequest,
|
||||
bodyRe: "invalid query timestamp",
|
||||
},
|
||||
{
|
||||
queryStr: "expr=(badexpression",
|
||||
status: http.StatusOK,
|
||||
bodyRe: `{"type":"error","value":"parse error at char 15: unclosed left parenthesis","version":1}`,
|
||||
},
|
||||
}
|
||||
|
||||
storage, closer := local.NewTestStorage(t, 2)
|
||||
defer closer.Close()
|
||||
storage.Append(&model.Sample{
|
||||
Metric: model.Metric{
|
||||
model.MetricNameLabel: "testmetric",
|
||||
},
|
||||
Timestamp: testTimestamp,
|
||||
Value: 0,
|
||||
})
|
||||
storage.WaitForIndexing()
|
||||
|
||||
api := &API{
|
||||
Now: testNow,
|
||||
Storage: storage,
|
||||
QueryEngine: promql.NewEngine(storage, nil),
|
||||
}
|
||||
rtr := route.New()
|
||||
api.Register(rtr.WithPrefix("/api"))
|
||||
|
||||
server := httptest.NewServer(rtr)
|
||||
defer server.Close()
|
||||
|
||||
for i, s := range scenarios {
|
||||
// Do query.
|
||||
resp, err := http.Get(server.URL + "/api/query?" + s.queryStr)
|
||||
if err != nil {
|
||||
t.Fatalf("%d. Error querying API: %s", i, err)
|
||||
}
|
||||
|
||||
// Check status code.
|
||||
if resp.StatusCode != s.status {
|
||||
t.Fatalf("%d. Unexpected status code; got %d, want %d", i, resp.StatusCode, s.status)
|
||||
}
|
||||
|
||||
// Check response headers.
|
||||
ct := resp.Header["Content-Type"]
|
||||
if len(ct) != 1 {
|
||||
t.Fatalf("%d. Unexpected number of 'Content-Type' headers; got %d, want 1", i, len(ct))
|
||||
}
|
||||
if ct[0] != "application/json" {
|
||||
t.Fatalf("%d. Unexpected 'Content-Type' header; got %s; want %s", i, ct[0], "application/json")
|
||||
}
|
||||
|
||||
// Check body.
|
||||
b, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("%d. Error reading response body: %s", i, err)
|
||||
}
|
||||
re := regexp.MustCompile(s.bodyRe)
|
||||
if !re.Match(b) {
|
||||
t.Fatalf("%d. Body didn't match '%s'. Body: %s", i, s.bodyRe, string(b))
|
||||
}
|
||||
}
|
||||
}
|
|
@ -1,280 +0,0 @@
|
|||
// Copyright 2013 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/common/log"
|
||||
"github.com/prometheus/common/model"
|
||||
)
|
||||
|
||||
// Enables cross-site script calls.
|
||||
func setAccessControlHeaders(w http.ResponseWriter) {
|
||||
w.Header().Set("Access-Control-Allow-Headers", "Accept, Authorization, Content-Type, Origin")
|
||||
w.Header().Set("Access-Control-Allow-Methods", "GET, OPTIONS")
|
||||
w.Header().Set("Access-Control-Allow-Origin", "*")
|
||||
w.Header().Set("Access-Control-Expose-Headers", "Date")
|
||||
}
|
||||
|
||||
func httpJSONError(w http.ResponseWriter, err error, code int) {
|
||||
w.WriteHeader(code)
|
||||
errorJSON(w, err)
|
||||
}
|
||||
|
||||
func parseTimestampOrNow(t string, now model.Time) (model.Time, error) {
|
||||
if t == "" {
|
||||
return now, nil
|
||||
}
|
||||
|
||||
tFloat, err := strconv.ParseFloat(t, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return model.TimeFromUnixNano(int64(tFloat * float64(time.Second/time.Nanosecond))), nil
|
||||
}
|
||||
|
||||
func parseDuration(d string) (time.Duration, error) {
|
||||
dFloat, err := strconv.ParseFloat(d, 64)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return time.Duration(dFloat * float64(time.Second/time.Nanosecond)), nil
|
||||
}
|
||||
|
||||
// Options handles OPTIONS requests to /api/... endpoints.
|
||||
func (api *API) Options(w http.ResponseWriter, r *http.Request) {
|
||||
setAccessControlHeaders(w)
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
// Query handles the /api/query endpoint.
|
||||
func (api *API) Query(w http.ResponseWriter, r *http.Request) {
|
||||
setAccessControlHeaders(w)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
params := getQueryParams(r)
|
||||
expr := params.Get("expr")
|
||||
|
||||
timestamp, err := parseTimestampOrNow(params.Get("timestamp"), api.Now())
|
||||
if err != nil {
|
||||
httpJSONError(w, fmt.Errorf("invalid query timestamp %s", err), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
query, err := api.QueryEngine.NewInstantQuery(expr, timestamp)
|
||||
if err != nil {
|
||||
httpJSONError(w, err, http.StatusOK)
|
||||
return
|
||||
}
|
||||
res := query.Exec()
|
||||
if res.Err != nil {
|
||||
httpJSONError(w, res.Err, http.StatusOK)
|
||||
return
|
||||
}
|
||||
log.Debugf("Instant query: %s\nQuery stats:\n%s\n", expr, query.Stats())
|
||||
|
||||
if vec, ok := res.Value.(model.Vector); ok {
|
||||
respondJSON(w, plainVec(vec))
|
||||
return
|
||||
}
|
||||
if sca, ok := res.Value.(*model.Scalar); ok {
|
||||
respondJSON(w, (*plainScalar)(sca))
|
||||
return
|
||||
}
|
||||
if str, ok := res.Value.(*model.String); ok {
|
||||
respondJSON(w, (*plainString)(str))
|
||||
return
|
||||
}
|
||||
|
||||
respondJSON(w, res.Value)
|
||||
}
|
||||
|
||||
// plainVec is an indirection that hides the original MarshalJSON method
|
||||
// which does not fit the response format for the legacy API.
|
||||
type plainVec model.Vector
|
||||
|
||||
func (pv plainVec) MarshalJSON() ([]byte, error) {
|
||||
type plainSmpl model.Sample
|
||||
|
||||
v := make([]*plainSmpl, len(pv))
|
||||
for i, sv := range pv {
|
||||
v[i] = (*plainSmpl)(sv)
|
||||
}
|
||||
|
||||
return json.Marshal(&v)
|
||||
}
|
||||
|
||||
func (pv plainVec) Type() model.ValueType {
|
||||
return model.ValVector
|
||||
}
|
||||
|
||||
func (pv plainVec) String() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// plainScalar is an indirection that hides the original MarshalJSON method
|
||||
// which does not fit the response format for the legacy API.
|
||||
type plainScalar model.Scalar
|
||||
|
||||
func (ps plainScalar) MarshalJSON() ([]byte, error) {
|
||||
s := strconv.FormatFloat(float64(ps.Value), 'f', -1, 64)
|
||||
return json.Marshal(&s)
|
||||
}
|
||||
|
||||
func (plainScalar) Type() model.ValueType {
|
||||
return model.ValScalar
|
||||
}
|
||||
|
||||
func (plainScalar) String() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// plainString is an indirection that hides the original MarshalJSON method
|
||||
// which does not fit the response format for the legacy API.
|
||||
type plainString model.String
|
||||
|
||||
func (pv plainString) Type() model.ValueType {
|
||||
return model.ValString
|
||||
}
|
||||
|
||||
func (pv plainString) String() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
// QueryRange handles the /api/query_range endpoint.
|
||||
func (api *API) QueryRange(w http.ResponseWriter, r *http.Request) {
|
||||
setAccessControlHeaders(w)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
params := getQueryParams(r)
|
||||
expr := params.Get("expr")
|
||||
|
||||
duration, err := parseDuration(params.Get("range"))
|
||||
if err != nil {
|
||||
httpJSONError(w, fmt.Errorf("invalid query range: %s", err), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
step, err := parseDuration(params.Get("step"))
|
||||
if err != nil {
|
||||
httpJSONError(w, fmt.Errorf("invalid query resolution: %s", err), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
end, err := parseTimestampOrNow(params.Get("end"), api.Now())
|
||||
if err != nil {
|
||||
httpJSONError(w, fmt.Errorf("invalid query timestamp: %s", err), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
// TODO(julius): Remove this special-case handling a while after PromDash and
|
||||
// other API consumers have been changed to no longer set "end=0" for setting
|
||||
// the current time as the end time. Instead, the "end" parameter should
|
||||
// simply be omitted or set to an empty string for that case.
|
||||
if end == 0 {
|
||||
end = api.Now()
|
||||
}
|
||||
|
||||
// For safety, limit the number of returned points per timeseries.
|
||||
// This is sufficient for 60s resolution for a week or 1h resolution for a year.
|
||||
if duration/step > 11000 {
|
||||
err := errors.New("exceeded maximum resolution of 11,000 points per timeseries. Try decreasing the query resolution (?step=XX)")
|
||||
httpJSONError(w, err, http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
// Align the start to step "tick" boundary.
|
||||
end = end.Add(-time.Duration(end.UnixNano() % int64(step)))
|
||||
start := end.Add(-duration)
|
||||
|
||||
query, err := api.QueryEngine.NewRangeQuery(expr, start, end, step)
|
||||
if err != nil {
|
||||
httpJSONError(w, err, http.StatusOK)
|
||||
return
|
||||
}
|
||||
matrix, err := query.Exec().Matrix()
|
||||
if err != nil {
|
||||
httpJSONError(w, err, http.StatusOK)
|
||||
return
|
||||
}
|
||||
|
||||
log.Debugf("Range query: %s\nQuery stats:\n%s\n", expr, query.Stats())
|
||||
respondJSON(w, matrix)
|
||||
}
|
||||
|
||||
// Metrics handles the /api/metrics endpoint.
|
||||
func (api *API) Metrics(w http.ResponseWriter, r *http.Request) {
|
||||
setAccessControlHeaders(w)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
|
||||
metricNames := api.Storage.LabelValuesForLabelName(model.MetricNameLabel)
|
||||
sort.Sort(metricNames)
|
||||
resultBytes, err := json.Marshal(metricNames)
|
||||
if err != nil {
|
||||
log.Error("Error marshalling metric names: ", err)
|
||||
httpJSONError(w, fmt.Errorf("error marshalling metric names: %s", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Write(resultBytes)
|
||||
}
|
||||
|
||||
// GetQueryParams calls r.ParseForm and returns r.Form.
|
||||
func getQueryParams(r *http.Request) url.Values {
|
||||
r.ParseForm()
|
||||
return r.Form
|
||||
}
|
||||
|
||||
var jsonFormatVersion = 1
|
||||
|
||||
// ErrorJSON writes the given error JSON-formatted to w.
|
||||
func errorJSON(w io.Writer, err error) error {
|
||||
data := struct {
|
||||
Type string `json:"type"`
|
||||
Value string `json:"value"`
|
||||
Version int `json:"version"`
|
||||
}{
|
||||
Type: "error",
|
||||
Value: err.Error(),
|
||||
Version: jsonFormatVersion,
|
||||
}
|
||||
enc := json.NewEncoder(w)
|
||||
return enc.Encode(data)
|
||||
}
|
||||
|
||||
// RespondJSON converts the given data value to JSON and writes it to w.
|
||||
func respondJSON(w io.Writer, val model.Value) error {
|
||||
data := struct {
|
||||
Type string `json:"type"`
|
||||
Value interface{} `json:"value"`
|
||||
Version int `json:"version"`
|
||||
}{
|
||||
Type: val.Type().String(),
|
||||
Value: val,
|
||||
Version: jsonFormatVersion,
|
||||
}
|
||||
// TODO(fabxc): Adding MarshalJSON to promql.Values might be a good idea.
|
||||
if sc, ok := val.(*model.Scalar); ok {
|
||||
data.Value = sc.Value
|
||||
}
|
||||
enc := json.NewEncoder(w)
|
||||
return enc.Encode(data)
|
||||
}
|
|
@ -1,66 +0,0 @@
|
|||
// Copyright 2015 The Prometheus Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package legacy
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/common/model"
|
||||
)
|
||||
|
||||
func TestParseTimestampOrNow(t *testing.T) {
|
||||
ts, err := parseTimestampOrNow("", testNow())
|
||||
if err != nil {
|
||||
t.Fatalf("err = %s; want nil", err)
|
||||
}
|
||||
if !testNow().Equal(ts) {
|
||||
t.Fatalf("ts = %v; want ts = %v", ts, testNow())
|
||||
}
|
||||
|
||||
ts, err = parseTimestampOrNow("1426956073.12345", testNow())
|
||||
if err != nil {
|
||||
t.Fatalf("err = %s; want nil", err)
|
||||
}
|
||||
expTS := model.TimeFromUnixNano(1426956073123000000)
|
||||
if !ts.Equal(expTS) {
|
||||
t.Fatalf("ts = %v; want %v", ts, expTS)
|
||||
}
|
||||
|
||||
_, err = parseTimestampOrNow("123.45foo", testNow())
|
||||
if err == nil {
|
||||
t.Fatalf("err = nil; want %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseDuration(t *testing.T) {
|
||||
_, err := parseDuration("")
|
||||
if err == nil {
|
||||
t.Fatalf("err = nil; want %s", err)
|
||||
}
|
||||
|
||||
_, err = parseDuration("1234.56foo")
|
||||
if err == nil {
|
||||
t.Fatalf("err = nil; want %s", err)
|
||||
}
|
||||
|
||||
d, err := parseDuration("1234.56789")
|
||||
if err != nil {
|
||||
t.Fatalf("err = %s; want nil", err)
|
||||
}
|
||||
expD := time.Duration(1234.56789 * float64(time.Second))
|
||||
if d != expD {
|
||||
t.Fatalf("d = %v; want %v", d, expD)
|
||||
}
|
||||
}
|
14
web/web.go
14
web/web.go
|
@ -44,8 +44,7 @@ import (
|
|||
"github.com/prometheus/prometheus/storage/local"
|
||||
"github.com/prometheus/prometheus/template"
|
||||
"github.com/prometheus/prometheus/util/httputil"
|
||||
"github.com/prometheus/prometheus/web/api/legacy"
|
||||
"github.com/prometheus/prometheus/web/api/v1"
|
||||
api_v1 "github.com/prometheus/prometheus/web/api/v1"
|
||||
"github.com/prometheus/prometheus/web/ui"
|
||||
)
|
||||
|
||||
|
@ -58,8 +57,7 @@ type Handler struct {
|
|||
queryEngine *promql.Engine
|
||||
storage local.Storage
|
||||
|
||||
apiV1 *v1.API
|
||||
apiLegacy *legacy.API
|
||||
apiV1 *api_v1.API
|
||||
|
||||
router *route.Router
|
||||
listenErrCh chan error
|
||||
|
@ -136,12 +134,7 @@ func New(
|
|||
queryEngine: qe,
|
||||
storage: st,
|
||||
|
||||
apiV1: v1.NewAPI(qe, st),
|
||||
apiLegacy: &legacy.API{
|
||||
QueryEngine: qe,
|
||||
Storage: st,
|
||||
Now: model.Now,
|
||||
},
|
||||
apiV1: api_v1.NewAPI(qe, st),
|
||||
}
|
||||
|
||||
if o.ExternalURL.Path != "" {
|
||||
|
@ -176,7 +169,6 @@ func New(
|
|||
Handler: http.HandlerFunc(h.federation),
|
||||
}))
|
||||
|
||||
h.apiLegacy.Register(router.WithPrefix("/api"))
|
||||
h.apiV1.Register(router.WithPrefix("/api/v1"))
|
||||
|
||||
router.Get("/consoles/*filepath", instrf("consoles", h.consoles))
|
||||
|
|
Loading…
Reference in a new issue