From 3b9b1c6ab48ecf33b6d4001b9daa38c97709dcee Mon Sep 17 00:00:00 2001 From: "Matt T. Proud" Date: Sun, 5 May 2013 19:32:04 +0200 Subject: [PATCH] Define dependencies for web. stack concretely. This commit destroys the use of AppState, which makes passing concrete state along to various serving components onerous. --- .gitignore | 8 +++++- Makefile | 36 +++++------------------- Makefile.INCLUDE | 16 +++++++++++ appstate/appstate.go | 33 ---------------------- main.go | 65 ++++++++++++++++++++++++++++---------------- model/Makefile | 2 +- web/Makefile | 8 ++++-- web/api/api.go | 16 +++++------ web/api/query.go | 2 +- web/api/targets.go | 42 ++++++++++++++++------------ web/blob/.gitignore | 3 +- web/status.go | 34 ++++++++++++++--------- web/web.go | 18 ++++++------ 13 files changed, 143 insertions(+), 140 deletions(-) delete mode 100644 appstate/appstate.go diff --git a/.gitignore b/.gitignore index 4c0a952bc..16bfed94a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,5 +21,11 @@ _cgo_* core *-stamp -prometheus.build prometheus + +.#* +command-line-arguments.test +*BACKUP* +*BASE* +*LOCAL* +*REMOTE* diff --git a/Makefile b/Makefile index 209d28b52..ca23c63f4 100644 --- a/Makefile +++ b/Makefile @@ -11,36 +11,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -.SUFFIXES: - -TEST_ARTIFACTS = prometheus prometheus.build search_index +TEST_ARTIFACTS = prometheus search_index include Makefile.INCLUDE -REV := $(shell git rev-parse --short HEAD) -BRANCH := $(shell git rev-parse --abbrev-ref HEAD) -HOSTNAME := $(shell hostname -f) -BUILD_DATE := $(shell date +%Y%m%d-%H:%M:%S) -BUILDFLAGS := -ldflags \ - " -X main.buildVersion $(REV)\ - -X main.buildBranch $(BRANCH)\ - -X main.buildUser $(USER)@$(HOSTNAME)\ - -X main.buildDate $(BUILD_DATE)\ - -X main.goVersion $(GO_VERSION)\ - -X main.leveldbVersion $(LEVELDB_VERSION)\ - -X main.protobufVersion $(PROTOCOL_BUFFERS_VERSION)\ - -X main.snappyVersion $(SNAPPY_VERSION)" - all: test advice: go tool vet . binary: build - go build $(BUILDFLAGS) -o prometheus.build + go build $(BUILDFLAGS) . build: preparation config model web - go build $(BUILDFLAGS) . clean: $(MAKE) -C build clean @@ -72,11 +55,14 @@ preparation: source_path $(MAKE) -C build run: binary - ./prometheus.build $(ARGUMENTS) + ./prometheus $(ARGUMENTS) search_index: godoc -index -write_index -index_files='search_index' +server: config model preparation + $(MAKE) -C server + # source_path is responsible for ensuring that the builder has not done anything # stupid like working on Prometheus outside of ${GOPATH}. source_path: @@ -84,15 +70,7 @@ source_path: [ -d "$(FULL_GOPATH)" ] test: build - go test ./appstate/... $(GO_TEST_FLAGS) - go test ./coding/... $(GO_TEST_FLAGS) - go test ./config/... $(GO_TEST_FLAGS) - go test ./model/... $(GO_TEST_FLAGS) - go test ./retrieval/... $(GO_TEST_FLAGS) - go test ./rules/... $(GO_TEST_FLAGS) - go test ./storage/... $(GO_TEST_FLAGS) - go test ./utility/... $(GO_TEST_FLAGS) - go test ./web/... $(GO_TEST_FLAGS) + go test ./... $(GO_TEST_FLAGS) web: preparation config model $(MAKE) -C web diff --git a/Makefile.INCLUDE b/Makefile.INCLUDE index 4f848b4ee..ecb419549 100644 --- a/Makefile.INCLUDE +++ b/Makefile.INCLUDE @@ -1,3 +1,5 @@ +# -*- Mode: makefile -*- + # Copyright 2013 Prometheus Team # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -52,3 +54,17 @@ BREW_INSTALL := brew install # By default, wget sets the creation time to match the server's, which throws # off Make. :-( WGET := wget --no-use-server-timestamps -c + +REV := $(shell git rev-parse --short HEAD) +BRANCH := $(shell git rev-parse --abbrev-ref HEAD) +HOSTNAME := $(shell hostname -f) +BUILD_DATE := $(shell date +%Y%m%d-%H:%M:%S) +BUILDFLAGS := -ldflags \ + " -X main.buildVersion $(REV)\ + -X main.buildBranch $(BRANCH)\ + -X main.buildUser $(USER)@$(HOSTNAME)\ + -X main.buildDate $(BUILD_DATE)\ + -X main.goVersion $(GO_VERSION)\ + -X main.leveldbVersion $(LEVELDB_VERSION)\ + -X main.protobufVersion $(PROTOCOL_BUFFERS_VERSION)\ + -X main.snappyVersion $(SNAPPY_VERSION)" diff --git a/appstate/appstate.go b/appstate/appstate.go deleted file mode 100644 index 5170df231..000000000 --- a/appstate/appstate.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2013 Prometheus Team -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package appstate - -import ( - "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/retrieval" - "github.com/prometheus/prometheus/rules" - "github.com/prometheus/prometheus/storage/metric" -) - -// ApplicationState is an encapsulation of all relevant Prometheus application -// runtime state. It enables simpler passing of this state to components that -// require it. -type ApplicationState struct { - Config config.Config - RuleManager rules.RuleManager - Storage metric.TieredStorage - TargetManager retrieval.TargetManager - BuildInfo map[string]string - CurationState chan metric.CurationState -} diff --git a/main.go b/main.go index 6f129deab..71b53e5ba 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,6 @@ package main import ( "flag" - "github.com/prometheus/prometheus/appstate" "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/retrieval" "github.com/prometheus/prometheus/retrieval/format" @@ -23,6 +22,7 @@ import ( "github.com/prometheus/prometheus/rules/ast" "github.com/prometheus/prometheus/storage/metric" "github.com/prometheus/prometheus/web" + "github.com/prometheus/prometheus/web/api" "log" "os" "os/signal" @@ -42,10 +42,10 @@ var ( ) type prometheus struct { - storage metric.TieredStorage - // TODO: Refactor channels to work with arrays of results for better chunking. - scrapeResults chan format.Result + curationState chan metric.CurationState ruleResults chan *rules.Result + storage metric.TieredStorage + scrapeResults chan format.Result } func (p prometheus) interruptHandler() { @@ -60,9 +60,8 @@ func (p prometheus) interruptHandler() { } func (p prometheus) close() { + close(p.curationState) p.storage.Close() - close(p.scrapeResults) - close(p.ruleResults) } func main() { @@ -92,22 +91,46 @@ func main() { scrapeResults := make(chan format.Result, *scrapeResultsQueueCapacity) ruleResults := make(chan *rules.Result, *ruleResultsQueueCapacity) + curationState := make(chan metric.CurationState, 1) + + // Queue depth will need to be exposed + targetManager := retrieval.NewTargetManager(scrapeResults, *concurrentRetrievalAllowance) + targetManager.AddTargetsFromConfig(conf) + + statusHandler := &web.StatusHandler{ + BuildInfo: BuildInfo, + Config: &conf, + CurationState: curationState, + // Furnish the default status. + PrometheusStatus: &web.PrometheusStatus{}, + TargetManager: targetManager, + } + + // The closing of curationState implicitly closes this routine. + go statusHandler.ServeRequestsForever() + + metricsService := &api.MetricsService{ + Config: &conf, + TargetManager: targetManager, + Storage: ts, + } + + webService := &web.WebService{ + StatusHandler: statusHandler, + MetricsHandler: metricsService, + } prometheus := prometheus{ - storage: *ts, - scrapeResults: scrapeResults, + curationState: curationState, ruleResults: ruleResults, + scrapeResults: scrapeResults, + storage: *ts, } defer prometheus.close() go ts.Serve() go prometheus.interruptHandler() - // Queue depth will need to be exposed - - targetManager := retrieval.NewTargetManager(scrapeResults, *concurrentRetrievalAllowance) - targetManager.AddTargetsFromConfig(conf) - ast.SetStorage(*ts) ruleManager := rules.NewRuleManager(ruleResults, conf.EvaluationInterval()) @@ -116,16 +139,12 @@ func main() { log.Fatalf("Error loading rule files: %v", err) } - appState := &appstate.ApplicationState{ - BuildInfo: BuildInfo, - Config: conf, - CurationState: make(chan metric.CurationState), - RuleManager: ruleManager, - Storage: *ts, - TargetManager: targetManager, - } - - web.StartServing(appState) + go func() { + err := webService.ServeForever() + if err != nil { + log.Fatal(err) + } + }() // TODO(all): Migrate this into prometheus.serve(). for { diff --git a/model/Makefile b/model/Makefile index 7fe2c1dec..55555b83a 100644 --- a/model/Makefile +++ b/model/Makefile @@ -29,4 +29,4 @@ clean: rm -rf generated/* -rm -f $(MAKE_ARTIFACTS) -.PHONY: generated +.PHONY: clean generated diff --git a/web/Makefile b/web/Makefile index c643948a8..adfa2043e 100644 --- a/web/Makefile +++ b/web/Makefile @@ -11,13 +11,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -all: blob +all: blob-stamp -blob: static/generated/protocol_buffer.descriptor +blob-stamp: static/generated/protocol_buffer.descriptor $(MAKE) -C blob + touch $@ clean: $(MAKE) -C blob clean + -rm -f *-stamp static/generated: mkdir -vp static/generated @@ -25,4 +27,4 @@ static/generated: static/generated/protocol_buffer.descriptor: static/generated ../model/generated/descriptor.blob cp -f ../model/generated/descriptor.blob $@ -.PHONY: blob clean +.PHONY: clean diff --git a/web/api/api.go b/web/api/api.go index b0da2de19..572eb515d 100644 --- a/web/api/api.go +++ b/web/api/api.go @@ -15,7 +15,9 @@ package api import ( "code.google.com/p/gorest" - "github.com/prometheus/prometheus/appstate" + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/retrieval" + "github.com/prometheus/prometheus/storage/metric" "github.com/prometheus/prometheus/utility" ) @@ -27,13 +29,9 @@ type MetricsService struct { metrics gorest.EndPoint `method:"GET" path:"/metrics" output:"string"` setTargets gorest.EndPoint `method:"PUT" path:"/jobs/{jobName:string}/targets" postdata:"[]TargetGroup"` + time utility.Time - appState *appstate.ApplicationState - time utility.Time -} - -func NewMetricsService(appState *appstate.ApplicationState) *MetricsService { - return &MetricsService{ - appState: appState, - } + Config *config.Config + TargetManager retrieval.TargetManager + Storage *metric.TieredStorage } diff --git a/web/api/query.go b/web/api/query.go index ac3641bf4..91bb503b7 100644 --- a/web/api/query.go +++ b/web/api/query.go @@ -96,7 +96,7 @@ func (serv MetricsService) QueryRange(expr string, end int64, duration int64, st } func (serv MetricsService) Metrics() string { - metricNames, err := serv.appState.Storage.GetAllValuesForLabel(model.MetricNameLabel) + metricNames, err := serv.Storage.GetAllValuesForLabel(model.MetricNameLabel) rb := serv.ResponseBuilder() serv.setAccessControlHeaders(rb) rb.SetContentType(gorest.Application_Json) diff --git a/web/api/targets.go b/web/api/targets.go index a60527001..f83a76caf 100644 --- a/web/api/targets.go +++ b/web/api/targets.go @@ -26,25 +26,31 @@ type TargetGroup struct { } func (serv MetricsService) SetTargets(targetGroups []TargetGroup, jobName string) { - if job := serv.appState.Config.GetJobByName(jobName); job == nil { + job := serv.Config.GetJobByName(jobName) + if job == nil { rb := serv.ResponseBuilder() rb.SetResponseCode(http.StatusNotFound) - } else { - newTargets := []retrieval.Target{} - for _, targetGroup := range targetGroups { - // Do mandatory map type conversion due to Go shortcomings. - baseLabels := model.LabelSet{ - model.JobLabel: model.LabelValue(job.GetName()), - } - for label, value := range targetGroup.BaseLabels { - baseLabels[model.LabelName(label)] = model.LabelValue(value) - } - - for _, endpoint := range targetGroup.Endpoints { - newTarget := retrieval.NewTarget(endpoint, time.Second*5, baseLabels) - newTargets = append(newTargets, newTarget) - } - } - serv.appState.TargetManager.ReplaceTargets(*job, newTargets, serv.appState.Config.ScrapeInterval()) + return } + + newTargets := []retrieval.Target{} + + for _, targetGroup := range targetGroups { + // Do mandatory map type conversion due to Go shortcomings. + baseLabels := model.LabelSet{ + model.JobLabel: model.LabelValue(job.GetName()), + } + for label, value := range targetGroup.BaseLabels { + baseLabels[model.LabelName(label)] = model.LabelValue(value) + } + + for _, endpoint := range targetGroup.Endpoints { + newTarget := retrieval.NewTarget(endpoint, time.Second*5, baseLabels) + newTargets = append(newTargets, newTarget) + } + } + + // BUG(julius): Validate that this ScrapeInterval is in fact the proper one + // for the job. + serv.TargetManager.ReplaceTargets(*job, newTargets, serv.Config.ScrapeInterval()) } diff --git a/web/blob/.gitignore b/web/blob/.gitignore index ee22a0c61..05c7ebd9e 100644 --- a/web/blob/.gitignore +++ b/web/blob/.gitignore @@ -1 +1,2 @@ -files.go \ No newline at end of file +files.go +protocol_buffer.descriptor diff --git a/web/status.go b/web/status.go index c87bf6199..828837a65 100644 --- a/web/status.go +++ b/web/status.go @@ -15,27 +15,32 @@ package web import ( "flag" - "github.com/prometheus/prometheus/appstate" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/retrieval" "github.com/prometheus/prometheus/storage/metric" "net/http" + "sync" ) type PrometheusStatus struct { + BuildInfo map[string]string Config string + Curation metric.CurationState + Flags map[string]string Rules string TargetPools map[string]*retrieval.TargetPool - BuildInfo map[string]string - Flags map[string]string - Curation metric.CurationState } type StatusHandler struct { - appState *appstate.ApplicationState + BuildInfo map[string]string + Config *config.Config + CurationState chan metric.CurationState PrometheusStatus *PrometheusStatus + TargetManager retrieval.TargetManager + mutex sync.Mutex } -func (h *StatusHandler) Run() { +func (h *StatusHandler) ServeRequestsForever() { flags := map[string]string{} flag.VisitAll(func(f *flag.Flag) { @@ -43,19 +48,22 @@ func (h *StatusHandler) Run() { }) h.PrometheusStatus = &PrometheusStatus{ - Config: h.appState.Config.String(), - Rules: "TODO: list rules here", - TargetPools: h.appState.TargetManager.Pools(), - BuildInfo: h.appState.BuildInfo, + BuildInfo: h.BuildInfo, + Config: h.Config.String(), Flags: flags, + Rules: "TODO: list rules here", + TargetPools: h.TargetManager.Pools(), } - // Law of Demeter :-( - for state := range h.appState.CurationState { + for state := range h.CurationState { + h.Lock() h.PrometheusStatus.Curation = state + h.Unlock() } } -func (h StatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { +func (h *StatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + h.Lock() + defer h.Unlock() executeTemplate(w, "status", h.PrometheusStatus) } diff --git a/web/web.go b/web/web.go index a05587ee3..aa5c19ec7 100644 --- a/web/web.go +++ b/web/web.go @@ -19,7 +19,6 @@ import ( "fmt" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/exp" - "github.com/prometheus/prometheus/appstate" "github.com/prometheus/prometheus/web/api" "github.com/prometheus/prometheus/web/blob" "html/template" @@ -34,8 +33,13 @@ var ( useLocalAssets = flag.Bool("useLocalAssets", false, "Read assets/templates from file instead of binary.") ) -func StartServing(appState *appstate.ApplicationState) { - gorest.RegisterService(api.NewMetricsService(appState)) +type WebService struct { + StatusHandler *StatusHandler + MetricsHandler *api.MetricsService +} + +func (w WebService) ServeForever() error { + gorest.RegisterService(w.MetricsHandler) // TODO(julius): This will need to be rewritten once the exp package provides // the coarse mux behaviors via a wrapper function. @@ -44,10 +48,7 @@ func StartServing(appState *appstate.ApplicationState) { exp.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile)) exp.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol)) - statusHandler := &StatusHandler{appState: appState} - go statusHandler.Run() - - exp.Handle("/", statusHandler) + exp.Handle("/", w.StatusHandler) exp.HandleFunc("/graph", graphHandler) exp.Handle("/api/", gorest.Handle()) @@ -59,7 +60,8 @@ func StartServing(appState *appstate.ApplicationState) { } log.Printf("listening on %s", *listenAddress) - go http.ListenAndServe(*listenAddress, exp.DefaultCoarseMux) + + return http.ListenAndServe(*listenAddress, exp.DefaultCoarseMux) } func getTemplate(name string) (t *template.Template, err error) {