From fc16580b4ce3354a30b290b714a07f1ebd678ef6 Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Mon, 18 Mar 2013 17:48:50 +0100 Subject: [PATCH 1/7] Ship assets in compiled binary. - utility/embed-static.sh, get called in Makefile to create go map from files - web/blob/blob.go implements http Handle for serving the files from the map - web/status.go uses blog.GetFile() to get the template file The assets are gzipped and decompressed on demand. --- .gitignore | 1 + Makefile | 2 ++ utility/embed-static.sh | 38 ++++++++++++++++++++++++++++++++++++ web/blob/blob.go | 43 +++++++++++++++++++++++++++++++++++++++++ web/status.go | 9 ++++++++- web/web.go | 3 ++- 6 files changed, 94 insertions(+), 2 deletions(-) create mode 100755 utility/embed-static.sh create mode 100644 web/blob/blob.go diff --git a/.gitignore b/.gitignore index 93fe42cb52..281b2bb721 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,4 @@ [568a].out _cgo_* core +web/blob/files.go diff --git a/Makefile b/Makefile index 9dc694f154..6c2367b5c0 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,13 @@ test: build go test ./... build: + ./utility/embed-static.sh web/static web/templates > web/blob/files.go $(MAKE) -C model go build ./... go build -o prometheus.build clean: + rm -rf web/static/blob/files.go rm -rf $(TEST_ARTIFACTS) $(MAKE) -C model clean -find . -type f -iname '*~' -exec rm '{}' ';' diff --git a/utility/embed-static.sh b/utility/embed-static.sh new file mode 100755 index 0000000000..0cf0fcb86b --- /dev/null +++ b/utility/embed-static.sh @@ -0,0 +1,38 @@ +#!/bin/sh + +cat < $type_file +var types = map [string] map [string] string { +EOF + + +for dir in $@ +do + pushd "$dir" > /dev/null + echo -e "\t\"`basename $dir`\": {" + echo -e "\t\"`basename $dir`\": {" >> $type_file + + find -type f | while read file + do + mime=`mimetype -b "$file"` + name=`echo "$file"|sed 's|\.\/||'` + echo -e "\t\t\"$name\": \"$mime\"," >> $type_file + + echo -e "\t\t\"$name\": {" + gzip -9 -c "$file" | xxd -p |sed 's/\(..\)/0x\1, /g' + echo -e "\t\t}," + echo + done + echo -e "\t}," >> $type_file + echo -e "\t}," + popd > /dev/null +done +echo '}' +cat $type_file +echo '}' +rm $type_file diff --git a/web/blob/blob.go b/web/blob/blob.go new file mode 100644 index 0000000000..479cba5bb1 --- /dev/null +++ b/web/blob/blob.go @@ -0,0 +1,43 @@ +package blob + +import ( + "bytes" + "compress/gzip" + "io" + "log" + "net/http" +) + +func GetFile(bucket string, name string) ([]byte, error) { + reader := bytes.NewReader(files[bucket][name]) + gz, err := gzip.NewReader(reader) + if err != nil { + return nil, err + } + + var b bytes.Buffer + io.Copy(&b, gz) + gz.Close() + + return b.Bytes(), nil +} + +type Handler struct{} + +func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + name := r.URL.String() + if name == "" { + name = "index.html" + } + + file, err := GetFile("static", name) + if err != nil { + if err != io.EOF { + log.Printf("Could not get file: %s", err) + } + w.WriteHeader(http.StatusNotFound) + return + } + w.Header().Set("Content-Type", types["static"][name]) + w.Write(file) +} diff --git a/web/status.go b/web/status.go index f8a7ed0ae4..c30cd42680 100644 --- a/web/status.go +++ b/web/status.go @@ -16,7 +16,9 @@ package web import ( "github.com/prometheus/prometheus/appstate" "github.com/prometheus/prometheus/retrieval" + "github.com/prometheus/prometheus/web/blob" "html/template" + "log" "net/http" ) @@ -38,6 +40,11 @@ func (h *StatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Status: "TODO: add status information here", TargetPools: h.appState.TargetManager.Pools(), } - t, _ := template.ParseFiles("web/templates/status.html") + template_file, err := blob.GetFile("templates", "status.html") + if err != nil { + log.Fatalf("Could not read template: %s", err) + } + + t, _ := template.New("status").Parse(string(template_file)) t.Execute(w, status) } diff --git a/web/web.go b/web/web.go index 827d3bad83..b7a9f4350d 100644 --- a/web/web.go +++ b/web/web.go @@ -19,6 +19,7 @@ import ( "github.com/prometheus/client_golang" "github.com/prometheus/prometheus/appstate" "github.com/prometheus/prometheus/web/api" + "github.com/prometheus/prometheus/web/blob" "net/http" _ "net/http/pprof" ) @@ -35,7 +36,7 @@ func StartServing(appState *appstate.ApplicationState) { http.Handle("/status", &StatusHandler{appState: appState}) http.Handle("/api/", gorest.Handle()) http.Handle("/metrics.json", exporter) - http.Handle("/static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static")))) + http.Handle("/static/", http.StripPrefix("/static/", new(blob.Handler))) go http.ListenAndServe(*listenAddress, nil) } From 59f8ba912147997c8e6738433691f2add32e583f Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Tue, 19 Mar 2013 15:51:05 +0100 Subject: [PATCH 2/7] Replace popd/pushd because travis' /bin/sh doesn't support it. --- utility/embed-static.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utility/embed-static.sh b/utility/embed-static.sh index 0cf0fcb86b..ec91dc9e30 100755 --- a/utility/embed-static.sh +++ b/utility/embed-static.sh @@ -10,10 +10,10 @@ cat < $type_file var types = map [string] map [string] string { EOF - +CDIR=`pwd` for dir in $@ do - pushd "$dir" > /dev/null + cd "$dir" echo -e "\t\"`basename $dir`\": {" echo -e "\t\"`basename $dir`\": {" >> $type_file @@ -30,7 +30,7 @@ do done echo -e "\t}," >> $type_file echo -e "\t}," - popd > /dev/null + cd $CDIR done echo '}' cat $type_file From a5d31cb2395309d730c9f7911d4b7783adf07a95 Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Tue, 19 Mar 2013 16:11:55 +0100 Subject: [PATCH 3/7] Use constants for blob bucket keys. --- web/blob/blob.go | 9 +++++++-- web/status.go | 4 ++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/web/blob/blob.go b/web/blob/blob.go index 479cba5bb1..27907ad2d1 100644 --- a/web/blob/blob.go +++ b/web/blob/blob.go @@ -8,6 +8,11 @@ import ( "net/http" ) +const ( + TemplateFiles = "templates" + StaticFiles = "static" +) + func GetFile(bucket string, name string) ([]byte, error) { reader := bytes.NewReader(files[bucket][name]) gz, err := gzip.NewReader(reader) @@ -30,7 +35,7 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { name = "index.html" } - file, err := GetFile("static", name) + file, err := GetFile(StaticFiles, name) if err != nil { if err != io.EOF { log.Printf("Could not get file: %s", err) @@ -38,6 +43,6 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) return } - w.Header().Set("Content-Type", types["static"][name]) + w.Header().Set("Content-Type", types[StaticFiles][name]) w.Write(file) } diff --git a/web/status.go b/web/status.go index c30cd42680..7f6bf6396d 100644 --- a/web/status.go +++ b/web/status.go @@ -40,11 +40,11 @@ func (h *StatusHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { Status: "TODO: add status information here", TargetPools: h.appState.TargetManager.Pools(), } - template_file, err := blob.GetFile("templates", "status.html") + templateFile, err := blob.GetFile(blob.TemplateFiles, "status.html") if err != nil { log.Fatalf("Could not read template: %s", err) } - t, _ := template.New("status").Parse(string(template_file)) + t, _ := template.New("status").Parse(string(templateFile)) t.Execute(w, status) } From 2434bac219f91afc5ca4747c5267d7316ce152b2 Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Tue, 19 Mar 2013 16:28:55 +0100 Subject: [PATCH 4/7] Detect mime type on runtime instead of hardcoding. --- utility/embed-static.sh | 13 ------------- web/blob/blob.go | 4 ++-- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/utility/embed-static.sh b/utility/embed-static.sh index ec91dc9e30..1e14602f2d 100755 --- a/utility/embed-static.sh +++ b/utility/embed-static.sh @@ -5,34 +5,21 @@ package blob var files = map [string] map [string] []byte { EOF -type_file=`tempfile` -cat < $type_file -var types = map [string] map [string] string { -EOF - CDIR=`pwd` for dir in $@ do cd "$dir" echo -e "\t\"`basename $dir`\": {" - echo -e "\t\"`basename $dir`\": {" >> $type_file find -type f | while read file do - mime=`mimetype -b "$file"` name=`echo "$file"|sed 's|\.\/||'` - echo -e "\t\t\"$name\": \"$mime\"," >> $type_file - echo -e "\t\t\"$name\": {" gzip -9 -c "$file" | xxd -p |sed 's/\(..\)/0x\1, /g' echo -e "\t\t}," echo done - echo -e "\t}," >> $type_file echo -e "\t}," cd $CDIR done echo '}' -cat $type_file -echo '}' -rm $type_file diff --git a/web/blob/blob.go b/web/blob/blob.go index 27907ad2d1..5d9e4c1353 100644 --- a/web/blob/blob.go +++ b/web/blob/blob.go @@ -10,7 +10,7 @@ import ( const ( TemplateFiles = "templates" - StaticFiles = "static" + StaticFiles = "static" ) func GetFile(bucket string, name string) ([]byte, error) { @@ -43,6 +43,6 @@ func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) return } - w.Header().Set("Content-Type", types[StaticFiles][name]) + w.Header().Set("Content-Type", http.DetectContentType(file)) w.Write(file) } From 920c59e12d3cf685d2bf5f071dcf6ffef4712a7e Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Tue, 19 Mar 2013 18:01:21 +0100 Subject: [PATCH 5/7] Added vim-common to travis build environment. vim-common includes `xxd` which we use to hexdump files for package blob. --- Makefile.TRAVIS | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Makefile.TRAVIS b/Makefile.TRAVIS index c9ee349185..b92d80dc43 100644 --- a/Makefile.TRAVIS +++ b/Makefile.TRAVIS @@ -41,7 +41,7 @@ preparation-stamp: build-dependencies build-dependencies: build-dependencies-stamp -build-dependencies-stamp: bison cc mercurial protoc goprotobuf gorest go instrumentation leveldb levigo skiplist +build-dependencies-stamp: bison cc mercurial protoc goprotobuf gorest go instrumentation leveldb levigo skiplist vim-common touch $@ overlay: overlay-stamp @@ -144,6 +144,12 @@ rsync: rsync-stamp rsync-stamp: [ -x "$$(which rsync)" ] || $(APT_GET_INSTALL) rsync +vim-common: vim-common-stamp + +vim-common-stamp: + $(APT_GET_INSTALL) vim-common + touch $@ + test: test-stamp test-stamp: preparation source @@ -183,4 +189,4 @@ clean: -rm snappy-$(SNAPPY_VERSION).tar.gz -.PHONY: all bison build-dependencies cc clean go goprotobuf gorest instrumentation leveldb levigo mercurial overlay preparation protoc rsync snappy source test wget +.PHONY: all bison build-dependencies cc clean go goprotobuf gorest instrumentation leveldb levigo mercurial overlay preparation protoc rsync snappy source test wget vim-common From aed8270a9797eca017d0b367b42f709f45134de1 Mon Sep 17 00:00:00 2001 From: Johannes 'fish' Ziemke Date: Wed, 20 Mar 2013 11:54:45 +0100 Subject: [PATCH 6/7] Enable debugging (set -x) on embed-static.sh. --- Makefile | 1 + utility/embed-static.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 6c2367b5c0..79f8ea1da2 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ test: build build: ./utility/embed-static.sh web/static web/templates > web/blob/files.go + cat web/blob/files.go $(MAKE) -C model go build ./... go build -o prometheus.build diff --git a/utility/embed-static.sh b/utility/embed-static.sh index 1e14602f2d..1942e324de 100755 --- a/utility/embed-static.sh +++ b/utility/embed-static.sh @@ -1,4 +1,5 @@ #!/bin/sh +set -x cat < Date: Wed, 20 Mar 2013 12:31:42 +0100 Subject: [PATCH 7/7] Remove echo -e because it's not available on Travis. Instead, files.go gets piped through gofmt. --- Makefile | 2 +- utility/embed-static.sh | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 79f8ea1da2..b2f9b6cc7e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ test: build go test ./... build: - ./utility/embed-static.sh web/static web/templates > web/blob/files.go + ./utility/embed-static.sh web/static web/templates | gofmt > web/blob/files.go cat web/blob/files.go $(MAKE) -C model go build ./... diff --git a/utility/embed-static.sh b/utility/embed-static.sh index 1942e324de..522610a01e 100755 --- a/utility/embed-static.sh +++ b/utility/embed-static.sh @@ -10,17 +10,17 @@ CDIR=`pwd` for dir in $@ do cd "$dir" - echo -e "\t\"`basename $dir`\": {" + echo "\"`basename $dir`\": {" find -type f | while read file do name=`echo "$file"|sed 's|\.\/||'` - echo -e "\t\t\"$name\": {" + echo "\"$name\": {" gzip -9 -c "$file" | xxd -p |sed 's/\(..\)/0x\1, /g' - echo -e "\t\t}," + echo "}," echo done - echo -e "\t}," + echo "}," cd $CDIR done echo '}'