Merge pull request #3928 from prometheus/snaphead

Add option to skip head on snapshots
This commit is contained in:
Fabian Reinartz 2018-03-08 13:25:22 +01:00 committed by GitHub
commit 60edc2b6d5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 104 additions and 52 deletions

View file

@ -452,9 +452,10 @@ We also expose a gRPC API whose definition can be found [here](https://github.co
### Snapshot
Snapshot creates a snapshot of all current data into `snapshots/<datetime>-<rand>` under the TSDB's data directory and returns the directory as response.
It will optionally skip snapshotting data that is only present in the head block, and which has not yet been compacted to disk.
```
POST /api/v1/admin/tsdb/snapshot
POST /api/v1/admin/tsdb/snapshot?skip_head=<bool>
```
```json

View file

@ -26,6 +26,7 @@ var _ = math.Inf
var _ = time.Kitchen
type TSDBSnapshotRequest struct {
SkipHead bool `protobuf:"varint,1,opt,name=skip_head,json=skipHead,proto3" json:"skip_head,omitempty"`
}
func (m *TSDBSnapshotRequest) Reset() { *m = TSDBSnapshotRequest{} }
@ -245,6 +246,16 @@ func (m *TSDBSnapshotRequest) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if m.SkipHead {
dAtA[i] = 0x8
i++
if m.SkipHead {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -388,6 +399,9 @@ func encodeVarintRpc(dAtA []byte, offset int, v uint64) int {
func (m *TSDBSnapshotRequest) Size() (n int) {
var l int
_ = l
if m.SkipHead {
n += 2
}
return n
}
@ -481,6 +495,26 @@ func (m *TSDBSnapshotRequest) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: TSDBSnapshotRequest: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field SkipHead", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRpc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.SkipHead = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipRpc(dAtA[iNdEx:])
@ -986,34 +1020,35 @@ var (
func init() { proto.RegisterFile("rpc.proto", fileDescriptorRpc) }
var fileDescriptorRpc = []byte{
// 451 bytes of a gzipped FileDescriptorProto
// 471 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x3d, 0x8f, 0xd3, 0x40,
0x10, 0xbd, 0xbd, 0x84, 0x83, 0xdb, 0x5c, 0xe5, 0x0b, 0x10, 0x4c, 0x88, 0x83, 0x0b, 0xee, 0x74,
0x85, 0x57, 0x0a, 0xdd, 0x51, 0x11, 0xae, 0x84, 0x26, 0x49, 0x45, 0x13, 0xad, 0x93, 0xc1, 0xb1,
0xe4, 0xfd, 0xc0, 0xbb, 0x41, 0x07, 0x25, 0x1d, 0x15, 0x48, 0xfc, 0xa9, 0x48, 0x34, 0x48, 0xf4,
0x7c, 0x44, 0xfc, 0x10, 0xb4, 0xbb, 0xf6, 0x5d, 0x6c, 0x19, 0x41, 0x37, 0x1e, 0xbf, 0x37, 0x6f,
0xde, 0x9b, 0xc5, 0x87, 0xb9, 0x5c, 0x44, 0x32, 0x17, 0x5a, 0x78, 0x58, 0xe6, 0x82, 0x81, 0x5e,
0xc1, 0x5a, 0xf9, 0x1d, 0xfd, 0x56, 0x82, 0x72, 0x3f, 0xfc, 0x20, 0x11, 0x22, 0xc9, 0x80, 0xd8,
0xaf, 0x78, 0xfd, 0x8a, 0xe8, 0x94, 0x81, 0xd2, 0x94, 0xc9, 0x02, 0xd0, 0x2f, 0x00, 0x54, 0xa6,
0x84, 0x72, 0x2e, 0x34, 0xd5, 0xa9, 0xe0, 0x25, 0xbd, 0x9b, 0x88, 0x44, 0xd8, 0x92, 0x98, 0xca,
0x75, 0xc3, 0xdb, 0xf8, 0x78, 0x36, 0xbd, 0x18, 0x4f, 0x39, 0x95, 0x6a, 0x25, 0xf4, 0x04, 0x5e,
0xaf, 0x41, 0xe9, 0xf0, 0x0c, 0x77, 0xab, 0x6d, 0x25, 0x05, 0x57, 0xe0, 0x79, 0xb8, 0xcd, 0x29,
0x83, 0x1e, 0x1a, 0xa2, 0xd3, 0xc3, 0x89, 0xad, 0xc3, 0x3e, 0xf6, 0x0d, 0xf6, 0x59, 0x06, 0x94,
0xcf, 0x04, 0x8b, 0x95, 0x16, 0x1c, 0x54, 0x39, 0xe9, 0x01, 0xbe, 0xdf, 0xf8, 0xd7, 0x0d, 0x0c,
0xbf, 0x20, 0x7c, 0x3c, 0x85, 0x3c, 0x05, 0x75, 0x01, 0x19, 0x68, 0x28, 0x68, 0xde, 0x13, 0x7c,
0x8b, 0xa5, 0x7c, 0x6e, 0x2c, 0x5a, 0xb1, 0xce, 0xc8, 0x8f, 0x9c, 0xbd, 0xa8, 0xf4, 0x1f, 0xcd,
0x4a, 0xff, 0xe3, 0xf6, 0xa7, 0x1f, 0x01, 0x9a, 0xdc, 0x64, 0x29, 0x37, 0x3d, 0x4b, 0xa6, 0x97,
0x8e, 0xbc, 0xff, 0xdf, 0x64, 0x7a, 0x69, 0xc9, 0xe7, 0x86, 0xac, 0x17, 0x2b, 0xc8, 0x55, 0xaf,
0x35, 0x6c, 0x9d, 0x76, 0x46, 0xbd, 0xe8, 0xfa, 0x24, 0xd1, 0x73, 0x1a, 0x43, 0xf6, 0xc2, 0x01,
0xc6, 0xed, 0xcd, 0xf7, 0x60, 0x6f, 0x72, 0x85, 0x0f, 0xef, 0xe0, 0x6e, 0xd5, 0x8c, 0x73, 0x39,
0xfa, 0xd0, 0xc2, 0x37, 0x9e, 0x2e, 0x59, 0xca, 0xbd, 0x1c, 0x1f, 0xed, 0x06, 0xeb, 0x05, 0xbb,
0xb3, 0x1b, 0x2e, 0xe1, 0x0f, 0xff, 0x0e, 0x28, 0x22, 0x0c, 0xde, 0x7f, 0xfb, 0xfd, 0x79, 0xff,
0x5e, 0x78, 0x97, 0xbc, 0x19, 0x11, 0x6a, 0x54, 0x88, 0x56, 0xcb, 0x98, 0xa8, 0x52, 0xe3, 0x23,
0x72, 0x47, 0xae, 0xdd, 0xc0, 0x7b, 0x54, 0x1f, 0xdd, 0x7c, 0x42, 0xff, 0xe4, 0x9f, 0xb8, 0x62,
0x93, 0x13, 0xbb, 0xc9, 0xc3, 0x30, 0xa8, 0x6d, 0xb2, 0x30, 0xf8, 0xb9, 0xbe, 0x56, 0x7e, 0x87,
0x8f, 0x5c, 0x42, 0x2e, 0xad, 0x6a, 0x0a, 0x0d, 0xcf, 0xa1, 0x9a, 0x42, 0x53, 0xc4, 0x57, 0xda,
0xfd, 0x9a, 0xf6, 0xd2, 0xc2, 0xe6, 0xca, 0x72, 0xce, 0xd1, 0xd9, 0xb8, 0xb7, 0xf9, 0x35, 0xd8,
0xdb, 0x6c, 0x07, 0xe8, 0xeb, 0x76, 0x80, 0x7e, 0x6e, 0x07, 0xe8, 0xe5, 0x81, 0x99, 0x2d, 0xe3,
0xf8, 0xc0, 0x3e, 0x8e, 0xc7, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x6d, 0x79, 0xf1, 0x8d,
0x03, 0x00, 0x00,
0x10, 0xbd, 0xbd, 0x84, 0x23, 0xd9, 0x5c, 0xe5, 0x8b, 0x20, 0xf8, 0x42, 0x1c, 0x5c, 0x70, 0xa7,
0x2b, 0x6c, 0xc9, 0x74, 0x47, 0x45, 0xb8, 0x82, 0x02, 0x1a, 0x27, 0x15, 0x4d, 0xb4, 0x8e, 0x87,
0xc4, 0x22, 0xfb, 0x81, 0x77, 0x83, 0x0e, 0x4a, 0x3a, 0x2a, 0x90, 0xf8, 0x53, 0x91, 0x68, 0x90,
0xe8, 0xf9, 0x88, 0xf8, 0x21, 0x68, 0x77, 0xed, 0xbb, 0xc4, 0x32, 0xe2, 0xba, 0xd9, 0xd9, 0xf7,
0xe6, 0xcd, 0xbc, 0x19, 0xdc, 0xce, 0xc5, 0x2c, 0x10, 0x39, 0x57, 0xdc, 0xc1, 0x22, 0xe7, 0x14,
0xd4, 0x02, 0x56, 0xd2, 0xed, 0xa8, 0x77, 0x02, 0xa4, 0xfd, 0x70, 0xbd, 0x39, 0xe7, 0xf3, 0x25,
0x84, 0xe6, 0x95, 0xac, 0x5e, 0x85, 0x2a, 0xa3, 0x20, 0x15, 0xa1, 0xa2, 0x00, 0xf4, 0x0b, 0x00,
0x11, 0x59, 0x48, 0x18, 0xe3, 0x8a, 0xa8, 0x8c, 0xb3, 0x92, 0xde, 0x9d, 0xf3, 0x39, 0x37, 0x61,
0xa8, 0x23, 0x9b, 0xf5, 0x23, 0x7c, 0x34, 0x19, 0x5f, 0x8c, 0xc6, 0x8c, 0x08, 0xb9, 0xe0, 0x2a,
0x86, 0x37, 0x2b, 0x90, 0xca, 0x39, 0xc6, 0x6d, 0xf9, 0x3a, 0x13, 0xd3, 0x05, 0x90, 0xb4, 0x87,
0x86, 0xe8, 0xb4, 0x15, 0xb7, 0x74, 0xe2, 0x19, 0x90, 0xd4, 0x3f, 0xc3, 0xdd, 0x5d, 0x8e, 0x14,
0x9c, 0x49, 0x70, 0x1c, 0xdc, 0x64, 0x84, 0x82, 0xc1, 0xb7, 0x63, 0x13, 0xfb, 0x7d, 0xec, 0x6a,
0xec, 0xd3, 0x25, 0x10, 0x36, 0xe1, 0x34, 0x91, 0x8a, 0x33, 0x90, 0x85, 0x8c, 0x7f, 0x1f, 0x1f,
0xd7, 0xfe, 0xda, 0x82, 0xfe, 0x57, 0x84, 0x8f, 0xc6, 0x90, 0x67, 0x20, 0x2f, 0x60, 0x09, 0x0a,
0xca, 0xee, 0x1e, 0xe3, 0x16, 0xcd, 0xd8, 0x54, 0xcf, 0x6f, 0xc4, 0x3a, 0x91, 0x1b, 0xd8, 0xd9,
0x83, 0xd2, 0x9c, 0x60, 0x52, 0x9a, 0x33, 0x6a, 0x7e, 0xfe, 0xe9, 0xa1, 0xf8, 0x36, 0xcd, 0x98,
0xce, 0x19, 0x32, 0xb9, 0xb4, 0xe4, 0xfd, 0x1b, 0x93, 0xc9, 0xa5, 0x21, 0x9f, 0x6b, 0xb2, 0x9a,
0x2d, 0x20, 0x97, 0xbd, 0xc6, 0xb0, 0x71, 0xda, 0x89, 0x7a, 0xc1, 0xf5, 0xbe, 0x82, 0xe7, 0x24,
0x81, 0xe5, 0x0b, 0x0b, 0x18, 0x35, 0xd7, 0x3f, 0xbc, 0xbd, 0xf8, 0x0a, 0xef, 0xdf, 0xc1, 0xdd,
0xdd, 0x61, 0xec, 0x94, 0xd1, 0xc7, 0x06, 0xbe, 0xf5, 0x24, 0xa5, 0x19, 0x73, 0x72, 0x7c, 0xb8,
0x6d, 0xac, 0xe3, 0x6d, 0xd7, 0xae, 0x59, 0x93, 0x3b, 0xfc, 0x37, 0xa0, 0xb0, 0xd0, 0xfb, 0xf0,
0xfd, 0xcf, 0x97, 0xfd, 0x7b, 0xfe, 0xdd, 0xf0, 0x6d, 0x14, 0x12, 0xad, 0x12, 0x2a, 0x99, 0x26,
0xa1, 0x2c, 0x35, 0x3e, 0x21, 0x7b, 0x01, 0x95, 0x1d, 0x38, 0x0f, 0xab, 0xa5, 0xeb, 0x57, 0xe8,
0x9e, 0xfc, 0x17, 0x57, 0x74, 0x72, 0x62, 0x3a, 0x79, 0xe0, 0x7b, 0x95, 0x4e, 0x66, 0x1a, 0x3f,
0x55, 0xd7, 0xca, 0xef, 0xf1, 0xa1, 0x75, 0xc8, 0xba, 0xb5, 0xeb, 0x42, 0xcd, 0x39, 0xec, 0xba,
0x50, 0x67, 0xf1, 0x95, 0x76, 0xbf, 0xa2, 0x9d, 0x1a, 0xd8, 0x54, 0x1a, 0xce, 0x39, 0x3a, 0x1b,
0xf5, 0xd6, 0xbf, 0x07, 0x7b, 0xeb, 0xcd, 0x00, 0x7d, 0xdb, 0x0c, 0xd0, 0xaf, 0xcd, 0x00, 0xbd,
0x3c, 0xd0, 0xb5, 0x45, 0x92, 0x1c, 0x98, 0xe3, 0x78, 0xf4, 0x37, 0x00, 0x00, 0xff, 0xff, 0x8b,
0x21, 0x42, 0x5d, 0xaa, 0x03, 0x00, 0x00,
}

View file

@ -28,10 +28,18 @@ var _ status.Status
var _ = runtime.String
var _ = utilities.NewDoubleArray
var (
filter_Admin_TSDBSnapshot_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)}
)
func request_Admin_TSDBSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, client AdminClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
var protoReq TSDBSnapshotRequest
var metadata runtime.ServerMetadata
if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_Admin_TSDBSnapshot_0); err != nil {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}
msg, err := client.TSDBSnapshot(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
return msg, metadata, err
@ -50,10 +58,8 @@ func request_Admin_DeleteSeries_0(ctx context.Context, marshaler runtime.Marshal
var protoReq SeriesDeleteRequest
var metadata runtime.ServerMetadata
if req.ContentLength > 0 {
if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}
if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}
msg, err := client.DeleteSeries(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
@ -100,7 +106,7 @@ func RegisterAdminHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc
func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, client AdminClient) error {
mux.Handle("POST", pattern_Admin_TSDBSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
if cn, ok := w.(http.CloseNotifier); ok {
go func(done <-chan struct{}, closed <-chan bool) {
@ -129,7 +135,7 @@ func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, clie
})
mux.Handle("POST", pattern_Admin_TSDBCleanTombstones_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
if cn, ok := w.(http.CloseNotifier); ok {
go func(done <-chan struct{}, closed <-chan bool) {
@ -158,7 +164,7 @@ func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, clie
})
mux.Handle("POST", pattern_Admin_DeleteSeries_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context())
ctx, cancel := context.WithCancel(ctx)
defer cancel()
if cn, ok := w.(http.CloseNotifier); ok {
go func(done <-chan struct{}, closed <-chan bool) {

View file

@ -53,6 +53,7 @@ service Admin {
message TSDBSnapshotRequest {
bool skip_head = 1;
}
message TSDBSnapshotResponse {

View file

@ -151,6 +151,13 @@ func (c *LeveledCompactor) Plan(dir string) ([]string, error) {
if err != nil {
return nil, err
}
// We do not include the most recently created block. This gives users a window
// of a full block size to piece-wise backup new data without having to care
// about data overlap.
if len(dirs) < 1 {
return nil, nil
}
dirs = dirs[:len(dirs)-1]
var dms []dirMeta

View file

@ -42,7 +42,7 @@ import (
)
// DefaultOptions used for the DB. They are sane for setups using
// millisecond precision timestampdb.
// millisecond precision timestamps.
var DefaultOptions = &Options{
WALFlushInterval: 5 * time.Second,
RetentionDuration: 15 * 24 * 60 * 60 * 1000, // 15 days in milliseconds
@ -633,8 +633,9 @@ func (db *DB) EnableCompactions() {
level.Info(db.logger).Log("msg", "compactions enabled")
}
// Snapshot writes the current data to the directory.
func (db *DB) Snapshot(dir string) error {
// Snapshot writes the current data to the directory. If withHead is set to true it
// will create a new block containing all data that's currently in the memory buffer/WAL.
func (db *DB) Snapshot(dir string, withHead bool) error {
if dir == db.dir {
return errors.Errorf("cannot snapshot into base directory")
}
@ -655,6 +656,9 @@ func (db *DB) Snapshot(dir string) error {
return errors.Wrapf(err, "error snapshotting block: %s", b.Dir())
}
}
if !withHead {
return nil
}
_, err := db.compactor.Write(dir, db.head, db.head.MinTime(), db.head.MaxTime())
return errors.Wrap(err, "snapshot head block")
}

View file

@ -265,7 +265,7 @@ func (h *Head) ReadWAL() error {
// TODO(fabxc): series entries spread between samples can starve the sample workers.
// Even with bufferd channels, this can impact startup time with lots of series churn.
// We must not pralellize series creation itself but could make the indexing asynchronous.
// We must not paralellize series creation itself but could make the indexing asynchronous.
seriesFunc := func(series []RefSeries) {
for _, s := range series {
h.getOrCreateWithID(s.Ref, s.Labels.Hash(), s.Labels)
@ -762,10 +762,6 @@ func (c *safeChunk) Iterator() chunkenc.Iterator {
return it
}
// func (c *safeChunk) Appender() (chunks.Appender, error) { panic("illegal") }
// func (c *safeChunk) Bytes() []byte { panic("illegal") }
// func (c *safeChunk) Encoding() chunks.Encoding { panic("illegal") }
type headIndexReader struct {
head *Head
mint, maxt int64
@ -1259,7 +1255,7 @@ func (s *memSeries) iterator(id int) chunkenc.Iterator {
if id-s.firstChunkID < len(s.chunks)-1 {
return c.chunk.Iterator()
}
// Serve the last 4 samples for the last chunk from the series buffer
// Serve the last 4 samples for the last chunk from the sample buffer
// as their compressed bytes may be mutated by added samples.
it := &memSafeIterator{
Iterator: c.chunk.Iterator(),

6
vendor/vendor.json vendored
View file

@ -800,10 +800,10 @@
"revisionTime": "2016-04-11T19:08:41Z"
},
{
"checksumSHA1": "CeD8QwiLL5CBkWMOfbaJxs4AFuM=",
"checksumSHA1": "zVgXlbZ1J8GhBN7tZji7M/SuiAU=",
"path": "github.com/prometheus/tsdb",
"revision": "494acd307058387ced7646f9996b0f7372eaa558",
"revisionTime": "2018-02-15T11:29:47Z"
"revision": "16b2bf1b45ce3e3536c78ebec5116ea09a69786e",
"revisionTime": "2018-03-02T11:51:49Z"
},
{
"checksumSHA1": "S7F4yWxVLhxQNHMdgoOo6plmOOs=",

View file

@ -657,6 +657,8 @@ func (api *API) snapshot(r *http.Request) (interface{}, *apiError) {
if !api.enableAdmin {
return nil, &apiError{errorUnavailable, errors.New("Admin APIs disabled")}
}
skipHead, _ := strconv.ParseBool(r.FormValue("skip_head"))
db := api.db()
if db == nil {
return nil, &apiError{errorUnavailable, errors.New("TSDB not ready")}
@ -672,7 +674,7 @@ func (api *API) snapshot(r *http.Request) (interface{}, *apiError) {
if err := os.MkdirAll(dir, 0777); err != nil {
return nil, &apiError{errorInternal, fmt.Errorf("create snapshot directory: %s", err)}
}
if err := db.Snapshot(dir); err != nil {
if err := db.Snapshot(dir, !skipHead); err != nil {
return nil, &apiError{errorInternal, fmt.Errorf("create snapshot: %s", err)}
}

View file

@ -167,7 +167,7 @@ func NewAdmin(db func() *tsdb.DB) *Admin {
}
// TSDBSnapshot implements pb.AdminServer.
func (s *Admin) TSDBSnapshot(_ old_ctx.Context, _ *pb.TSDBSnapshotRequest) (*pb.TSDBSnapshotResponse, error) {
func (s *Admin) TSDBSnapshot(_ old_ctx.Context, req *pb.TSDBSnapshotRequest) (*pb.TSDBSnapshotResponse, error) {
db := s.db()
if db == nil {
return nil, status.Errorf(codes.Unavailable, "TSDB not ready")
@ -182,7 +182,7 @@ func (s *Admin) TSDBSnapshot(_ old_ctx.Context, _ *pb.TSDBSnapshotRequest) (*pb.
if err := os.MkdirAll(dir, 0777); err != nil {
return nil, status.Errorf(codes.Internal, "created snapshot directory: %s", err)
}
if err := db.Snapshot(dir); err != nil {
if err := db.Snapshot(dir, !req.SkipHead); err != nil {
return nil, status.Errorf(codes.Internal, "create snapshot: %s", err)
}
return &pb.TSDBSnapshotResponse{Name: name}, nil