api: add flag to skip head on snapshots

This commit is contained in:
Fabian Reinartz 2018-03-08 11:43:41 +01:00
parent 07299cec17
commit 3e6c890aea
6 changed files with 85 additions and 40 deletions

View file

@ -452,9 +452,10 @@ We also expose a gRPC API whose definition can be found [here](https://github.co
### Snapshot ### Snapshot
Snapshot creates a snapshot of all current data into `snapshots/<datetime>-<rand>` under the TSDB's data directory and returns the directory as response. Snapshot creates a snapshot of all current data into `snapshots/<datetime>-<rand>` under the TSDB's data directory and returns the directory as response.
It will optionally skip snapshotting data that is only present in the head block, and which has not yet been compacted to disk.
``` ```
POST /api/v1/admin/tsdb/snapshot POST /api/v1/admin/tsdb/snapshot?skip_head=<bool>
``` ```
```json ```json

View file

@ -26,6 +26,7 @@ var _ = math.Inf
var _ = time.Kitchen var _ = time.Kitchen
type TSDBSnapshotRequest struct { type TSDBSnapshotRequest struct {
SkipHead bool `protobuf:"varint,1,opt,name=skip_head,json=skipHead,proto3" json:"skip_head,omitempty"`
} }
func (m *TSDBSnapshotRequest) Reset() { *m = TSDBSnapshotRequest{} } func (m *TSDBSnapshotRequest) Reset() { *m = TSDBSnapshotRequest{} }
@ -245,6 +246,16 @@ func (m *TSDBSnapshotRequest) MarshalTo(dAtA []byte) (int, error) {
_ = i _ = i
var l int var l int
_ = l _ = l
if m.SkipHead {
dAtA[i] = 0x8
i++
if m.SkipHead {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil return i, nil
} }
@ -388,6 +399,9 @@ func encodeVarintRpc(dAtA []byte, offset int, v uint64) int {
func (m *TSDBSnapshotRequest) Size() (n int) { func (m *TSDBSnapshotRequest) Size() (n int) {
var l int var l int
_ = l _ = l
if m.SkipHead {
n += 2
}
return n return n
} }
@ -481,6 +495,26 @@ func (m *TSDBSnapshotRequest) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: TSDBSnapshotRequest: illegal tag %d (wire type %d)", fieldNum, wire) return fmt.Errorf("proto: TSDBSnapshotRequest: illegal tag %d (wire type %d)", fieldNum, wire)
} }
switch fieldNum { switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field SkipHead", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowRpc
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.SkipHead = bool(v != 0)
default: default:
iNdEx = preIndex iNdEx = preIndex
skippy, err := skipRpc(dAtA[iNdEx:]) skippy, err := skipRpc(dAtA[iNdEx:])
@ -986,34 +1020,35 @@ var (
func init() { proto.RegisterFile("rpc.proto", fileDescriptorRpc) } func init() { proto.RegisterFile("rpc.proto", fileDescriptorRpc) }
var fileDescriptorRpc = []byte{ var fileDescriptorRpc = []byte{
// 451 bytes of a gzipped FileDescriptorProto // 471 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x3d, 0x8f, 0xd3, 0x40, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0x3d, 0x8f, 0xd3, 0x40,
0x10, 0xbd, 0xbd, 0x84, 0x83, 0xdb, 0x5c, 0xe5, 0x0b, 0x10, 0x4c, 0x88, 0x83, 0x0b, 0xee, 0x74, 0x10, 0xbd, 0xbd, 0x84, 0x23, 0xd9, 0x5c, 0xe5, 0x8b, 0x20, 0xf8, 0x42, 0x1c, 0x5c, 0x70, 0xa7,
0x85, 0x57, 0x0a, 0xdd, 0x51, 0x11, 0xae, 0x84, 0x26, 0x49, 0x45, 0x13, 0xad, 0x93, 0xc1, 0xb1, 0x2b, 0x6c, 0xc9, 0x74, 0x47, 0x45, 0xb8, 0x82, 0x02, 0x1a, 0x27, 0x15, 0x4d, 0xb4, 0x8e, 0x87,
0xe4, 0xfd, 0xc0, 0xbb, 0x41, 0x07, 0x25, 0x1d, 0x15, 0x48, 0xfc, 0xa9, 0x48, 0x34, 0x48, 0xf4, 0xc4, 0x22, 0xfb, 0x81, 0x77, 0x83, 0x0e, 0x4a, 0x3a, 0x2a, 0x90, 0xf8, 0x53, 0x91, 0x68, 0x90,
0x7c, 0x44, 0xfc, 0x10, 0xb4, 0xbb, 0xf6, 0x5d, 0x6c, 0x19, 0x41, 0x37, 0x1e, 0xbf, 0x37, 0x6f, 0xe8, 0xf9, 0x88, 0xf8, 0x21, 0x68, 0x77, 0xed, 0xbb, 0xc4, 0x32, 0xe2, 0xba, 0xd9, 0xd9, 0xf7,
0xde, 0x9b, 0xc5, 0x87, 0xb9, 0x5c, 0x44, 0x32, 0x17, 0x5a, 0x78, 0x58, 0xe6, 0x82, 0x81, 0x5e, 0xe6, 0xcd, 0xbc, 0x19, 0xdc, 0xce, 0xc5, 0x2c, 0x10, 0x39, 0x57, 0xdc, 0xc1, 0x22, 0xe7, 0x14,
0xc1, 0x5a, 0xf9, 0x1d, 0xfd, 0x56, 0x82, 0x72, 0x3f, 0xfc, 0x20, 0x11, 0x22, 0xc9, 0x80, 0xd8, 0xd4, 0x02, 0x56, 0xd2, 0xed, 0xa8, 0x77, 0x02, 0xa4, 0xfd, 0x70, 0xbd, 0x39, 0xe7, 0xf3, 0x25,
0xaf, 0x78, 0xfd, 0x8a, 0xe8, 0x94, 0x81, 0xd2, 0x94, 0xc9, 0x02, 0xd0, 0x2f, 0x00, 0x54, 0xa6, 0x84, 0xe6, 0x95, 0xac, 0x5e, 0x85, 0x2a, 0xa3, 0x20, 0x15, 0xa1, 0xa2, 0x00, 0xf4, 0x0b, 0x00,
0x84, 0x72, 0x2e, 0x34, 0xd5, 0xa9, 0xe0, 0x25, 0xbd, 0x9b, 0x88, 0x44, 0xd8, 0x92, 0x98, 0xca, 0x11, 0x59, 0x48, 0x18, 0xe3, 0x8a, 0xa8, 0x8c, 0xb3, 0x92, 0xde, 0x9d, 0xf3, 0x39, 0x37, 0x61,
0x75, 0xc3, 0xdb, 0xf8, 0x78, 0x36, 0xbd, 0x18, 0x4f, 0x39, 0x95, 0x6a, 0x25, 0xf4, 0x04, 0x5e, 0xa8, 0x23, 0x9b, 0xf5, 0x23, 0x7c, 0x34, 0x19, 0x5f, 0x8c, 0xc6, 0x8c, 0x08, 0xb9, 0xe0, 0x2a,
0xaf, 0x41, 0xe9, 0xf0, 0x0c, 0x77, 0xab, 0x6d, 0x25, 0x05, 0x57, 0xe0, 0x79, 0xb8, 0xcd, 0x29, 0x86, 0x37, 0x2b, 0x90, 0xca, 0x39, 0xc6, 0x6d, 0xf9, 0x3a, 0x13, 0xd3, 0x05, 0x90, 0xb4, 0x87,
0x83, 0x1e, 0x1a, 0xa2, 0xd3, 0xc3, 0x89, 0xad, 0xc3, 0x3e, 0xf6, 0x0d, 0xf6, 0x59, 0x06, 0x94, 0x86, 0xe8, 0xb4, 0x15, 0xb7, 0x74, 0xe2, 0x19, 0x90, 0xd4, 0x3f, 0xc3, 0xdd, 0x5d, 0x8e, 0x14,
0xcf, 0x04, 0x8b, 0x95, 0x16, 0x1c, 0x54, 0x39, 0xe9, 0x01, 0xbe, 0xdf, 0xf8, 0xd7, 0x0d, 0x0c, 0x9c, 0x49, 0x70, 0x1c, 0xdc, 0x64, 0x84, 0x82, 0xc1, 0xb7, 0x63, 0x13, 0xfb, 0x7d, 0xec, 0x6a,
0xbf, 0x20, 0x7c, 0x3c, 0x85, 0x3c, 0x05, 0x75, 0x01, 0x19, 0x68, 0x28, 0x68, 0xde, 0x13, 0x7c, 0xec, 0xd3, 0x25, 0x10, 0x36, 0xe1, 0x34, 0x91, 0x8a, 0x33, 0x90, 0x85, 0x8c, 0x7f, 0x1f, 0x1f,
0x8b, 0xa5, 0x7c, 0x6e, 0x2c, 0x5a, 0xb1, 0xce, 0xc8, 0x8f, 0x9c, 0xbd, 0xa8, 0xf4, 0x1f, 0xcd, 0xd7, 0xfe, 0xda, 0x82, 0xfe, 0x57, 0x84, 0x8f, 0xc6, 0x90, 0x67, 0x20, 0x2f, 0x60, 0x09, 0x0a,
0x4a, 0xff, 0xe3, 0xf6, 0xa7, 0x1f, 0x01, 0x9a, 0xdc, 0x64, 0x29, 0x37, 0x3d, 0x4b, 0xa6, 0x97, 0xca, 0xee, 0x1e, 0xe3, 0x16, 0xcd, 0xd8, 0x54, 0xcf, 0x6f, 0xc4, 0x3a, 0x91, 0x1b, 0xd8, 0xd9,
0x8e, 0xbc, 0xff, 0xdf, 0x64, 0x7a, 0x69, 0xc9, 0xe7, 0x86, 0xac, 0x17, 0x2b, 0xc8, 0x55, 0xaf, 0x83, 0xd2, 0x9c, 0x60, 0x52, 0x9a, 0x33, 0x6a, 0x7e, 0xfe, 0xe9, 0xa1, 0xf8, 0x36, 0xcd, 0x98,
0x35, 0x6c, 0x9d, 0x76, 0x46, 0xbd, 0xe8, 0xfa, 0x24, 0xd1, 0x73, 0x1a, 0x43, 0xf6, 0xc2, 0x01, 0xce, 0x19, 0x32, 0xb9, 0xb4, 0xe4, 0xfd, 0x1b, 0x93, 0xc9, 0xa5, 0x21, 0x9f, 0x6b, 0xb2, 0x9a,
0xc6, 0xed, 0xcd, 0xf7, 0x60, 0x6f, 0x72, 0x85, 0x0f, 0xef, 0xe0, 0x6e, 0xd5, 0x8c, 0x73, 0x39, 0x2d, 0x20, 0x97, 0xbd, 0xc6, 0xb0, 0x71, 0xda, 0x89, 0x7a, 0xc1, 0xf5, 0xbe, 0x82, 0xe7, 0x24,
0xfa, 0xd0, 0xc2, 0x37, 0x9e, 0x2e, 0x59, 0xca, 0xbd, 0x1c, 0x1f, 0xed, 0x06, 0xeb, 0x05, 0xbb, 0x81, 0xe5, 0x0b, 0x0b, 0x18, 0x35, 0xd7, 0x3f, 0xbc, 0xbd, 0xf8, 0x0a, 0xef, 0xdf, 0xc1, 0xdd,
0xb3, 0x1b, 0x2e, 0xe1, 0x0f, 0xff, 0x0e, 0x28, 0x22, 0x0c, 0xde, 0x7f, 0xfb, 0xfd, 0x79, 0xff, 0xdd, 0x61, 0xec, 0x94, 0xd1, 0xc7, 0x06, 0xbe, 0xf5, 0x24, 0xa5, 0x19, 0x73, 0x72, 0x7c, 0xb8,
0x5e, 0x78, 0x97, 0xbc, 0x19, 0x11, 0x6a, 0x54, 0x88, 0x56, 0xcb, 0x98, 0xa8, 0x52, 0xe3, 0x23, 0x6d, 0xac, 0xe3, 0x6d, 0xd7, 0xae, 0x59, 0x93, 0x3b, 0xfc, 0x37, 0xa0, 0xb0, 0xd0, 0xfb, 0xf0,
0x72, 0x47, 0xae, 0xdd, 0xc0, 0x7b, 0x54, 0x1f, 0xdd, 0x7c, 0x42, 0xff, 0xe4, 0x9f, 0xb8, 0x62, 0xfd, 0xcf, 0x97, 0xfd, 0x7b, 0xfe, 0xdd, 0xf0, 0x6d, 0x14, 0x12, 0xad, 0x12, 0x2a, 0x99, 0x26,
0x93, 0x13, 0xbb, 0xc9, 0xc3, 0x30, 0xa8, 0x6d, 0xb2, 0x30, 0xf8, 0xb9, 0xbe, 0x56, 0x7e, 0x87, 0xa1, 0x2c, 0x35, 0x3e, 0x21, 0x7b, 0x01, 0x95, 0x1d, 0x38, 0x0f, 0xab, 0xa5, 0xeb, 0x57, 0xe8,
0x8f, 0x5c, 0x42, 0x2e, 0xad, 0x6a, 0x0a, 0x0d, 0xcf, 0xa1, 0x9a, 0x42, 0x53, 0xc4, 0x57, 0xda, 0x9e, 0xfc, 0x17, 0x57, 0x74, 0x72, 0x62, 0x3a, 0x79, 0xe0, 0x7b, 0x95, 0x4e, 0x66, 0x1a, 0x3f,
0xfd, 0x9a, 0xf6, 0xd2, 0xc2, 0xe6, 0xca, 0x72, 0xce, 0xd1, 0xd9, 0xb8, 0xb7, 0xf9, 0x35, 0xd8, 0x55, 0xd7, 0xca, 0xef, 0xf1, 0xa1, 0x75, 0xc8, 0xba, 0xb5, 0xeb, 0x42, 0xcd, 0x39, 0xec, 0xba,
0xdb, 0x6c, 0x07, 0xe8, 0xeb, 0x76, 0x80, 0x7e, 0x6e, 0x07, 0xe8, 0xe5, 0x81, 0x99, 0x2d, 0xe3, 0x50, 0x67, 0xf1, 0x95, 0x76, 0xbf, 0xa2, 0x9d, 0x1a, 0xd8, 0x54, 0x1a, 0xce, 0x39, 0x3a, 0x1b,
0xf8, 0xc0, 0x3e, 0x8e, 0xc7, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x6d, 0x79, 0xf1, 0x8d, 0xf5, 0xd6, 0xbf, 0x07, 0x7b, 0xeb, 0xcd, 0x00, 0x7d, 0xdb, 0x0c, 0xd0, 0xaf, 0xcd, 0x00, 0xbd,
0x03, 0x00, 0x00, 0x3c, 0xd0, 0xb5, 0x45, 0x92, 0x1c, 0x98, 0xe3, 0x78, 0xf4, 0x37, 0x00, 0x00, 0xff, 0xff, 0x8b,
0x21, 0x42, 0x5d, 0xaa, 0x03, 0x00, 0x00,
} }

View file

@ -28,10 +28,18 @@ var _ status.Status
var _ = runtime.String var _ = runtime.String
var _ = utilities.NewDoubleArray var _ = utilities.NewDoubleArray
var (
filter_Admin_TSDBSnapshot_0 = &utilities.DoubleArray{Encoding: map[string]int{}, Base: []int(nil), Check: []int(nil)}
)
func request_Admin_TSDBSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, client AdminClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) { func request_Admin_TSDBSnapshot_0(ctx context.Context, marshaler runtime.Marshaler, client AdminClient, req *http.Request, pathParams map[string]string) (proto.Message, runtime.ServerMetadata, error) {
var protoReq TSDBSnapshotRequest var protoReq TSDBSnapshotRequest
var metadata runtime.ServerMetadata var metadata runtime.ServerMetadata
if err := runtime.PopulateQueryParameters(&protoReq, req.URL.Query(), filter_Admin_TSDBSnapshot_0); err != nil {
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}
msg, err := client.TSDBSnapshot(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) msg, err := client.TSDBSnapshot(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
return msg, metadata, err return msg, metadata, err
@ -50,10 +58,8 @@ func request_Admin_DeleteSeries_0(ctx context.Context, marshaler runtime.Marshal
var protoReq SeriesDeleteRequest var protoReq SeriesDeleteRequest
var metadata runtime.ServerMetadata var metadata runtime.ServerMetadata
if req.ContentLength > 0 { if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil {
if err := marshaler.NewDecoder(req.Body).Decode(&protoReq); err != nil { return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
return nil, metadata, status.Errorf(codes.InvalidArgument, "%v", err)
}
} }
msg, err := client.DeleteSeries(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD)) msg, err := client.DeleteSeries(ctx, &protoReq, grpc.Header(&metadata.HeaderMD), grpc.Trailer(&metadata.TrailerMD))
@ -100,7 +106,7 @@ func RegisterAdminHandler(ctx context.Context, mux *runtime.ServeMux, conn *grpc
func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, client AdminClient) error { func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, client AdminClient) error {
mux.Handle("POST", pattern_Admin_TSDBSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { mux.Handle("POST", pattern_Admin_TSDBSnapshot_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context()) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
if cn, ok := w.(http.CloseNotifier); ok { if cn, ok := w.(http.CloseNotifier); ok {
go func(done <-chan struct{}, closed <-chan bool) { go func(done <-chan struct{}, closed <-chan bool) {
@ -129,7 +135,7 @@ func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, clie
}) })
mux.Handle("POST", pattern_Admin_TSDBCleanTombstones_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { mux.Handle("POST", pattern_Admin_TSDBCleanTombstones_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context()) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
if cn, ok := w.(http.CloseNotifier); ok { if cn, ok := w.(http.CloseNotifier); ok {
go func(done <-chan struct{}, closed <-chan bool) { go func(done <-chan struct{}, closed <-chan bool) {
@ -158,7 +164,7 @@ func RegisterAdminHandlerClient(ctx context.Context, mux *runtime.ServeMux, clie
}) })
mux.Handle("POST", pattern_Admin_DeleteSeries_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) { mux.Handle("POST", pattern_Admin_DeleteSeries_0, func(w http.ResponseWriter, req *http.Request, pathParams map[string]string) {
ctx, cancel := context.WithCancel(req.Context()) ctx, cancel := context.WithCancel(ctx)
defer cancel() defer cancel()
if cn, ok := w.(http.CloseNotifier); ok { if cn, ok := w.(http.CloseNotifier); ok {
go func(done <-chan struct{}, closed <-chan bool) { go func(done <-chan struct{}, closed <-chan bool) {

View file

@ -53,6 +53,7 @@ service Admin {
message TSDBSnapshotRequest { message TSDBSnapshotRequest {
bool skip_head = 1;
} }
message TSDBSnapshotResponse { message TSDBSnapshotResponse {

View file

@ -657,6 +657,8 @@ func (api *API) snapshot(r *http.Request) (interface{}, *apiError) {
if !api.enableAdmin { if !api.enableAdmin {
return nil, &apiError{errorUnavailable, errors.New("Admin APIs disabled")} return nil, &apiError{errorUnavailable, errors.New("Admin APIs disabled")}
} }
skipHead, _ := strconv.ParseBool(r.FormValue("skip_head"))
db := api.db() db := api.db()
if db == nil { if db == nil {
return nil, &apiError{errorUnavailable, errors.New("TSDB not ready")} return nil, &apiError{errorUnavailable, errors.New("TSDB not ready")}
@ -672,7 +674,7 @@ func (api *API) snapshot(r *http.Request) (interface{}, *apiError) {
if err := os.MkdirAll(dir, 0777); err != nil { if err := os.MkdirAll(dir, 0777); err != nil {
return nil, &apiError{errorInternal, fmt.Errorf("create snapshot directory: %s", err)} return nil, &apiError{errorInternal, fmt.Errorf("create snapshot directory: %s", err)}
} }
if err := db.Snapshot(dir); err != nil { if err := db.Snapshot(dir, !skipHead); err != nil {
return nil, &apiError{errorInternal, fmt.Errorf("create snapshot: %s", err)} return nil, &apiError{errorInternal, fmt.Errorf("create snapshot: %s", err)}
} }

View file

@ -167,7 +167,7 @@ func NewAdmin(db func() *tsdb.DB) *Admin {
} }
// TSDBSnapshot implements pb.AdminServer. // TSDBSnapshot implements pb.AdminServer.
func (s *Admin) TSDBSnapshot(_ old_ctx.Context, _ *pb.TSDBSnapshotRequest) (*pb.TSDBSnapshotResponse, error) { func (s *Admin) TSDBSnapshot(_ old_ctx.Context, req *pb.TSDBSnapshotRequest) (*pb.TSDBSnapshotResponse, error) {
db := s.db() db := s.db()
if db == nil { if db == nil {
return nil, status.Errorf(codes.Unavailable, "TSDB not ready") return nil, status.Errorf(codes.Unavailable, "TSDB not ready")
@ -182,7 +182,7 @@ func (s *Admin) TSDBSnapshot(_ old_ctx.Context, _ *pb.TSDBSnapshotRequest) (*pb.
if err := os.MkdirAll(dir, 0777); err != nil { if err := os.MkdirAll(dir, 0777); err != nil {
return nil, status.Errorf(codes.Internal, "created snapshot directory: %s", err) return nil, status.Errorf(codes.Internal, "created snapshot directory: %s", err)
} }
if err := db.Snapshot(dir); err != nil { if err := db.Snapshot(dir, !req.SkipHead); err != nil {
return nil, status.Errorf(codes.Internal, "create snapshot: %s", err) return nil, status.Errorf(codes.Internal, "create snapshot: %s", err)
} }
return &pb.TSDBSnapshotResponse{Name: name}, nil return &pb.TSDBSnapshotResponse{Name: name}, nil