mirror of
https://github.com/prometheus/prometheus.git
synced 2024-12-24 05:04:05 -08:00
buffer-panic when reading a record after recPageTerm (#429)
Signed-off-by: Krasi Georgiev <kgeorgie@redhat.com>
This commit is contained in:
parent
5a9ddeecef
commit
3385571ddf
12
wal/wal.go
12
wal/wal.go
|
@ -96,8 +96,9 @@ func (e *CorruptionErr) Error() string {
|
||||||
}
|
}
|
||||||
|
|
||||||
// OpenWriteSegment opens segment k in dir. The returned segment is ready for new appends.
|
// OpenWriteSegment opens segment k in dir. The returned segment is ready for new appends.
|
||||||
func OpenWriteSegment(dir string, k int) (*Segment, error) {
|
func OpenWriteSegment(logger log.Logger, dir string, k int) (*Segment, error) {
|
||||||
f, err := os.OpenFile(SegmentName(dir, k), os.O_WRONLY|os.O_APPEND, 0666)
|
segName := SegmentName(dir, k)
|
||||||
|
f, err := os.OpenFile(segName, os.O_WRONLY|os.O_APPEND, 0666)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -112,6 +113,7 @@ func OpenWriteSegment(dir string, k int) (*Segment, error) {
|
||||||
// If it was torn mid-record, a full read (which the caller should do anyway
|
// If it was torn mid-record, a full read (which the caller should do anyway
|
||||||
// to ensure integrity) will detect it as a corruption by the end.
|
// to ensure integrity) will detect it as a corruption by the end.
|
||||||
if d := stat.Size() % pageSize; d != 0 {
|
if d := stat.Size() % pageSize; d != 0 {
|
||||||
|
level.Warn(logger).Log("msg", "last page of the wal is torn, filling it with zeros", "segment", segName)
|
||||||
if _, err := f.Write(make([]byte, pageSize-d)); err != nil {
|
if _, err := f.Write(make([]byte, pageSize-d)); err != nil {
|
||||||
f.Close()
|
f.Close()
|
||||||
return nil, errors.Wrap(err, "zero-pad torn page")
|
return nil, errors.Wrap(err, "zero-pad torn page")
|
||||||
|
@ -229,7 +231,7 @@ func NewSize(logger log.Logger, reg prometheus.Registerer, dir string, segmentSi
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
if w.segment, err = OpenWriteSegment(w.dir, j); err != nil {
|
if w.segment, err = OpenWriteSegment(w.logger, w.dir, j); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// Correctly initialize donePages.
|
// Correctly initialize donePages.
|
||||||
|
@ -752,6 +754,10 @@ func (r *Reader) next() (err error) {
|
||||||
|
|
||||||
// Gobble up zero bytes.
|
// Gobble up zero bytes.
|
||||||
if typ == recPageTerm {
|
if typ == recPageTerm {
|
||||||
|
// recPageTerm is a single byte that indicates that the rest of the page is padded.
|
||||||
|
// If it's the first byte in a page, buf is too small and we have to resize buf to fit pageSize-1 bytes.
|
||||||
|
buf = r.buf[1:]
|
||||||
|
|
||||||
// We are pedantic and check whether the zeros are actually up
|
// We are pedantic and check whether the zeros are actually up
|
||||||
// to a page boundary.
|
// to a page boundary.
|
||||||
// It's not strictly necessary but may catch sketchy state early.
|
// It's not strictly necessary but may catch sketchy state early.
|
||||||
|
|
|
@ -217,6 +217,14 @@ func TestWAL_FuzzWriteRead(t *testing.T) {
|
||||||
|
|
||||||
func TestWAL_Repair(t *testing.T) {
|
func TestWAL_Repair(t *testing.T) {
|
||||||
for name, cf := range map[string]func(f *os.File){
|
for name, cf := range map[string]func(f *os.File){
|
||||||
|
// Ensures that the page buffer is big enough to fit and entyre page size without panicing.
|
||||||
|
// https://github.com/prometheus/tsdb/pull/414
|
||||||
|
"bad_header": func(f *os.File) {
|
||||||
|
_, err := f.Seek(pageSize, 0)
|
||||||
|
testutil.Ok(t, err)
|
||||||
|
_, err = f.Write([]byte{byte(recPageTerm)})
|
||||||
|
testutil.Ok(t, err)
|
||||||
|
},
|
||||||
"bad_fragment_sequence": func(f *os.File) {
|
"bad_fragment_sequence": func(f *os.File) {
|
||||||
_, err := f.Seek(pageSize, 0)
|
_, err := f.Seek(pageSize, 0)
|
||||||
testutil.Ok(t, err)
|
testutil.Ok(t, err)
|
||||||
|
|
Loading…
Reference in a new issue