mudhorn/prometheus
1
0
Fork 0
mirror of https://github.com/prometheus/prometheus.git synced 2024-12-26 22:19:40 -08:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Add workaround for deadlocks

Browse source 
This adds a workaround to avoid deadlocks for inconsistent write lock
order across headBlocks.
Things keep working if transactions only append data for the same
timestamp, which is generally the case for Prometheus.

Full behavior should be restored in a subsequent change.
This commit is contained in:
Fabian Reinartz 2017-03-27 19:05:34 +02:00
parent 3be4ef94ce
commit a52980e0a8
2 changed files with 34 additions and 13 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

43
db.go
View file

@ -462,18 +462,22 @@ func (db *DB) Appender() Appender {
db.mtx.RLock() db.mtx.RLock()
a := &dbAppender{db: db} a := &dbAppender{db: db}
// Only instantiate appender after returning the headmtx to avoid // XXX(fabxc): turn off creating initial appender as it will happen on-demand
// questionable locking order. // anyway. For now this, with combination of only having a single timestamp per batch,
db.headmtx.RLock() // prevents opening more than one appender and hitting an unresolved deadlock (#11).
app := db.appendable() //
db.headmtx.RUnlock() // // Only instantiate appender after returning the headmtx to avoid
// // questionable locking order.
// db.headmtx.RLock()
// app := db.appendable()
// db.headmtx.RUnlock()
for _, b := range app { // for _, b := range app {
a.heads = append(a.heads, &metaAppender{ // a.heads = append(a.heads, &metaAppender{
meta: b.Meta(), // meta: b.Meta(),
app: b.Appender().(*headAppender), // app: b.Appender().(*headAppender),
}) // })
} // }
return a return a
} }
@ -551,14 +555,27 @@ func (a *dbAppender) appenderFor(t int64) (*metaAppender, error) {
a.db.headmtx.Unlock() a.db.headmtx.Unlock()
// Instantiate appenders after returning headmtx to avoid questionable // XXX(fabxc): temporary workaround. See comment on instantiating DB.Appender.
// locking order.
for _, b := range newHeads { for _, b := range newHeads {
// Only get appender for the block with the specific timestamp.
if t >= b.Meta().MaxTime {
continue
}
a.heads = append(a.heads, &metaAppender{ a.heads = append(a.heads, &metaAppender{
app: b.Appender(), app: b.Appender(),
meta: b.Meta(), meta: b.Meta(),
}) })
break
} }
// // Instantiate appenders after returning headmtx to avoid questionable
// // locking order.
// for _, b := range newHeads {
// a.heads = append(a.heads, &metaAppender{
// app: b.Appender(),
// meta: b.Meta(),
// })
// }
} }
for i := len(a.heads) - 1; i >= 0; i-- { for i := len(a.heads) - 1; i >= 0; i-- {
if h := a.heads[i]; t >= h.meta.MinTime { if h := a.heads[i]; t >= h.meta.MinTime {

4
head.go
View file

@ -34,6 +34,10 @@ var (
ErrOutOfBounds = errors.New("out of bounds") ErrOutOfBounds = errors.New("out of bounds")
) )
// func init() {
// deadlock.Opts.OnPotentialDeadlock = func() { fmt.Println("found deadlock") }
// }
// headBlock handles reads and writes of time series data within a time window. // headBlock handles reads and writes of time series data within a time window.
type headBlock struct { type headBlock struct {
mtx sync.RWMutex mtx sync.RWMutex