adminer/zrepl_patched
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

daemon/active: implement watchdog to handle stuck replication / pruners

Browse Source 
ActiveSide.do() can only run sequentially, i.e. we cannot run
replication and pruning in parallel. Why?

* go-streamrpc only allows one active request at a time
(this is bad design and should be fixed at some point)
* replication and pruning are implemented independently, but work on the
same resources (snapshots)

A: pruning might destroy a snapshot that is planned to be replicated
B: replication might replicate snapshots that should be pruned

We do not have any resource management / locking for A and B, but we
have a use case where users don't want their machine fill up with
snapshots if replication does not work.
That means we _have_ to run the pruners.

A further complication is that we cannot just cancel the replication
context after a timeout and move on to the pruner: it could be initial
replication and we don't know how long it will take.
(And we don't have resumable send & recv yet).

With the previous commits, we can implement the watchdog using context
cancellation.
Note that the 'MadeProgress()' calls can only be placed right before
non-error state transition. Otherwise, we could end up in a live-lock.
This commit is contained in:
Christian Schwarz
2018-10-19 16:27:05 +02:00
parent 4ede99b08c
commit 69bfcb7bed
8 changed files with 240 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
replication/fsrep/fsfsm.go
+15 -9
View File
@@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/util/watchdog"
"io"
"net"
"sync"
@@ -191,7 +192,7 @@ type ReplicationStep struct {
expectedSize int64 // 0 means no size estimate present / possible
}
func (f *Replication) TakeStep(ctx context.Context, sender Sender, receiver Receiver) (post State, nextStepDate, retryWaitUntil time.Time) {
func (f *Replication) TakeStep(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver) (post State, nextStepDate, retryWaitUntil time.Time) {
var u updater = func(fu func(*Replication)) State {
f.lock.Lock()
@@ -205,7 +206,7 @@ func (f *Replication) TakeStep(ctx context.Context, sender Sender, receiver Rece
pre := u(nil)
preTime := time.Now()
s = s(ctx, sender, receiver, u)
s = s(ctx, ka, sender, receiver, u)
delta := time.Now().Sub(preTime)
post = u(func(f *Replication) {
@@ -233,11 +234,11 @@ func (f *Replication) RetryWaitUntil() time.Time {
type updater func(func(fsr *Replication)) State
type state func(ctx context.Context, sender Sender, receiver Receiver, u updater) state
type state func(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state
var RetrySleepDuration = 10 * time.Second // FIXME make configurable
func stateReady(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
func stateReady(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state {
var current *ReplicationStep
s := u(func(f *Replication) {
@@ -251,7 +252,7 @@ func stateReady(ctx context.Context, sender Sender, receiver Receiver, u updater
return s.fsrsf()
}
stepState := current.doReplication(ctx, sender, receiver)
stepState := current.doReplication(ctx, ka, sender, receiver)
return u(func(f *Replication) {
switch stepState {
@@ -277,7 +278,7 @@ func stateReady(ctx context.Context, sender Sender, receiver Receiver, u updater
}).fsrsf()
}
func stateRetryWait(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
func stateRetryWait(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state {
var sleepUntil time.Time
u(func(f *Replication) {
sleepUntil = f.retryWaitUntil
@@ -337,7 +338,7 @@ func shouldRetry(err error) bool {
return false
}
func (s *ReplicationStep) doReplication(ctx context.Context, sender Sender, receiver Receiver) StepState {
func (s *ReplicationStep) doReplication(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver) StepState {
fs := s.parent.fs
@@ -380,6 +381,9 @@ func (s *ReplicationStep) doReplication(ctx context.Context, sender Sender, rece
}
s.byteCounter = util.NewByteCounterReader(sstream)
s.byteCounter.SetCallback(1*time.Second, func(i int64) {
ka.MadeProgress()
})
defer func() {
s.parent.promBytesReplicated.Add(float64(s.byteCounter.Bytes()))
}()
@@ -404,14 +408,15 @@ func (s *ReplicationStep) doReplication(ctx context.Context, sender Sender, rece
return updateStateError(err)
}
log.Debug("receive finished")
ka.MadeProgress()
updateStateCompleted()
return s.doMarkReplicated(ctx, sender)
return s.doMarkReplicated(ctx, ka, sender)
}
func (s *ReplicationStep) doMarkReplicated(ctx context.Context, sender Sender) StepState {
func (s *ReplicationStep) doMarkReplicated(ctx context.Context, ka *watchdog.KeepAlive, sender Sender) StepState {
log := getLogger(ctx).
WithField("filesystem", s.parent.fs).
@@ -456,6 +461,7 @@ func (s *ReplicationStep) doMarkReplicated(ctx context.Context, sender Sender) S
log.Error(err.Error())
return updateStateError(err)
}
ka.MadeProgress()
return updateStateCompleted()
}
replication/mainfsm.go
+23 -7
View File
@@ -8,6 +8,7 @@ import (
"fmt"
"github.com/prometheus/client_golang/prometheus"
"github.com/zrepl/zrepl/daemon/job/wakeup"
"github.com/zrepl/zrepl/util/watchdog"
"math/bits"
"net"
"sync"
@@ -47,6 +48,10 @@ func (s State) rsf() state {
return m[idx]
}
func (s State) IsTerminal() bool {
return s.rsf() == nil
}
// Replication implements the replication of multiple file systems from a Sender to a Receiver.
//
// It is a state machine that is driven by the Drive method
@@ -56,6 +61,8 @@ type Replication struct {
promSecsPerState *prometheus.HistogramVec // labels: state
promBytesReplicated *prometheus.CounterVec // labels: filesystem
Progress watchdog.KeepAlive
// lock protects all fields of this struct (but not the fields behind pointers!)
lock sync.Mutex
@@ -122,7 +129,7 @@ func NewFilteredError(fs string) *FilteredError {
func (f FilteredError) Error() string { return "endpoint does not allow access to filesystem " + f.fs }
type updater func(func(*Replication)) (newState State)
type state func(ctx context.Context, sender Sender, receiver Receiver, u updater) state
type state func(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state
// Drive starts the state machine and returns only after replication has finished (with or without errors).
// The Logger in ctx is used for both debug and error logging, but is not guaranteed to be stable
@@ -147,7 +154,7 @@ func (r *Replication) Drive(ctx context.Context, sender Sender, receiver Receive
for s != nil {
preTime := time.Now()
pre = u(nil)
s = s(ctx, sender, receiver, u)
s = s(ctx, &r.Progress, sender, receiver, u)
delta := time.Now().Sub(preTime)
r.promSecsPerState.WithLabelValues(pre.String()).Observe(delta.Seconds())
post = u(nil)
@@ -198,7 +205,7 @@ func isPermanent(err error) bool {
return false
}
func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
func statePlanning(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state {
log := getLogger(ctx)
@@ -307,9 +314,12 @@ func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u upda
log.WithError(err).Error("error computing size estimate")
return handlePlanningError(err)
}
q.Add(qitem)
}
ka.MadeProgress()
return u(func(r *Replication) {
r.completed = nil
r.queue = q
@@ -318,7 +328,7 @@ func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u upda
}).rsf()
}
func statePlanningError(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
func statePlanningError(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state {
var sleepUntil time.Time
u(func(r *Replication) {
sleepUntil = r.sleepUntil
@@ -340,7 +350,7 @@ func statePlanningError(ctx context.Context, sender Sender, receiver Receiver, u
}).rsf()
}
func stateWorking(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
func stateWorking(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state {
var active *ReplicationQueueItemHandle
rsfNext := u(func(r *Replication) {
@@ -365,7 +375,7 @@ func stateWorking(ctx context.Context, sender Sender, receiver Receiver, u updat
}).rsf()
}
state, nextStepDate, retryWaitUntil := active.GetFSReplication().TakeStep(ctx, sender, receiver)
state, nextStepDate, retryWaitUntil := active.GetFSReplication().TakeStep(ctx, ka, sender, receiver)
return u(func(r *Replication) {
active.Update(state, nextStepDate, retryWaitUntil)
r.active = nil
@@ -383,7 +393,7 @@ func stateWorking(ctx context.Context, sender Sender, receiver Receiver, u updat
return u(nil).rsf()
}
func stateWorkingWait(ctx context.Context, sender Sender, receiver Receiver, u updater) state {
func stateWorkingWait(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state {
var sleepUntil time.Time
u(func(r *Replication) {
sleepUntil = r.sleepUntil
@@ -445,3 +455,9 @@ func (r *Replication) Report() *Report {
return &rep
}
func (r *Replication) State() State {
r.lock.Lock()
defer r.lock.Unlock()
return r.state
}