WIP state-machine based replication
This commit is contained in:
Christian Schwarz
@@ -0,0 +1,474 @@
|
||||
package replication
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
//go:generate stringer -type=ReplicationState
|
||||
type ReplicationState int
|
||||
|
||||
const (
|
||||
Planning ReplicationState = iota
|
||||
PlanningError
|
||||
Working
|
||||
WorkingWait
|
||||
Completed
|
||||
ContextDone
|
||||
)
|
||||
|
||||
type Replication struct {
|
||||
state ReplicationState
|
||||
|
||||
// Working / WorkingWait
|
||||
|
||||
pending, completed []*FSReplication
|
||||
|
||||
// PlanningError
|
||||
planningError error
|
||||
|
||||
// ContextDone
|
||||
contextError error
|
||||
}
|
||||
|
||||
type FSReplicationState int
|
||||
|
||||
//go:generate stringer -type=FSReplicationState
|
||||
const (
|
||||
FSQueued FSReplicationState = 1 << iota
|
||||
FSActive
|
||||
FSRetry
|
||||
FSPermanentError
|
||||
FSCompleted
|
||||
)
|
||||
|
||||
type FSReplication struct {
|
||||
state FSReplicationState
|
||||
fs *Filesystem
|
||||
permanentError error
|
||||
retryAt time.Time
|
||||
completed, pending []*FSReplicationStep
|
||||
}
|
||||
|
||||
func newFSReplicationPermanentError(fs *Filesystem, err error) *FSReplication {
|
||||
return &FSReplication{
|
||||
state: FSPermanentError,
|
||||
fs: fs,
|
||||
permanentError: err,
|
||||
}
|
||||
}
|
||||
|
||||
type FSReplicationBuilder struct {
|
||||
r *FSReplication
|
||||
steps []*FSReplicationStep
|
||||
}
|
||||
|
||||
func buildNewFSReplication(fs *Filesystem) *FSReplicationBuilder {
|
||||
return &FSReplicationBuilder{
|
||||
r: &FSReplication{
|
||||
fs: fs,
|
||||
pending: make([]*FSReplicationStep, 0),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (b *FSReplicationBuilder) AddStep(from, to *FilesystemVersion) *FSReplication {
|
||||
step := &FSReplicationStep{
|
||||
state: StepPending,
|
||||
fsrep: b.r,
|
||||
from: from,
|
||||
to: to,
|
||||
}
|
||||
b.r.pending = append(b.r.pending, step)
|
||||
return b.r
|
||||
}
|
||||
|
||||
func (b *FSReplicationBuilder) Complete() *FSReplication {
|
||||
if len(b.r.pending) > 0 {
|
||||
b.r.state = FSQueued
|
||||
} else {
|
||||
b.r.state = FSCompleted
|
||||
}
|
||||
r := b.r
|
||||
return r
|
||||
}
|
||||
|
||||
//go:generate stringer -type=FSReplicationStepState
|
||||
type FSReplicationStepState int
|
||||
|
||||
const (
|
||||
StepPending FSReplicationStepState = iota
|
||||
StepActive
|
||||
StepRetry
|
||||
StepPermanentError
|
||||
StepCompleted
|
||||
)
|
||||
|
||||
type FSReplicationStep struct {
|
||||
state FSReplicationStepState
|
||||
from, to *FilesystemVersion
|
||||
fsrep *FSReplication
|
||||
|
||||
// both retry and permanent error
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *Replication) Drive(ctx context.Context, ep EndpointPair, retryNow chan struct{}) {
|
||||
for !(r.state == Completed || r.state == ContextDone) {
|
||||
pre := r.state
|
||||
preTime := time.Now()
|
||||
r.doDrive(ctx, ep, retryNow)
|
||||
delta := time.Now().Sub(preTime)
|
||||
post := r.state
|
||||
getLogger(ctx).
|
||||
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
|
||||
WithField("duration", delta).
|
||||
Debug("state transition")
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Replication) doDrive(ctx context.Context, ep EndpointPair, retryNow chan struct{}) {
|
||||
|
||||
switch r.state {
|
||||
|
||||
case Planning:
|
||||
r.tryBuildPlan(ctx, ep)
|
||||
|
||||
case PlanningError:
|
||||
w := time.NewTimer(10 * time.Second) // FIXME constant make configurable
|
||||
defer w.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
r.state = ContextDone
|
||||
r.contextError = ctx.Err()
|
||||
case <-retryNow:
|
||||
r.state = Planning
|
||||
r.planningError = nil
|
||||
case <-w.C:
|
||||
r.state = Planning
|
||||
r.planningError = nil
|
||||
}
|
||||
|
||||
case Working:
|
||||
|
||||
if len(r.pending) == 0 {
|
||||
r.state = Completed
|
||||
return
|
||||
}
|
||||
|
||||
sort.Slice(r.pending, func(i, j int) bool {
|
||||
a, b := r.pending[i], r.pending[j]
|
||||
statePrio := func(x *FSReplication) int {
|
||||
if !(x.state == FSQueued || x.state == FSRetry) {
|
||||
panic(x)
|
||||
}
|
||||
if x.state == FSQueued {
|
||||
return 0
|
||||
} else {
|
||||
return 1
|
||||
}
|
||||
}
|
||||
aprio, bprio := statePrio(a), statePrio(b)
|
||||
if aprio != bprio {
|
||||
return aprio < bprio
|
||||
}
|
||||
// now we know they are the same state
|
||||
if a.state == FSQueued {
|
||||
return a.nextStepDate().Before(b.nextStepDate())
|
||||
}
|
||||
if a.state == FSRetry {
|
||||
return a.retryAt.Before(b.retryAt)
|
||||
}
|
||||
panic("should not be reached")
|
||||
})
|
||||
|
||||
fsrep := r.pending[0]
|
||||
|
||||
if fsrep.state == FSRetry {
|
||||
r.state = WorkingWait
|
||||
return
|
||||
}
|
||||
if fsrep.state != FSQueued {
|
||||
panic(fsrep)
|
||||
}
|
||||
|
||||
fsState := fsrep.takeStep(ctx, ep)
|
||||
if fsState&(FSPermanentError|FSCompleted) != 0 {
|
||||
r.pending = r.pending[1:]
|
||||
r.completed = append(r.completed, fsrep)
|
||||
}
|
||||
|
||||
case WorkingWait:
|
||||
fsrep := r.pending[0]
|
||||
w := time.NewTimer(fsrep.retryAt.Sub(time.Now()))
|
||||
defer w.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
r.state = ContextDone
|
||||
r.contextError = ctx.Err()
|
||||
case <-retryNow:
|
||||
for _, fsr := range r.pending {
|
||||
fsr.retryNow()
|
||||
}
|
||||
r.state = Working
|
||||
case <-w.C:
|
||||
fsrep.retryNow() // avoid timer jitter
|
||||
r.state = Working
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Replication) tryBuildPlan(ctx context.Context, ep EndpointPair) ReplicationState {
|
||||
|
||||
log := getLogger(ctx)
|
||||
|
||||
planningError := func(err error) ReplicationState {
|
||||
r.state = PlanningError
|
||||
r.planningError = err
|
||||
return r.state
|
||||
}
|
||||
done := func() ReplicationState {
|
||||
r.state = Working
|
||||
r.planningError = nil
|
||||
return r.state
|
||||
}
|
||||
|
||||
sfss, err := ep.Sender().ListFilesystems(ctx)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error listing sender filesystems")
|
||||
return planningError(err)
|
||||
}
|
||||
|
||||
rfss, err := ep.Receiver().ListFilesystems(ctx)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error listing receiver filesystems")
|
||||
return planningError(err)
|
||||
}
|
||||
|
||||
r.pending = make([]*FSReplication, 0, len(sfss))
|
||||
r.completed = make([]*FSReplication, 0, len(sfss))
|
||||
mainlog := log
|
||||
for _, fs := range sfss {
|
||||
|
||||
log := mainlog.WithField("filesystem", fs.Path)
|
||||
|
||||
log.Info("assessing filesystem")
|
||||
|
||||
sfsvs, err := ep.Sender().ListFilesystemVersions(ctx, fs.Path)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot get remote filesystem versions")
|
||||
return planningError(err)
|
||||
}
|
||||
|
||||
if len(sfsvs) <= 1 {
|
||||
err := errors.New("sender does not have any versions")
|
||||
log.Error(err.Error())
|
||||
r.completed = append(r.completed, newFSReplicationPermanentError(fs, err))
|
||||
continue
|
||||
}
|
||||
|
||||
receiverFSExists := false
|
||||
for _, rfs := range rfss {
|
||||
if rfs.Path == fs.Path {
|
||||
receiverFSExists = true
|
||||
}
|
||||
}
|
||||
|
||||
var rfsvs []*FilesystemVersion
|
||||
if receiverFSExists {
|
||||
rfsvs, err = ep.Receiver().ListFilesystemVersions(ctx, fs.Path)
|
||||
if err != nil {
|
||||
if _, ok := err.(FilteredError); ok {
|
||||
log.Info("receiver ignores filesystem")
|
||||
continue
|
||||
}
|
||||
log.WithError(err).Error("receiver error")
|
||||
return planningError(err)
|
||||
}
|
||||
} else {
|
||||
rfsvs = []*FilesystemVersion{}
|
||||
}
|
||||
|
||||
path, conflict := IncrementalPath(rfsvs, sfsvs)
|
||||
if conflict != nil {
|
||||
var msg string
|
||||
path, msg = resolveConflict(conflict) // no shadowing allowed!
|
||||
if path != nil {
|
||||
log.WithField("conflict", conflict).Info("conflict")
|
||||
log.WithField("resolution", msg).Info("automatically resolved")
|
||||
} else {
|
||||
log.WithField("conflict", conflict).Error("conflict")
|
||||
log.WithField("problem", msg).Error("cannot resolve conflict")
|
||||
}
|
||||
}
|
||||
if path == nil {
|
||||
r.completed = append(r.completed, newFSReplicationPermanentError(fs, conflict))
|
||||
continue
|
||||
}
|
||||
|
||||
fsreplbuilder := buildNewFSReplication(fs)
|
||||
if len(path) == 1 {
|
||||
fsreplbuilder.AddStep(nil, path[0])
|
||||
} else {
|
||||
for i := 0; i < len(path)-1; i++ {
|
||||
fsreplbuilder.AddStep(path[i], path[i+1])
|
||||
}
|
||||
}
|
||||
fsrepl := fsreplbuilder.Complete()
|
||||
switch fsrepl.state {
|
||||
case FSCompleted:
|
||||
r.completed = append(r.completed, fsreplbuilder.Complete())
|
||||
case FSQueued:
|
||||
r.pending = append(r.pending, fsreplbuilder.Complete())
|
||||
default:
|
||||
panic(fsrepl)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return done()
|
||||
}
|
||||
|
||||
func (f *FSReplication) nextStepDate() time.Time {
|
||||
if f.state != FSQueued {
|
||||
panic(f)
|
||||
}
|
||||
ct, err := f.pending[0].to.CreationAsTime()
|
||||
if err != nil {
|
||||
panic(err) // FIXME
|
||||
}
|
||||
return ct
|
||||
}
|
||||
|
||||
func (f *FSReplication) takeStep(ctx context.Context, ep EndpointPair) FSReplicationState {
|
||||
if f.state != FSQueued {
|
||||
panic(f)
|
||||
}
|
||||
|
||||
f.state = FSActive
|
||||
step := f.pending[0]
|
||||
stepState := step.do(ctx, ep)
|
||||
|
||||
switch stepState {
|
||||
case StepCompleted:
|
||||
f.pending = f.pending[1:]
|
||||
f.completed = append(f.completed, step)
|
||||
if len(f.pending) > 0 {
|
||||
f.state = FSQueued
|
||||
} else {
|
||||
f.state = FSCompleted
|
||||
}
|
||||
|
||||
case StepRetry:
|
||||
f.state = FSRetry
|
||||
f.retryAt = time.Now().Add(10 * time.Second) // FIXME hardcoded constant
|
||||
|
||||
case StepPermanentError:
|
||||
f.state = FSPermanentError
|
||||
|
||||
}
|
||||
return f.state
|
||||
}
|
||||
|
||||
func (f *FSReplication) retryNow() {
|
||||
if f.state != FSRetry {
|
||||
panic(f)
|
||||
}
|
||||
f.retryAt = time.Time{}
|
||||
f.state = FSQueued
|
||||
}
|
||||
|
||||
func (s *FSReplicationStep) do(ctx context.Context, ep EndpointPair) FSReplicationStepState {
|
||||
|
||||
fs := s.fsrep.fs
|
||||
|
||||
log := getLogger(ctx).
|
||||
WithField("filesystem", fs.Path).
|
||||
WithField("step", s.String())
|
||||
|
||||
updateStateError := func(err error) FSReplicationStepState {
|
||||
s.err = err
|
||||
switch err {
|
||||
case io.EOF: fallthrough
|
||||
case io.ErrUnexpectedEOF: fallthrough
|
||||
case io.ErrClosedPipe:
|
||||
return StepRetry
|
||||
}
|
||||
if _, ok := err.(net.Error); ok {
|
||||
return StepRetry
|
||||
}
|
||||
return StepPermanentError
|
||||
}
|
||||
|
||||
updateStateCompleted := func() FSReplicationStepState {
|
||||
s.err = nil
|
||||
s.state = StepCompleted
|
||||
return s.state
|
||||
}
|
||||
|
||||
// FIXME refresh fs resume token
|
||||
fs.ResumeToken = ""
|
||||
|
||||
var sr *SendReq
|
||||
if fs.ResumeToken != "" {
|
||||
sr = &SendReq{
|
||||
Filesystem: fs.Path,
|
||||
ResumeToken: fs.ResumeToken,
|
||||
}
|
||||
} else if s.from == nil {
|
||||
sr = &SendReq{
|
||||
Filesystem: fs.Path,
|
||||
From: s.to.RelName(), // FIXME fix protocol to use To, like zfs does internally
|
||||
}
|
||||
} else {
|
||||
sr = &SendReq{
|
||||
Filesystem: fs.Path,
|
||||
From: s.from.RelName(),
|
||||
To: s.to.RelName(),
|
||||
}
|
||||
}
|
||||
|
||||
log.WithField("request", sr).Debug("initiate send request")
|
||||
sres, sstream, err := ep.Sender().Send(ctx, sr)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("send request failed")
|
||||
return updateStateError(err)
|
||||
}
|
||||
if sstream == nil {
|
||||
err := errors.New("send request did not return a stream, broken endpoint implementation")
|
||||
return updateStateError(err)
|
||||
}
|
||||
|
||||
rr := &ReceiveReq{
|
||||
Filesystem: fs.Path,
|
||||
ClearResumeToken: !sres.UsedResumeToken,
|
||||
}
|
||||
log.WithField("request", rr).Debug("initiate receive request")
|
||||
err = ep.Receiver().Receive(ctx, rr, sstream)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("receive request failed (might also be error on sender)")
|
||||
sstream.Close()
|
||||
// This failure could be due to
|
||||
// - an unexpected exit of ZFS on the sending side
|
||||
// - an unexpected exit of ZFS on the receiving side
|
||||
// - a connectivity issue
|
||||
return updateStateError(err)
|
||||
}
|
||||
log.Info("receive finished")
|
||||
return updateStateCompleted()
|
||||
|
||||
}
|
||||
|
||||
func (s *FSReplicationStep) String() string {
|
||||
if s.from == nil { // FIXME: ZFS semantics are that to is nil on non-incremental send
|
||||
return fmt.Sprintf("%s%s (full)", s.fsrep.fs.Path, s.to.RelName())
|
||||
} else {
|
||||
return fmt.Sprintf("%s(%s => %s)", s.fsrep.fs.Path, s.from.RelName(), s.to.RelName())
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user