enforce encapsulation by breaking up replication package into packages
not perfect yet, public shouldn't be required to use 'common' package to use replication package
This commit is contained in:
Christian Schwarz
@@ -0,0 +1,361 @@
|
||||
package fsfsm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/bits"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/zrepl/zrepl/cmd/replication/pdu"
|
||||
. "github.com/zrepl/zrepl/cmd/replication/common"
|
||||
)
|
||||
|
||||
type StepReport struct {
|
||||
From, To string
|
||||
Status string
|
||||
Problem string
|
||||
}
|
||||
|
||||
type FilesystemReplicationReport struct {
|
||||
Filesystem string
|
||||
Status string
|
||||
Problem string
|
||||
Completed,Pending []*StepReport
|
||||
}
|
||||
|
||||
|
||||
//go:generate stringer -type=FSReplicationState
|
||||
type FSReplicationState uint
|
||||
|
||||
const (
|
||||
FSReady FSReplicationState = 1 << iota
|
||||
FSRetryWait
|
||||
FSPermanentError
|
||||
FSCompleted
|
||||
)
|
||||
|
||||
func (s FSReplicationState) fsrsf() fsrsf {
|
||||
idx := bits.TrailingZeros(uint(s))
|
||||
if idx == bits.UintSize {
|
||||
panic(s)
|
||||
}
|
||||
m := []fsrsf{
|
||||
fsrsfReady,
|
||||
fsrsfRetryWait,
|
||||
nil,
|
||||
nil,
|
||||
}
|
||||
return m[idx]
|
||||
}
|
||||
|
||||
type FSReplication struct {
|
||||
// lock protects all fields in this struct, but not the data behind pointers
|
||||
lock sync.Mutex
|
||||
state FSReplicationState
|
||||
fs string
|
||||
err error
|
||||
retryWaitUntil time.Time
|
||||
completed, pending []*FSReplicationStep
|
||||
}
|
||||
|
||||
func (f *FSReplication) State() FSReplicationState {
|
||||
f.lock.Lock()
|
||||
defer f.lock.Unlock()
|
||||
return f.state
|
||||
}
|
||||
|
||||
type FSReplicationBuilder struct {
|
||||
r *FSReplication
|
||||
}
|
||||
|
||||
func BuildFSReplication(fs string) *FSReplicationBuilder {
|
||||
return &FSReplicationBuilder{&FSReplication{fs: fs}}
|
||||
}
|
||||
|
||||
func (b *FSReplicationBuilder) AddStep(from, to FilesystemVersion) *FSReplicationBuilder {
|
||||
step := &FSReplicationStep{
|
||||
state: StepReady,
|
||||
fsrep: b.r,
|
||||
from: from,
|
||||
to: to,
|
||||
}
|
||||
b.r.pending = append(b.r.pending, step)
|
||||
return b
|
||||
}
|
||||
|
||||
func (b *FSReplicationBuilder) Done() (r *FSReplication) {
|
||||
if len(b.r.pending) > 0 {
|
||||
b.r.state = FSReady
|
||||
} else {
|
||||
b.r.state = FSCompleted
|
||||
}
|
||||
r = b.r
|
||||
b.r = nil
|
||||
return r
|
||||
}
|
||||
|
||||
func NewFSReplicationWithPermanentError(fs string, err error) *FSReplication {
|
||||
return &FSReplication{
|
||||
state: FSPermanentError,
|
||||
fs: fs,
|
||||
err: err,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//go:generate stringer -type=FSReplicationStepState
|
||||
type FSReplicationStepState uint
|
||||
|
||||
const (
|
||||
StepReady FSReplicationStepState = 1 << iota
|
||||
StepRetry
|
||||
StepPermanentError
|
||||
StepCompleted
|
||||
)
|
||||
|
||||
type FilesystemVersion interface {
|
||||
SnapshotTime() time.Time
|
||||
RelName() string
|
||||
}
|
||||
|
||||
type FSReplicationStep struct {
|
||||
// only protects state, err
|
||||
// from, to and fsrep are assumed to be immutable
|
||||
lock sync.Mutex
|
||||
|
||||
state FSReplicationStepState
|
||||
from, to FilesystemVersion
|
||||
fsrep *FSReplication
|
||||
|
||||
// both retry and permanent error
|
||||
err error
|
||||
}
|
||||
|
||||
func (f *FSReplication) TakeStep(ctx context.Context, ep EndpointPair) (post FSReplicationState, nextStepDate time.Time) {
|
||||
|
||||
var u fsrUpdater = func(fu func(*FSReplication)) FSReplicationState {
|
||||
f.lock.Lock()
|
||||
defer f.lock.Unlock()
|
||||
if fu != nil {
|
||||
fu(f)
|
||||
}
|
||||
return f.state
|
||||
}
|
||||
var s fsrsf = u(nil).fsrsf()
|
||||
|
||||
pre := u(nil)
|
||||
preTime := time.Now()
|
||||
s = s(ctx, ep, u)
|
||||
delta := time.Now().Sub(preTime)
|
||||
post = u(func(f *FSReplication) {
|
||||
if len(f.pending) == 0 {
|
||||
return
|
||||
}
|
||||
nextStepDate = f.pending[0].to.SnapshotTime()
|
||||
})
|
||||
|
||||
GetLogger(ctx).
|
||||
WithField("fs", f.fs).
|
||||
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
|
||||
WithField("duration", delta).
|
||||
Debug("fsr step taken")
|
||||
|
||||
return post, nextStepDate
|
||||
}
|
||||
|
||||
type fsrUpdater func(func(fsr *FSReplication)) FSReplicationState
|
||||
|
||||
type fsrsf func(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf
|
||||
|
||||
func fsrsfReady(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf {
|
||||
|
||||
var current *FSReplicationStep
|
||||
s := u(func(f *FSReplication) {
|
||||
if len(f.pending) == 0 {
|
||||
f.state = FSCompleted
|
||||
return
|
||||
}
|
||||
current = f.pending[0]
|
||||
})
|
||||
if s != FSReady {
|
||||
return s.fsrsf()
|
||||
}
|
||||
|
||||
stepState := current.do(ctx, ep)
|
||||
|
||||
return u(func(f *FSReplication) {
|
||||
switch stepState {
|
||||
case StepCompleted:
|
||||
f.completed = append(f.completed, current)
|
||||
f.pending = f.pending[1:]
|
||||
if len(f.pending) > 0 {
|
||||
f.state = FSReady
|
||||
} else {
|
||||
f.state = FSCompleted
|
||||
}
|
||||
case StepRetry:
|
||||
f.retryWaitUntil = time.Now().Add(10 * time.Second) // FIXME make configurable
|
||||
f.state = FSRetryWait
|
||||
case StepPermanentError:
|
||||
f.state = FSPermanentError
|
||||
f.err = errors.New("a replication step failed with a permanent error")
|
||||
default:
|
||||
panic(f)
|
||||
}
|
||||
}).fsrsf()
|
||||
}
|
||||
|
||||
func fsrsfRetryWait(ctx context.Context, ep EndpointPair, u fsrUpdater) fsrsf {
|
||||
var sleepUntil time.Time
|
||||
u(func(f *FSReplication) {
|
||||
sleepUntil = f.retryWaitUntil
|
||||
})
|
||||
t := time.NewTimer(sleepUntil.Sub(time.Now()))
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return u(func(f *FSReplication) {
|
||||
f.state = FSPermanentError
|
||||
f.err = ctx.Err()
|
||||
}).fsrsf()
|
||||
case <-t.C:
|
||||
}
|
||||
return u(func(f *FSReplication) {
|
||||
f.state = FSReady
|
||||
}).fsrsf()
|
||||
}
|
||||
|
||||
// access to fsr's members must be exclusive
|
||||
func (fsr *FSReplication) Report() *FilesystemReplicationReport {
|
||||
fsr.lock.Lock()
|
||||
defer fsr.lock.Unlock()
|
||||
|
||||
rep := FilesystemReplicationReport{
|
||||
Filesystem: fsr.fs,
|
||||
Status: fsr.state.String(),
|
||||
}
|
||||
|
||||
if fsr.state&FSPermanentError != 0 {
|
||||
rep.Problem = fsr.err.Error()
|
||||
return &rep
|
||||
}
|
||||
|
||||
rep.Completed = make([]*StepReport, len(fsr.completed))
|
||||
for i := range fsr.completed {
|
||||
rep.Completed[i] = fsr.completed[i].Report()
|
||||
}
|
||||
rep.Pending = make([]*StepReport, len(fsr.pending))
|
||||
for i := range fsr.pending {
|
||||
rep.Pending[i] = fsr.pending[i].Report()
|
||||
}
|
||||
return &rep
|
||||
}
|
||||
|
||||
func (s *FSReplicationStep) do(ctx context.Context, ep EndpointPair) FSReplicationStepState {
|
||||
|
||||
fs := s.fsrep.fs
|
||||
|
||||
log := GetLogger(ctx).
|
||||
WithField("filesystem", fs).
|
||||
WithField("step", s.String())
|
||||
|
||||
updateStateError := func(err error) FSReplicationStepState {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
|
||||
s.err = err
|
||||
switch err {
|
||||
case io.EOF:
|
||||
fallthrough
|
||||
case io.ErrUnexpectedEOF:
|
||||
fallthrough
|
||||
case io.ErrClosedPipe:
|
||||
s.state = StepRetry
|
||||
return s.state
|
||||
}
|
||||
if _, ok := err.(net.Error); ok {
|
||||
s.state = StepRetry
|
||||
return s.state
|
||||
}
|
||||
s.state = StepPermanentError
|
||||
return s.state
|
||||
}
|
||||
|
||||
updateStateCompleted := func() FSReplicationStepState {
|
||||
s.lock.Lock()
|
||||
defer s.lock.Unlock()
|
||||
s.err = nil
|
||||
s.state = StepCompleted
|
||||
return s.state
|
||||
}
|
||||
|
||||
var sr *pdu.SendReq
|
||||
if s.from == nil {
|
||||
sr = &pdu.SendReq{
|
||||
Filesystem: fs,
|
||||
From: s.to.RelName(), // FIXME fix protocol to use To, like zfs does internally
|
||||
}
|
||||
} else {
|
||||
sr = &pdu.SendReq{
|
||||
Filesystem: fs,
|
||||
From: s.from.RelName(),
|
||||
To: s.to.RelName(),
|
||||
}
|
||||
}
|
||||
|
||||
log.WithField("request", sr).Debug("initiate send request")
|
||||
sres, sstream, err := ep.Sender().Send(ctx, sr)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("send request failed")
|
||||
return updateStateError(err)
|
||||
}
|
||||
if sstream == nil {
|
||||
err := errors.New("send request did not return a stream, broken endpoint implementation")
|
||||
return updateStateError(err)
|
||||
}
|
||||
|
||||
rr := &pdu.ReceiveReq{
|
||||
Filesystem: fs,
|
||||
ClearResumeToken: !sres.UsedResumeToken,
|
||||
}
|
||||
log.WithField("request", rr).Debug("initiate receive request")
|
||||
err = ep.Receiver().Receive(ctx, rr, sstream)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("receive request failed (might also be error on sender)")
|
||||
sstream.Close()
|
||||
// This failure could be due to
|
||||
// - an unexpected exit of ZFS on the sending side
|
||||
// - an unexpected exit of ZFS on the receiving side
|
||||
// - a connectivity issue
|
||||
return updateStateError(err)
|
||||
}
|
||||
log.Info("receive finished")
|
||||
return updateStateCompleted()
|
||||
|
||||
}
|
||||
|
||||
func (s *FSReplicationStep) String() string {
|
||||
if s.from == nil { // FIXME: ZFS semantics are that to is nil on non-incremental send
|
||||
return fmt.Sprintf("%s%s (full)", s.fsrep.fs, s.to.RelName())
|
||||
} else {
|
||||
return fmt.Sprintf("%s(%s => %s)", s.fsrep.fs, s.from, s.to.RelName())
|
||||
}
|
||||
}
|
||||
|
||||
func (step *FSReplicationStep) Report() *StepReport {
|
||||
var from string // FIXME follow same convention as ZFS: to should be nil on full send
|
||||
if step.from != nil {
|
||||
from = step.from.RelName()
|
||||
}
|
||||
rep := StepReport{
|
||||
From: from,
|
||||
To: step.to.RelName(),
|
||||
Status: step.state.String(),
|
||||
}
|
||||
return &rep
|
||||
}
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
// Code generated by "stringer -type=FSReplicationState"; DO NOT EDIT.
|
||||
|
||||
package fsfsm
|
||||
|
||||
import "strconv"
|
||||
|
||||
const (
|
||||
_FSReplicationState_name_0 = "FSReadyFSRetryWait"
|
||||
_FSReplicationState_name_1 = "FSPermanentError"
|
||||
_FSReplicationState_name_2 = "FSCompleted"
|
||||
)
|
||||
|
||||
var (
|
||||
_FSReplicationState_index_0 = [...]uint8{0, 7, 18}
|
||||
)
|
||||
|
||||
func (i FSReplicationState) String() string {
|
||||
switch {
|
||||
case 1 <= i && i <= 2:
|
||||
i -= 1
|
||||
return _FSReplicationState_name_0[_FSReplicationState_index_0[i]:_FSReplicationState_index_0[i+1]]
|
||||
case i == 4:
|
||||
return _FSReplicationState_name_1
|
||||
case i == 8:
|
||||
return _FSReplicationState_name_2
|
||||
default:
|
||||
return "FSReplicationState(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
// Code generated by "stringer -type=FSReplicationStepState"; DO NOT EDIT.
|
||||
|
||||
package fsfsm
|
||||
|
||||
import "strconv"
|
||||
|
||||
const (
|
||||
_FSReplicationStepState_name_0 = "StepReadyStepRetry"
|
||||
_FSReplicationStepState_name_1 = "StepPermanentError"
|
||||
_FSReplicationStepState_name_2 = "StepCompleted"
|
||||
)
|
||||
|
||||
var (
|
||||
_FSReplicationStepState_index_0 = [...]uint8{0, 9, 18}
|
||||
)
|
||||
|
||||
func (i FSReplicationStepState) String() string {
|
||||
switch {
|
||||
case 1 <= i && i <= 2:
|
||||
i -= 1
|
||||
return _FSReplicationStepState_name_0[_FSReplicationStepState_index_0[i]:_FSReplicationStepState_index_0[i+1]]
|
||||
case i == 4:
|
||||
return _FSReplicationStepState_name_1
|
||||
case i == 8:
|
||||
return _FSReplicationStepState_name_2
|
||||
default:
|
||||
return "FSReplicationStepState(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package mainfsm
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
. "github.com/zrepl/zrepl/cmd/replication/pdu"
|
||||
)
|
||||
|
||||
type ConflictNoCommonAncestor struct {
|
||||
SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
|
||||
}
|
||||
|
||||
func (c *ConflictNoCommonAncestor) Error() string {
|
||||
return "no common snapshot or suitable bookmark between sender and receiver"
|
||||
}
|
||||
|
||||
type ConflictDiverged struct {
|
||||
SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
|
||||
CommonAncestor *FilesystemVersion
|
||||
SenderOnly, ReceiverOnly []*FilesystemVersion
|
||||
}
|
||||
|
||||
func (c *ConflictDiverged) Error() string {
|
||||
return "the receiver's latest snapshot is not present on sender"
|
||||
}
|
||||
|
||||
func SortVersionListByCreateTXGThenBookmarkLTSnapshot(fsvslice []*FilesystemVersion) []*FilesystemVersion {
|
||||
lesser := func(s []*FilesystemVersion) func(i, j int) bool {
|
||||
return func(i, j int) bool {
|
||||
if s[i].CreateTXG < s[j].CreateTXG {
|
||||
return true
|
||||
}
|
||||
if s[i].CreateTXG == s[j].CreateTXG {
|
||||
// Bookmark < Snapshot
|
||||
return s[i].Type == FilesystemVersion_Bookmark && s[j].Type == FilesystemVersion_Snapshot
|
||||
}
|
||||
return false
|
||||
}
|
||||
}
|
||||
if sort.SliceIsSorted(fsvslice, lesser(fsvslice)) {
|
||||
return fsvslice
|
||||
}
|
||||
sorted := make([]*FilesystemVersion, len(fsvslice))
|
||||
copy(sorted, fsvslice)
|
||||
sort.Slice(sorted, lesser(sorted))
|
||||
return sorted
|
||||
}
|
||||
|
||||
// conflict may be a *ConflictDiverged or a *ConflictNoCommonAncestor
|
||||
func IncrementalPath(receiver, sender []*FilesystemVersion) (incPath []*FilesystemVersion, conflict error) {
|
||||
|
||||
if receiver == nil {
|
||||
panic("receiver must not be nil")
|
||||
}
|
||||
if sender == nil {
|
||||
panic("sender must not be nil")
|
||||
}
|
||||
|
||||
receiver = SortVersionListByCreateTXGThenBookmarkLTSnapshot(receiver)
|
||||
sender = SortVersionListByCreateTXGThenBookmarkLTSnapshot(sender)
|
||||
|
||||
if len(sender) == 0 {
|
||||
return []*FilesystemVersion{}, nil
|
||||
}
|
||||
|
||||
// Find most recent common ancestor by name, preferring snapshots over bookmarks
|
||||
|
||||
mrcaRcv := len(receiver) - 1
|
||||
mrcaSnd := len(sender) - 1
|
||||
|
||||
for mrcaRcv >= 0 && mrcaSnd >= 0 {
|
||||
if receiver[mrcaRcv].Guid == sender[mrcaSnd].Guid {
|
||||
if mrcaSnd-1 >= 0 && sender[mrcaSnd-1].Guid == sender[mrcaSnd].Guid && sender[mrcaSnd-1].Type == FilesystemVersion_Bookmark {
|
||||
// prefer bookmarks over snapshots as the snapshot might go away sooner
|
||||
mrcaSnd -= 1
|
||||
}
|
||||
break
|
||||
}
|
||||
if receiver[mrcaRcv].CreateTXG < sender[mrcaSnd].CreateTXG {
|
||||
mrcaSnd--
|
||||
} else {
|
||||
mrcaRcv--
|
||||
}
|
||||
}
|
||||
|
||||
if mrcaRcv == -1 || mrcaSnd == -1 {
|
||||
return nil, &ConflictNoCommonAncestor{
|
||||
SortedSenderVersions: sender,
|
||||
SortedReceiverVersions: receiver,
|
||||
}
|
||||
}
|
||||
|
||||
if mrcaRcv != len(receiver)-1 {
|
||||
return nil, &ConflictDiverged{
|
||||
SortedSenderVersions: sender,
|
||||
SortedReceiverVersions: receiver,
|
||||
CommonAncestor: sender[mrcaSnd],
|
||||
SenderOnly: sender[mrcaSnd+1:],
|
||||
ReceiverOnly: receiver[mrcaRcv+1:],
|
||||
}
|
||||
}
|
||||
|
||||
// incPath must not contain bookmarks except initial one,
|
||||
incPath = make([]*FilesystemVersion, 0, len(sender))
|
||||
incPath = append(incPath, sender[mrcaSnd])
|
||||
// it's ok if incPath[0] is a bookmark, but not the subsequent ones in the incPath
|
||||
for i := mrcaSnd + 1; i < len(sender); i++ {
|
||||
if sender[i].Type == FilesystemVersion_Snapshot && incPath[len(incPath)-1].Guid != sender[i].Guid {
|
||||
incPath = append(incPath, sender[i])
|
||||
}
|
||||
}
|
||||
if len(incPath) == 1 {
|
||||
// nothing to do
|
||||
incPath = incPath[1:]
|
||||
}
|
||||
return incPath, nil
|
||||
}
|
||||
@@ -0,0 +1,342 @@
|
||||
package mainfsm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/bits"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
. "github.com/zrepl/zrepl/cmd/replication/common"
|
||||
"github.com/zrepl/zrepl/cmd/replication/pdu"
|
||||
"github.com/zrepl/zrepl/cmd/replication/internal/fsfsm"
|
||||
. "github.com/zrepl/zrepl/cmd/replication/internal/mainfsm/queue"
|
||||
)
|
||||
|
||||
//go:generate stringer -type=ReplicationState
|
||||
type ReplicationState uint
|
||||
|
||||
const (
|
||||
Planning ReplicationState = 1 << iota
|
||||
PlanningError
|
||||
Working
|
||||
WorkingWait
|
||||
Completed
|
||||
ContextDone
|
||||
)
|
||||
|
||||
func (s ReplicationState) rsf() replicationStateFunc {
|
||||
idx := bits.TrailingZeros(uint(s))
|
||||
if idx == bits.UintSize {
|
||||
panic(s) // invalid value
|
||||
}
|
||||
m := []replicationStateFunc{
|
||||
rsfPlanning,
|
||||
rsfPlanningError,
|
||||
rsfWorking,
|
||||
rsfWorkingWait,
|
||||
nil,
|
||||
nil,
|
||||
}
|
||||
return m[idx]
|
||||
}
|
||||
|
||||
type Replication struct {
|
||||
// lock protects all fields of this struct (but not the fields behind pointers!)
|
||||
lock sync.Mutex
|
||||
|
||||
state ReplicationState
|
||||
|
||||
// Working, WorkingWait, Completed, ContextDone
|
||||
queue *ReplicationQueue
|
||||
completed []*fsfsm.FSReplication
|
||||
active *ReplicationQueueItemHandle
|
||||
|
||||
// PlanningError
|
||||
planningError error
|
||||
|
||||
// ContextDone
|
||||
contextError error
|
||||
|
||||
// PlanningError, WorkingWait
|
||||
sleepUntil time.Time
|
||||
}
|
||||
|
||||
type Report struct {
|
||||
Status string
|
||||
Problem string
|
||||
Completed []*fsfsm.FilesystemReplicationReport
|
||||
Pending []*fsfsm.FilesystemReplicationReport
|
||||
Active *fsfsm.FilesystemReplicationReport
|
||||
}
|
||||
|
||||
|
||||
func NewReplication() *Replication {
|
||||
r := Replication{
|
||||
state: Planning,
|
||||
}
|
||||
return &r
|
||||
}
|
||||
|
||||
type replicationUpdater func(func(*Replication)) (newState ReplicationState)
|
||||
type replicationStateFunc func(context.Context, EndpointPair, replicationUpdater) replicationStateFunc
|
||||
|
||||
func (r *Replication) Drive(ctx context.Context, ep EndpointPair) {
|
||||
|
||||
var u replicationUpdater = func(f func(*Replication)) ReplicationState {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
if f != nil {
|
||||
f(r)
|
||||
}
|
||||
return r.state
|
||||
}
|
||||
|
||||
var s replicationStateFunc = rsfPlanning
|
||||
var pre, post ReplicationState
|
||||
for s != nil {
|
||||
preTime := time.Now()
|
||||
pre = u(nil)
|
||||
s = s(ctx, ep, u)
|
||||
delta := time.Now().Sub(preTime)
|
||||
post = u(nil)
|
||||
GetLogger(ctx).
|
||||
WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
|
||||
WithField("duration", delta).
|
||||
Debug("main state transition")
|
||||
}
|
||||
|
||||
GetLogger(ctx).
|
||||
WithField("final_state", post).
|
||||
Debug("main final state")
|
||||
}
|
||||
|
||||
func resolveConflict(conflict error) (path []*pdu.FilesystemVersion, msg string) {
|
||||
if noCommonAncestor, ok := conflict.(*ConflictNoCommonAncestor); ok {
|
||||
if len(noCommonAncestor.SortedReceiverVersions) == 0 {
|
||||
// FIXME hard-coded replication policy: most recent
|
||||
// snapshot as source
|
||||
var mostRecentSnap *pdu.FilesystemVersion
|
||||
for n := len(noCommonAncestor.SortedSenderVersions) - 1; n >= 0; n-- {
|
||||
if noCommonAncestor.SortedSenderVersions[n].Type == pdu.FilesystemVersion_Snapshot {
|
||||
mostRecentSnap = noCommonAncestor.SortedSenderVersions[n]
|
||||
break
|
||||
}
|
||||
}
|
||||
if mostRecentSnap == nil {
|
||||
return nil, "no snapshots available on sender side"
|
||||
}
|
||||
return []*pdu.FilesystemVersion{mostRecentSnap}, fmt.Sprintf("start replication at most recent snapshot %s", mostRecentSnap.RelName())
|
||||
}
|
||||
}
|
||||
return nil, "no automated way to handle conflict type"
|
||||
}
|
||||
|
||||
func rsfPlanning(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
||||
|
||||
log := GetLogger(ctx)
|
||||
|
||||
handlePlanningError := func(err error) replicationStateFunc {
|
||||
return u(func(r *Replication) {
|
||||
r.planningError = err
|
||||
r.state = PlanningError
|
||||
}).rsf()
|
||||
}
|
||||
|
||||
sfss, err := ep.Sender().ListFilesystems(ctx)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error listing sender filesystems")
|
||||
return handlePlanningError(err)
|
||||
}
|
||||
|
||||
rfss, err := ep.Receiver().ListFilesystems(ctx)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("error listing receiver filesystems")
|
||||
return handlePlanningError(err)
|
||||
}
|
||||
|
||||
q := NewReplicationQueue()
|
||||
mainlog := log
|
||||
for _, fs := range sfss {
|
||||
|
||||
log := mainlog.WithField("filesystem", fs.Path)
|
||||
|
||||
log.Info("assessing filesystem")
|
||||
|
||||
sfsvs, err := ep.Sender().ListFilesystemVersions(ctx, fs.Path)
|
||||
if err != nil {
|
||||
log.WithError(err).Error("cannot get remote filesystem versions")
|
||||
return handlePlanningError(err)
|
||||
}
|
||||
|
||||
if len(sfsvs) <= 1 {
|
||||
err := errors.New("sender does not have any versions")
|
||||
log.Error(err.Error())
|
||||
q.Add(fsfsm.NewFSReplicationWithPermanentError(fs.Path, err))
|
||||
continue
|
||||
}
|
||||
|
||||
receiverFSExists := false
|
||||
for _, rfs := range rfss {
|
||||
if rfs.Path == fs.Path {
|
||||
receiverFSExists = true
|
||||
}
|
||||
}
|
||||
|
||||
var rfsvs []*pdu.FilesystemVersion
|
||||
if receiverFSExists {
|
||||
rfsvs, err = ep.Receiver().ListFilesystemVersions(ctx, fs.Path)
|
||||
if err != nil {
|
||||
if _, ok := err.(*FilteredError); ok {
|
||||
log.Info("receiver ignores filesystem")
|
||||
continue
|
||||
}
|
||||
log.WithError(err).Error("receiver error")
|
||||
return handlePlanningError(err)
|
||||
}
|
||||
} else {
|
||||
rfsvs = []*pdu.FilesystemVersion{}
|
||||
}
|
||||
|
||||
path, conflict := IncrementalPath(rfsvs, sfsvs)
|
||||
if conflict != nil {
|
||||
var msg string
|
||||
path, msg = resolveConflict(conflict) // no shadowing allowed!
|
||||
if path != nil {
|
||||
log.WithField("conflict", conflict).Info("conflict")
|
||||
log.WithField("resolution", msg).Info("automatically resolved")
|
||||
} else {
|
||||
log.WithField("conflict", conflict).Error("conflict")
|
||||
log.WithField("problem", msg).Error("cannot resolve conflict")
|
||||
}
|
||||
}
|
||||
if path == nil {
|
||||
q.Add(fsfsm.NewFSReplicationWithPermanentError(fs.Path, conflict))
|
||||
continue
|
||||
}
|
||||
|
||||
fsrfsm := fsfsm.BuildFSReplication(fs.Path)
|
||||
if len(path) == 1 {
|
||||
fsrfsm.AddStep(nil, path[0])
|
||||
} else {
|
||||
for i := 0; i < len(path)-1; i++ {
|
||||
fsrfsm.AddStep(path[i], path[i+1])
|
||||
}
|
||||
}
|
||||
qitem := fsrfsm.Done()
|
||||
q.Add(qitem)
|
||||
}
|
||||
|
||||
return u(func(r *Replication) {
|
||||
r.completed = nil
|
||||
r.queue = q
|
||||
r.planningError = nil
|
||||
r.state = Working
|
||||
}).rsf()
|
||||
}
|
||||
|
||||
func rsfPlanningError(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
||||
sleepTime := 10 * time.Second
|
||||
u(func(r *Replication) {
|
||||
r.sleepUntil = time.Now().Add(sleepTime)
|
||||
})
|
||||
t := time.NewTimer(sleepTime) // FIXME make constant onfigurable
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return u(func(r *Replication) {
|
||||
r.state = ContextDone
|
||||
r.contextError = ctx.Err()
|
||||
}).rsf()
|
||||
case <-t.C:
|
||||
return u(func(r *Replication) {
|
||||
r.state = Planning
|
||||
}).rsf()
|
||||
}
|
||||
}
|
||||
|
||||
func rsfWorking(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
||||
|
||||
var active *ReplicationQueueItemHandle
|
||||
rsfNext := u(func(r *Replication) {
|
||||
done, next := r.queue.GetNext()
|
||||
r.completed = append(r.completed, done...)
|
||||
if next == nil {
|
||||
r.state = Completed
|
||||
}
|
||||
r.active = next
|
||||
active = next
|
||||
}).rsf()
|
||||
|
||||
if active == nil {
|
||||
return rsfNext
|
||||
}
|
||||
|
||||
state, nextStepDate := active.GetFSReplication().TakeStep(ctx, ep)
|
||||
|
||||
return u(func(r *Replication) {
|
||||
active.Update(state, nextStepDate)
|
||||
r.active = nil
|
||||
}).rsf()
|
||||
}
|
||||
|
||||
func rsfWorkingWait(ctx context.Context, ep EndpointPair, u replicationUpdater) replicationStateFunc {
|
||||
sleepTime := 10 * time.Second
|
||||
u(func(r *Replication) {
|
||||
r.sleepUntil = time.Now().Add(sleepTime)
|
||||
})
|
||||
t := time.NewTimer(sleepTime)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return u(func(r *Replication) {
|
||||
r.state = ContextDone
|
||||
r.contextError = ctx.Err()
|
||||
}).rsf()
|
||||
case <-t.C:
|
||||
return u(func(r *Replication) {
|
||||
r.state = Working
|
||||
}).rsf()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Replication) Report() *Report {
|
||||
r.lock.Lock()
|
||||
defer r.lock.Unlock()
|
||||
|
||||
rep := Report{
|
||||
Status: r.state.String(),
|
||||
}
|
||||
|
||||
if r.state&(Planning|PlanningError|ContextDone) != 0 {
|
||||
switch r.state {
|
||||
case PlanningError:
|
||||
rep.Problem = r.planningError.Error()
|
||||
case ContextDone:
|
||||
rep.Problem = r.contextError.Error()
|
||||
}
|
||||
return &rep
|
||||
}
|
||||
|
||||
rep.Pending = make([]*fsfsm.FilesystemReplicationReport, 0, r.queue.Len())
|
||||
rep.Completed = make([]*fsfsm.FilesystemReplicationReport, 0, len(r.completed)) // room for active (potentially)
|
||||
|
||||
var active *fsfsm.FSReplication
|
||||
if r.active != nil {
|
||||
active = r.active.GetFSReplication()
|
||||
rep.Active = active.Report()
|
||||
}
|
||||
r.queue.Foreach(func (h *ReplicationQueueItemHandle){
|
||||
fsr := h.GetFSReplication()
|
||||
if active != fsr {
|
||||
rep.Pending = append(rep.Pending, fsr.Report())
|
||||
}
|
||||
})
|
||||
for _, fsr := range r.completed {
|
||||
rep.Completed = append(rep.Completed, fsr.Report())
|
||||
}
|
||||
|
||||
return &rep
|
||||
}
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"time"
|
||||
"sort"
|
||||
|
||||
. "github.com/zrepl/zrepl/cmd/replication/internal/fsfsm"
|
||||
)
|
||||
|
||||
type replicationQueueItem struct {
|
||||
retriesSinceLastError int
|
||||
// duplicates fsr.state to avoid accessing and locking fsr
|
||||
state FSReplicationState
|
||||
// duplicates fsr.current.nextStepDate to avoid accessing & locking fsr
|
||||
nextStepDate time.Time
|
||||
|
||||
fsr *FSReplication
|
||||
}
|
||||
|
||||
type ReplicationQueue []*replicationQueueItem
|
||||
|
||||
func NewReplicationQueue() *ReplicationQueue {
|
||||
q := make(ReplicationQueue, 0)
|
||||
return &q
|
||||
}
|
||||
|
||||
func (q ReplicationQueue) Len() int { return len(q) }
|
||||
func (q ReplicationQueue) Swap(i, j int) { q[i], q[j] = q[j], q[i] }
|
||||
|
||||
type lessmapEntry struct{
|
||||
prio int
|
||||
less func(a,b *replicationQueueItem) bool
|
||||
}
|
||||
|
||||
var lessmap = map[FSReplicationState]lessmapEntry {
|
||||
FSReady: {
|
||||
prio: 0,
|
||||
less: func(a, b *replicationQueueItem) bool {
|
||||
return a.nextStepDate.Before(b.nextStepDate)
|
||||
},
|
||||
},
|
||||
FSRetryWait: {
|
||||
prio: 1,
|
||||
less: func(a, b *replicationQueueItem) bool {
|
||||
return a.retriesSinceLastError < b.retriesSinceLastError
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func (q ReplicationQueue) Less(i, j int) bool {
|
||||
|
||||
a, b := q[i], q[j]
|
||||
al, aok := lessmap[a.state]
|
||||
if !aok {
|
||||
panic(a)
|
||||
}
|
||||
bl, bok := lessmap[b.state]
|
||||
if !bok {
|
||||
panic(b)
|
||||
}
|
||||
|
||||
if al.prio != bl.prio {
|
||||
return al.prio < bl.prio
|
||||
}
|
||||
|
||||
return al.less(a, b)
|
||||
}
|
||||
|
||||
func (q *ReplicationQueue) sort() (done []*FSReplication) {
|
||||
// pre-scan for everything that is not ready
|
||||
newq := make(ReplicationQueue, 0, len(*q))
|
||||
done = make([]*FSReplication, 0, len(*q))
|
||||
for _, qitem := range *q {
|
||||
if _, ok := lessmap[qitem.state]; !ok {
|
||||
done = append(done, qitem.fsr)
|
||||
} else {
|
||||
newq = append(newq, qitem)
|
||||
}
|
||||
}
|
||||
sort.Stable(newq) // stable to avoid flickering in reports
|
||||
*q = newq
|
||||
return done
|
||||
}
|
||||
|
||||
// next remains valid until the next call to GetNext()
|
||||
func (q *ReplicationQueue) GetNext() (done []*FSReplication, next *ReplicationQueueItemHandle) {
|
||||
done = q.sort()
|
||||
if len(*q) == 0 {
|
||||
return done, nil
|
||||
}
|
||||
next = &ReplicationQueueItemHandle{(*q)[0]}
|
||||
return done, next
|
||||
}
|
||||
|
||||
func (q *ReplicationQueue) Add(fsr *FSReplication) {
|
||||
*q = append(*q, &replicationQueueItem{
|
||||
fsr: fsr,
|
||||
state: fsr.State(),
|
||||
})
|
||||
}
|
||||
|
||||
func (q *ReplicationQueue) Foreach(fu func(*ReplicationQueueItemHandle)) {
|
||||
for _, qitem := range *q {
|
||||
fu(&ReplicationQueueItemHandle{qitem})
|
||||
}
|
||||
}
|
||||
|
||||
type ReplicationQueueItemHandle struct {
|
||||
i *replicationQueueItem
|
||||
}
|
||||
|
||||
func (h ReplicationQueueItemHandle) GetFSReplication() *FSReplication {
|
||||
return h.i.fsr
|
||||
}
|
||||
|
||||
func (h ReplicationQueueItemHandle) Update(newState FSReplicationState, nextStepDate time.Time) {
|
||||
h.i.state = newState
|
||||
h.i.nextStepDate = nextStepDate
|
||||
if h.i.state&FSReady != 0 {
|
||||
h.i.retriesSinceLastError = 0
|
||||
} else if h.i.state&FSRetryWait != 0 {
|
||||
h.i.retriesSinceLastError++
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
// Code generated by "stringer -type=ReplicationState"; DO NOT EDIT.
|
||||
|
||||
package mainfsm
|
||||
|
||||
import "strconv"
|
||||
|
||||
const (
|
||||
_ReplicationState_name_0 = "PlanningPlanningError"
|
||||
_ReplicationState_name_1 = "Working"
|
||||
_ReplicationState_name_2 = "WorkingWait"
|
||||
_ReplicationState_name_3 = "Completed"
|
||||
_ReplicationState_name_4 = "ContextDone"
|
||||
)
|
||||
|
||||
var (
|
||||
_ReplicationState_index_0 = [...]uint8{0, 8, 21}
|
||||
)
|
||||
|
||||
func (i ReplicationState) String() string {
|
||||
switch {
|
||||
case 1 <= i && i <= 2:
|
||||
i -= 1
|
||||
return _ReplicationState_name_0[_ReplicationState_index_0[i]:_ReplicationState_index_0[i+1]]
|
||||
case i == 4:
|
||||
return _ReplicationState_name_1
|
||||
case i == 8:
|
||||
return _ReplicationState_name_2
|
||||
case i == 16:
|
||||
return _ReplicationState_name_3
|
||||
case i == 32:
|
||||
return _ReplicationState_name_4
|
||||
default:
|
||||
return "ReplicationState(" + strconv.FormatInt(int64(i), 10) + ")"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user