replication: rewrite error handling + simplify state machines

* Remove explicity state machine code for all but replication.Replication
* Introduce explicit error types that satisfy interfaces which provide
  sufficient information for replication.Replication to make intelligent
  retry + queuing decisions

  * Temporary()
  * LocalToFS()

* Remove the queue and replace it with a simple array that we sort each
  time (yay no generics :( )

replication/fsrep/fsfsm.go

@@ -71,38 +71,29 @@ type Report struct {
 	Completed, Pending []*StepReport
 }
 
-//go:generate stringer -type=State
+//go:generate enumer -type=State
 type State uint
 
 const (
 	Ready State = 1 << iota
-	Retry
-	PermanentError
 	Completed
 )
 
-func (s State) fsrsf() state {
-	m := map[State]state{
-		Ready:          stateReady,
-		Retry:          stateRetry,
-		PermanentError: nil,
-		Completed:      nil,
-	}
-	return m[s]
-}
-
-func (s State) IsErrorState() bool {
-	return s & (Retry|PermanentError) != 0
+type Error interface {
+	error
+	Temporary() bool
+	LocalToFS() bool
 }
 
 type Replication struct {
 	promBytesReplicated prometheus.Counter
 
+	fs                 string
+
 	// lock protects all fields below it in this struct, but not the data behind pointers
 	lock               sync.Mutex
 	state              State
-	fs                 string
-	err                error
+	err                Error
 	completed, pending []*ReplicationStep
 }
 
@@ -112,13 +103,35 @@ func (f *Replication) State() State {
 	return f.state
 }
 
-func (f *Replication) Err() error {
+func (f *Replication) FS() string { return f.fs }
+
+// returns zero value time.Time{} if no more pending steps
+func (f *Replication) NextStepDate() time.Time {
+	if len(f.pending) == 0 {
+		return time.Time{}
+	}
+	return f.pending[0].to.SnapshotTime()
+}
+
+func (f *Replication) Err() Error {
 	f.lock.Lock()
 	defer f.lock.Unlock()
-	if f.state & (Retry|PermanentError) != 0 {
-		return f.err
+	return f.err
+}
+
+func (f *Replication) CanRetry() bool {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+	if f.state == Completed {
+		return false
 	}
-	return nil
+	if f.state != Ready {
+		panic(fmt.Sprintf("implementation error: %v", f.state))
+	}
+	if f.err == nil {
+		return true
+	}
+	return f.err.Temporary()
 }
 
 func (f *Replication) UpdateSizeEsitmate(ctx context.Context, sender Sender) error {
@@ -162,26 +175,37 @@ func (b *ReplicationBuilder) Done() (r *Replication) {
 	return r
 }
 
-func NewReplicationWithPermanentError(fs string, err error) *Replication {
+type ReplicationConflictError struct {
+	Err error
+}
+
+func (e *ReplicationConflictError) Timeout() bool { return false }
+
+func (e *ReplicationConflictError) Temporary() bool { return false }
+
+func (e *ReplicationConflictError) Error() string { return fmt.Sprintf("permanent error: %s", e.Err.Error()) }
+
+func (e *ReplicationConflictError) LocalToFS() bool { return true }
+
+func NewReplicationConflictError(fs string, err error) *Replication {
 	return &Replication{
-		state: PermanentError,
+		state: Completed,
 		fs:    fs,
-		err:   err,
+		err:   &ReplicationConflictError{Err: err},
 	}
 }
 
-//go:generate stringer -type=StepState
+//go:generate enumer -type=StepState
 type StepState uint
 
 const (
 	StepReplicationReady StepState = 1 << iota
-	StepReplicationRetry
 	StepMarkReplicatedReady
-	StepMarkReplicatedRetry
-	StepPermanentError
 	StepCompleted
 )
 
+func (s StepState) IsTerminal() bool { return s == StepCompleted }
+
 type FilesystemVersion interface {
 	SnapshotTime() time.Time
 	GetName() string // name without @ or #
@@ -204,7 +228,7 @@ type ReplicationStep struct {
 	expectedSize int64 // 0 means no size estimate present / possible
 }
 
-func (f *Replication) TakeStep(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver) (post State, nextStepDate time.Time) {
+func (f *Replication) Retry(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver) Error {
 
 	var u updater = func(fu func(*Replication)) State {
 		f.lock.Lock()
@@ -214,76 +238,98 @@ func (f *Replication) TakeStep(ctx context.Context, ka *watchdog.KeepAlive, send
 		}
 		return f.state
 	}
-	var s state = u(nil).fsrsf()
 
-	pre := u(nil)
-	preTime := time.Now()
-	s = s(ctx, ka, sender, receiver, u)
-	delta := time.Now().Sub(preTime)
-
-	post = u(func(f *Replication) {
-		if len(f.pending) == 0 {
-			return
-		}
-		nextStepDate = f.pending[0].to.SnapshotTime()
-	})
-
-	getLogger(ctx).
-		WithField("fs", f.fs).
-		WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
-		WithField("duration", delta).
-		Debug("fsr step taken")
-
-	return post, nextStepDate
-}
-
-type updater func(func(fsr *Replication)) State
-
-type state func(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state
-
-func stateReady(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state {
 
 	var current *ReplicationStep
-	s := u(func(f *Replication) {
+	pre := u(nil)
+	getLogger(ctx).WithField("fsrep_state", pre).Debug("begin fsrep.Retry")
+	defer func() {
+		post := u(nil)
+		getLogger(ctx).WithField("fsrep_transition", post).Debug("end fsrep.Retry")
+	}()
+
+	st := u(func(f *Replication) {
 		if len(f.pending) == 0 {
 			f.state = Completed
 			return
 		}
 		current = f.pending[0]
 	})
-	if s != Ready {
-		return s.fsrsf()
+	if st == Completed {
+		return nil
+	}
+	if st != Ready {
+		panic(fmt.Sprintf("implementation error: %v", st))
 	}
 
-	stepState := current.doReplication(ctx, ka, sender, receiver)
+	stepCtx := WithLogger(ctx, getLogger(ctx).WithField("step", current))
+	getLogger(stepCtx).Debug("take step")
+	err := current.Retry(stepCtx, ka, sender, receiver)
+	if err != nil {
+		getLogger(stepCtx).WithError(err).Error("step could not be completed")
+	}
 
-	return u(func(f *Replication) {
-		switch stepState {
-		case StepCompleted:
-			f.completed = append(f.completed, current)
-			f.pending = f.pending[1:]
-			if len(f.pending) > 0 {
-				f.state = Ready
-			} else {
-				f.state = Completed
-			}
-		case StepReplicationRetry:
-			fallthrough
-		case StepMarkReplicatedRetry:
-			f.state = Retry
-		case StepPermanentError:
-			f.state = PermanentError
-			f.err = errors.New("a replication step failed with a permanent error")
-		default:
-			panic(f)
+	u(func(fsr *Replication) {
+		if err != nil {
+			f.err = &StepError{stepStr: current.String(), err: err}
+			return
 		}
-	}).fsrsf()
+		if err == nil && current.state != StepCompleted {
+			panic(fmt.Sprintf("implementation error: %v", current.state))
+		}
+		f.err = nil
+		f.completed = append(f.completed, current)
+		f.pending = f.pending[1:]
+		if len(f.pending) > 0 {
+			f.state = Ready
+		} else {
+			f.state = Completed
+		}
+	})
+	var retErr Error = nil
+	u(func(fsr *Replication) {
+		retErr = fsr.err
+	})
+	return retErr
 }
 
-func stateRetry(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state {
-	return u(func(fsr *Replication) {
-		fsr.state = Ready
-	}).fsrsf()
+type updater func(func(fsr *Replication)) State
+
+type state func(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver, u updater) state
+
+type StepError struct {
+	stepStr string
+	err     error
+}
+
+var _ Error = &StepError{}
+
+func (e StepError) Error() string {
+	if e.LocalToFS() {
+		return fmt.Sprintf("step %s failed: %s", e.stepStr, e.err)
+	}
+	return e.err.Error()
+}
+
+func (e StepError) Timeout() bool {
+	if neterr, ok := e.err.(net.Error); ok {
+		return neterr.Timeout()
+	}
+	return false
+}
+
+func (e StepError) Temporary() bool {
+	if neterr, ok := e.err.(net.Error); ok {
+		return neterr.Temporary()
+	}
+	return false
+}
+
+func (e StepError) LocalToFS() bool {
+	if _, ok := e.err.(net.Error); ok {
+		return false
+	}
+	return true // conservative approximation: we'd like to check for specific errors returned over RPC here...
 }
 
 func (fsr *Replication) Report() *Report {
@@ -295,9 +341,8 @@ func (fsr *Replication) Report() *Report {
 		Status:     fsr.state.String(),
 	}
 
-	if fsr.state&PermanentError != 0 {
+	if fsr.err != nil && fsr.err.LocalToFS() {
 		rep.Problem = fsr.err.Error()
-		return &rep
 	}
 
 	rep.Completed = make([]*StepReport, len(fsr.completed))
@@ -309,70 +354,48 @@ func (fsr *Replication) Report() *Report {
 		rep.Pending[i] = fsr.pending[i].Report()
 	}
 
-	if fsr.state&Retry != 0 {
-		if len(rep.Pending) != 0 { // should always be true for Retry == true?
-			rep.Problem = rep.Pending[0].Problem
-		}
-	}
-
 	return &rep
 }
 
-func shouldRetry(err error) bool {
-	switch err {
-	case io.EOF:
-		fallthrough
-	case io.ErrUnexpectedEOF:
-		fallthrough
-	case io.ErrClosedPipe:
-		return true
+func (s *ReplicationStep) Retry(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver) error {
+	switch s.state {
+	case StepReplicationReady:
+		return s.doReplication(ctx, ka, sender, receiver)
+	case StepMarkReplicatedReady:
+		return s.doMarkReplicated(ctx, ka, sender)
+	case StepCompleted:
+		return nil
 	}
-	if _, ok := err.(net.Error); ok {
-		return true
-	}
-	return false
+	panic(fmt.Sprintf("implementation error: %v", s.state))
 }
 
-func (s *ReplicationStep) doReplication(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver) StepState {
+func (s *ReplicationStep) Error() error {
+	if s.state & (StepReplicationReady|StepMarkReplicatedReady) != 0 {
+		return s.err
+	}
+	return nil
+}
+
+func (s *ReplicationStep) doReplication(ctx context.Context, ka *watchdog.KeepAlive, sender Sender, receiver Receiver) error {
+
+	if s.state != StepReplicationReady {
+		panic(fmt.Sprintf("implementation error: %v", s.state))
+	}
 
 	fs := s.parent.fs
 
-	log := getLogger(ctx).
-		WithField("filesystem", fs).
-		WithField("step", s.String())
-
-	updateStateError := func(err error) StepState {
-		s.lock.Lock()
-		defer s.lock.Unlock()
-
-		s.err = err
-		if shouldRetry(s.err) {
-			s.state = StepReplicationRetry
-			return s.state
-		}
-		s.state = StepPermanentError
-		return s.state
-	}
-
-	updateStateCompleted := func() StepState {
-		s.lock.Lock()
-		defer s.lock.Unlock()
-		s.err = nil
-		s.state = StepMarkReplicatedReady
-		return s.state
-	}
-
+	log := getLogger(ctx)
 	sr := s.buildSendRequest(false)
 
 	log.Debug("initiate send request")
 	sres, sstream, err := sender.Send(ctx, sr)
 	if err != nil {
 		log.WithError(err).Error("send request failed")
-		return updateStateError(err)
+		return err
 	}
 	if sstream == nil {
 		err := errors.New("send request did not return a stream, broken endpoint implementation")
-		return updateStateError(err)
+		return err
 	}
 
 	s.byteCounter = util.NewByteCounterReader(sstream)
@@ -400,42 +423,23 @@ func (s *ReplicationStep) doReplication(ctx context.Context, ka *watchdog.KeepAl
 		// 	- an unexpected exit of ZFS on the sending side
 		//  - an unexpected exit of ZFS on the receiving side
 		//  - a connectivity issue
-		return updateStateError(err)
+		return err
 	}
 	log.Debug("receive finished")
 	ka.MadeProgress()
 
-	updateStateCompleted()
-
+	s.state = StepMarkReplicatedReady
 	return s.doMarkReplicated(ctx, ka, sender)
 
 }
 
-func (s *ReplicationStep) doMarkReplicated(ctx context.Context, ka *watchdog.KeepAlive, sender Sender) StepState {
+func (s *ReplicationStep) doMarkReplicated(ctx context.Context, ka *watchdog.KeepAlive, sender Sender) error {
 
-	log := getLogger(ctx).
-		WithField("filesystem", s.parent.fs).
-		WithField("step", s.String())
-
-	updateStateError := func(err error) StepState {
-		s.lock.Lock()
-		defer s.lock.Unlock()
-
-		s.err = err
-		if shouldRetry(s.err) {
-			s.state = StepMarkReplicatedRetry
-			return s.state
-		}
-		s.state = StepPermanentError
-		return s.state
+	if s.state != StepMarkReplicatedReady {
+		panic(fmt.Sprintf("implementation error: %v", s.state))
 	}
 
-	updateStateCompleted := func() StepState {
-		s.lock.Lock()
-		defer s.lock.Unlock()
-		s.state = StepCompleted
-		return s.state
-	}
+	log := getLogger(ctx)
 
 	log.Debug("advance replication cursor")
 	req := &pdu.ReplicationCursorReq{
@@ -449,25 +453,22 @@ func (s *ReplicationStep) doMarkReplicated(ctx context.Context, ka *watchdog.Kee
 	res, err := sender.ReplicationCursor(ctx, req)
 	if err != nil {
 		log.WithError(err).Error("error advancing replication cursor")
-		return updateStateError(err)
+		return err
 	}
 	if res.GetError() != "" {
 		err := fmt.Errorf("cannot advance replication cursor: %s", res.GetError())
 		log.Error(err.Error())
-		return updateStateError(err)
+		return err
 	}
 	ka.MadeProgress()
 
-	return updateStateCompleted()
+	s.state = StepCompleted
+	return err
 }
 
 func (s *ReplicationStep) updateSizeEstimate(ctx context.Context, sender Sender) error {
 
-	fs := s.parent.fs
-
-	log := getLogger(ctx).
-		WithField("filesystem", fs).
-		WithField("step", s.String())
+	log := getLogger(ctx)
 
 	sr := s.buildSendRequest(true)
 

replication/fsrep/state_enumer.go

@@ -0,0 +1,50 @@
+// Code generated by "enumer -type=State"; DO NOT EDIT.
+
+package fsrep
+
+import (
+	"fmt"
+)
+
+const _StateName = "ReadyCompleted"
+
+var _StateIndex = [...]uint8{0, 5, 14}
+
+func (i State) String() string {
+	i -= 1
+	if i >= State(len(_StateIndex)-1) {
+		return fmt.Sprintf("State(%d)", i+1)
+	}
+	return _StateName[_StateIndex[i]:_StateIndex[i+1]]
+}
+
+var _StateValues = []State{1, 2}
+
+var _StateNameToValueMap = map[string]State{
+	_StateName[0:5]:  1,
+	_StateName[5:14]: 2,
+}
+
+// StateString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func StateString(s string) (State, error) {
+	if val, ok := _StateNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to State values", s)
+}
+
+// StateValues returns all values of the enum
+func StateValues() []State {
+	return _StateValues
+}
+
+// IsAState returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i State) IsAState() bool {
+	for _, v := range _StateValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

replication/fsrep/state_string.go

@@ -1,29 +0,0 @@
-// Code generated by "stringer -type=State"; DO NOT EDIT.
-
-package fsrep
-
-import "strconv"
-
-const (
-	_State_name_0 = "ReadyRetry"
-	_State_name_1 = "PermanentError"
-	_State_name_2 = "Completed"
-)
-
-var (
-	_State_index_0 = [...]uint8{0, 5, 10}
-)
-
-func (i State) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _State_name_0[_State_index_0[i]:_State_index_0[i+1]]
-	case i == 4:
-		return _State_name_1
-	case i == 8:
-		return _State_name_2
-	default:
-		return "State(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-}

replication/fsrep/stepstate_enumer.go

@@ -0,0 +1,61 @@
+// Code generated by "enumer -type=StepState"; DO NOT EDIT.
+
+package fsrep
+
+import (
+	"fmt"
+)
+
+const (
+	_StepStateName_0 = "StepReplicationReadyStepMarkReplicatedReady"
+	_StepStateName_1 = "StepCompleted"
+)
+
+var (
+	_StepStateIndex_0 = [...]uint8{0, 20, 43}
+	_StepStateIndex_1 = [...]uint8{0, 13}
+)
+
+func (i StepState) String() string {
+	switch {
+	case 1 <= i && i <= 2:
+		i -= 1
+		return _StepStateName_0[_StepStateIndex_0[i]:_StepStateIndex_0[i+1]]
+	case i == 4:
+		return _StepStateName_1
+	default:
+		return fmt.Sprintf("StepState(%d)", i)
+	}
+}
+
+var _StepStateValues = []StepState{1, 2, 4}
+
+var _StepStateNameToValueMap = map[string]StepState{
+	_StepStateName_0[0:20]:  1,
+	_StepStateName_0[20:43]: 2,
+	_StepStateName_1[0:13]:  4,
+}
+
+// StepStateString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func StepStateString(s string) (StepState, error) {
+	if val, ok := _StepStateNameToValueMap[s]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to StepState values", s)
+}
+
+// StepStateValues returns all values of the enum
+func StepStateValues() []StepState {
+	return _StepStateValues
+}
+
+// IsAStepState returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i StepState) IsAStepState() bool {
+	for _, v := range _StepStateValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}

replication/fsrep/stepstate_string.go

@@ -1,35 +0,0 @@
-// Code generated by "stringer -type=StepState"; DO NOT EDIT.
-
-package fsrep
-
-import "strconv"
-
-const (
-	_StepState_name_0 = "StepReplicationReadyStepReplicationRetry"
-	_StepState_name_1 = "StepMarkReplicatedReady"
-	_StepState_name_2 = "StepMarkReplicatedRetry"
-	_StepState_name_3 = "StepPermanentError"
-	_StepState_name_4 = "StepCompleted"
-)
-
-var (
-	_StepState_index_0 = [...]uint8{0, 20, 40}
-)
-
-func (i StepState) String() string {
-	switch {
-	case 1 <= i && i <= 2:
-		i -= 1
-		return _StepState_name_0[_StepState_index_0[i]:_StepState_index_0[i+1]]
-	case i == 4:
-		return _StepState_name_1
-	case i == 8:
-		return _StepState_name_2
-	case i == 16:
-		return _StepState_name_3
-	case i == 32:
-		return _StepState_name_4
-	default:
-		return "StepState(" + strconv.FormatInt(int64(i), 10) + ")"
-	}
-}