replication: refactor driving logic (no more explicit state machine)

daemon/job/active.go

@@ -17,10 +17,12 @@ import (
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/replication"
+	"github.com/zrepl/zrepl/replication/driver"
+	"github.com/zrepl/zrepl/replication/logic"
+	"github.com/zrepl/zrepl/replication/report"
 	"github.com/zrepl/zrepl/rpc"
 	"github.com/zrepl/zrepl/transport"
 	"github.com/zrepl/zrepl/transport/fromconfig"
-	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -53,7 +55,7 @@ type activeSideTasks struct {
 	state ActiveSideState
 
 	// valid for state ActiveSideReplicating, ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
-	replication       *replication.Replication
+	replicationReport driver.ReportFunc
 	replicationCancel context.CancelFunc
 
 	// valid for state ActiveSidePruneSender, ActiveSidePruneReceiver, ActiveSideDone
@@ -79,7 +81,7 @@ func (a *ActiveSide) updateTasks(u func(*activeSideTasks)) activeSideTasks {
 type activeMode interface {
 	ConnectEndpoints(rpcLoggers rpc.Loggers, connecter transport.Connecter)
 	DisconnectEndpoints()
-	SenderReceiver() (replication.Sender, replication.Receiver)
+	SenderReceiver() (logic.Sender, logic.Receiver)
 	Type() Type
 	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
 	ResetConnectBackoff()
@@ -111,7 +113,7 @@ func (m *modePush) DisconnectEndpoints() {
 	m.receiver = nil
 }
 
-func (m *modePush) SenderReceiver() (replication.Sender, replication.Receiver) {
+func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
 	m.setupMtx.Lock()
 	defer m.setupMtx.Unlock()
 	return m.sender, m.receiver
@@ -172,7 +174,7 @@ func (m *modePull) DisconnectEndpoints() {
 	m.receiver = nil
 }
 
-func (m *modePull) SenderReceiver() (replication.Sender, replication.Receiver) {
+func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
 	m.setupMtx.Lock()
 	defer m.setupMtx.Unlock()
 	return m.sender, m.receiver
@@ -274,7 +276,7 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
 func (j *ActiveSide) Name() string { return j.name }
 
 type ActiveSideStatus struct {
-	Replication                    *replication.Report
+	Replication                    *report.Report
 	PruningSender, PruningReceiver *pruner.Report
 }
 
@@ -283,8 +285,8 @@ func (j *ActiveSide) Status() *Status {
 
 	s := &ActiveSideStatus{}
 	t := j.mode.Type()
-	if tasks.replication != nil {
-		s.Replication = tasks.replication.Report()
+	if tasks.replicationReport != nil {
+		s.Replication = tasks.replicationReport()
 	}
 	if tasks.prunerSender != nil {
 		s.PruningSender = tasks.prunerSender.Report()
@@ -345,78 +347,6 @@ func (j *ActiveSide) do(ctx context.Context) {
 		}
 	}()
 
-	// The code after this watchdog goroutine is sequential and transitions the state from
-	//   ActiveSideReplicating -> ActiveSidePruneSender -> ActiveSidePruneReceiver -> ActiveSideDone
-	// If any of those sequential tasks 'gets stuck' (livelock, no progress), the watchdog will eventually
-	// cancel its context.
-	// If the task is written to support context cancellation, it will return immediately (in permanent error state),
-	// and the sequential code above transitions to the next state.
-	go func() {
-
-		wdto := envconst.Duration("ZREPL_JOB_WATCHDOG_TIMEOUT", 10*time.Minute)
-		jitter := envconst.Duration("ZREPL_JOB_WATCHDOG_JITTER", 1*time.Second)
-		// shadowing!
-		log := log.WithField("watchdog_timeout", wdto.String())
-
-		log.Debug("starting watchdog")
-		defer log.Debug("watchdog stopped")
-
-		t := time.NewTicker(wdto)
-		defer t.Stop()
-
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case <-t.C: // fall
-			}
-
-			j.updateTasks(func(tasks *activeSideTasks) {
-				// Since cancelling a task will cause the sequential code to transition to the next state immediately,
-				// we cannot check for its progress right then (no fallthrough).
-				// Instead, we return (not continue because we are in a closure) and give the new state another
-				// ZREPL_JOB_WATCHDOG_TIMEOUT interval to try make some progress.
-
-				log.WithField("state", tasks.state).Debug("watchdog firing")
-
-				const WATCHDOG_ENVCONST_NOTICE = " (adjust ZREPL_JOB_WATCHDOG_TIMEOUT env variable if inappropriate)"
-
-				switch tasks.state {
-				case ActiveSideReplicating:
-					log.WithField("replication_progress", tasks.replication.Progress.String()).
-						Debug("check replication progress")
-					if tasks.replication.Progress.CheckTimeout(wdto, jitter) {
-						log.Error("replication did not make progress, cancelling" + WATCHDOG_ENVCONST_NOTICE)
-						tasks.replicationCancel()
-						return
-					}
-				case ActiveSidePruneSender:
-					log.WithField("prune_sender_progress", tasks.replication.Progress.String()).
-						Debug("check pruner_sender progress")
-					if tasks.prunerSender.Progress.CheckTimeout(wdto, jitter) {
-						log.Error("pruner_sender did not make progress, cancelling" + WATCHDOG_ENVCONST_NOTICE)
-						tasks.prunerSenderCancel()
-						return
-					}
-				case ActiveSidePruneReceiver:
-					log.WithField("prune_receiver_progress", tasks.replication.Progress.String()).
-						Debug("check pruner_receiver progress")
-					if tasks.prunerReceiver.Progress.CheckTimeout(wdto, jitter) {
-						log.Error("pruner_receiver did not make progress, cancelling" + WATCHDOG_ENVCONST_NOTICE)
-						tasks.prunerReceiverCancel()
-						return
-					}
-				case ActiveSideDone:
-					// ignore, ctx will be Done() in a few milliseconds and the watchdog will exit
-				default:
-					log.WithField("state", tasks.state).
-						Error("watchdog implementation error: unknown active side state")
-				}
-			})
-
-		}
-	}()
-
 	sender, receiver := j.mode.SenderReceiver()
 
 	{
@@ -426,16 +356,19 @@ func (j *ActiveSide) do(ctx context.Context) {
 		default:
 		}
 		ctx, repCancel := context.WithCancel(ctx)
-		tasks := j.updateTasks(func(tasks *activeSideTasks) {
+		var repWait driver.WaitFunc
+		j.updateTasks(func(tasks *activeSideTasks) {
 			// reset it
 			*tasks = activeSideTasks{}
 			tasks.replicationCancel = repCancel
-			tasks.replication = replication.NewReplication(j.promRepStateSecs, j.promBytesReplicated)
+			tasks.replicationReport, repWait = replication.Do(
+				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver),
+			)
 			tasks.state = ActiveSideReplicating
 		})
 		log.Info("start replication")
-		tasks.replication.Drive(ctx, sender, receiver)
-		repCancel() // always cancel to free up context resources
+		repWait(true) // wait blocking
+		repCancel()   // always cancel to free up context resources
 	}
 
 	{

daemon/logging/build_logging.go

@@ -14,7 +14,6 @@ import (
 	"github.com/zrepl/zrepl/daemon/snapper"
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
-	"github.com/zrepl/zrepl/replication"
 	"github.com/zrepl/zrepl/rpc"
 	"github.com/zrepl/zrepl/rpc/transportmux"
 	"github.com/zrepl/zrepl/tlsconf"
@@ -78,7 +77,6 @@ const (
 )
 
 func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
-	ctx = replication.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
 	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsyEndpoint))
 	ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
 	ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))

daemon/pruner/pruner.go

@@ -8,7 +8,7 @@ import (
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/pruning"
-	"github.com/zrepl/zrepl/replication/pdu"
+	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/util/watchdog"
 	"net"

daemon/pruner/pruner_test.go

@@ -6,7 +6,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/pruning"
-	"github.com/zrepl/zrepl/replication/pdu"
+	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"net"
 	"testing"
 	"time"