bring back prometheus metrics, with new metrics for replication state machine

replication/fsrep/fsfsm.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
 	"io"
 	"net"
 	"sync"
@@ -90,7 +91,9 @@ func (s State) fsrsf() state {
 }
 
 type Replication struct {
-	// lock protects all fields in this struct, but not the data behind pointers
+	promBytesReplicated prometheus.Counter
+
+	// lock protects all fields below it in this struct, but not the data behind pointers
 	lock               sync.Mutex
 	state              State
 	fs                 string
@@ -120,8 +123,8 @@ type ReplicationBuilder struct {
 	r *Replication
 }
 
-func BuildReplication(fs string) *ReplicationBuilder {
-	return &ReplicationBuilder{&Replication{fs: fs}}
+func BuildReplication(fs string, promBytesReplicated prometheus.Counter) *ReplicationBuilder {
+	return &ReplicationBuilder{&Replication{fs: fs, promBytesReplicated: promBytesReplicated}}
 }
 
 func (b *ReplicationBuilder) AddStep(from, to FilesystemVersion) *ReplicationBuilder {
@@ -204,6 +207,7 @@ func (f *Replication) TakeStep(ctx context.Context, sender Sender, receiver Rece
 	preTime := time.Now()
 	s = s(ctx, sender, receiver, u)
 	delta := time.Now().Sub(preTime)
+
 	post = u(func(f *Replication) {
 		if len(f.pending) == 0 {
 			return
@@ -369,6 +373,9 @@ func (s *ReplicationStep) doReplication(ctx context.Context, sender Sender, rece
 	}
 
 	s.byteCounter = util.NewByteCounterReader(sstream)
+	defer func() {
+		s.parent.promBytesReplicated.Add(float64(s.byteCounter.Bytes()))
+	}()
 	sstream = s.byteCounter
 
 	rr := &pdu.ReceiveReq{

replication/mainfsm.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/prometheus/client_golang/prometheus"
 	"math/bits"
 	"sync"
 	"time"
@@ -49,6 +50,10 @@ func (s State) rsf() state {
 // It is a state machine that is driven by the Drive method
 // and provides asynchronous reporting via the Report method (i.e. from another goroutine).
 type Replication struct {
+	// not protected by lock
+	promSecsPerState *prometheus.HistogramVec // labels: state
+	promBytesReplicated *prometheus.CounterVec // labels: filesystem
+
 	// lock protects all fields of this struct (but not the fields behind pointers!)
 	lock sync.Mutex
 
@@ -77,9 +82,11 @@ type Report struct {
 	Active    *fsrep.Report
 }
 
-func NewReplication() *Replication {
+func NewReplication(secsPerState *prometheus.HistogramVec, bytesReplicated *prometheus.CounterVec) *Replication {
 	r := Replication{
-		state: Planning,
+		promSecsPerState: secsPerState,
+		promBytesReplicated: bytesReplicated,
+		state:            Planning,
 	}
 	return &r
 }
@@ -142,6 +149,7 @@ func (r *Replication) Drive(ctx context.Context, sender Sender, receiver Receive
 		pre = u(nil)
 		s = s(ctx, sender, receiver, u)
 		delta := time.Now().Sub(preTime)
+		r.promSecsPerState.WithLabelValues(pre.String()).Observe(delta.Seconds())
 		post = u(nil)
 		getLogger(ctx).
 			WithField("transition", fmt.Sprintf("%s => %s", pre, post)).
@@ -262,7 +270,11 @@ func statePlanning(ctx context.Context, sender Sender, receiver Receiver, u upda
 			continue
 		}
 
-		fsrfsm := fsrep.BuildReplication(fs.Path)
+		var promBytesReplicated *prometheus.CounterVec
+		u(func(replication *Replication) { // FIXME args struct like in pruner (also use for sender and receiver)
+			promBytesReplicated = replication.promBytesReplicated
+		})
+		fsrfsm := fsrep.BuildReplication(fs.Path, promBytesReplicated.WithLabelValues(fs.Path))
 		if len(path) == 1 {
 			fsrfsm.AddStep(nil, path[0])
 		} else {