build: make platformtest-* usable

* use plain `go build` instead of the `go test` + `func TestMain` hack
* add target cover-platform and use that to generate coverage reports

Makefile

@@ -147,11 +147,26 @@ zrepl-bin:
 generate-platform-test-list:
 	$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
 
-test-platform-bin:
+COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
+cover-platform-bin:
 	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
-		-c -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
+		-c -o "$(COVER_PLATFORM_BIN_PATH)" \
 		-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
 		./platformtest/harness
+cover-platform:
+	# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
+	export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
+		-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
+		-test.v \
+		__DEVEL--i-heard-you-like-tests"; \
+		$(MAKE) _test-or-cover-platform-impl
+
+TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
+test-platform-bin:
+	$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
+test-platform:
+	export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
+		$(MAKE) _test-or-cover-platform-impl
 
 ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
 ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
@@ -159,14 +174,14 @@ ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
 ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
 # ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
 ZREPL_PLATFORMTEST_ARGS := 
-test-platform: $(ARTIFACTDIR) # do not track dependency on test-platform-bin to allow build of platformtest outside of test VM
+_test-or-cover-platform-impl: $(ARTIFACTDIR)
+ifndef _TEST_PLATFORM_CMD
+	$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
+endif
 	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
 	rm -f "$(ARTIFACTDIR)/platformtest.cover"
 	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
-	"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
+		$(_TEST_PLATFORM_CMD) \
-		-test.coverprofile "$(ARTIFACTDIR)/platformtest.cover" \
-		-test.v \
-		__DEVEL--i-heard-you-like-tests \
 		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
 		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
 		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
@@ -178,10 +193,11 @@ cover-merge: $(ARTIFACTDIR)
 cover-html: cover-merge
 	$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
 
-test-full:
+cover-full:
 	test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
 	$(MAKE) test-go COVER=1
-	$(MAKE) test-platform
+	$(MAKE) cover-platform-bin
+	$(MAKE) cover-platform
 	$(MAKE) cover-html
 
 ##################### DEV TARGETS #####################

README.md

@@ -21,7 +21,7 @@ zrepl is a one-stop ZFS backup & replication solution.
 
 ## Feature Requests
 
-1. Does you feature request require default values / some kind of configuration?
+1. Does your feature request require default values / some kind of configuration?
    If so, think of an expressive configuration example.
 2. Think of at least one use case that generalizes from your concrete application.
 3. Open an issue on GitHub with example conf & use case attached.

daemon/job/active.go

@@ -34,9 +34,10 @@ type ActiveSide struct {
 
 	prunerFactory *pruner.PrunerFactory
 
-	promRepStateSecs    *prometheus.HistogramVec // labels: state
+	promRepStateSecs      *prometheus.HistogramVec // labels: state
-	promPruneSecs       *prometheus.HistogramVec // labels: prune_side
+	promPruneSecs         *prometheus.HistogramVec // labels: prune_side
-	promBytesReplicated *prometheus.CounterVec   // labels: filesystem
+	promBytesReplicated   *prometheus.CounterVec   // labels: filesystem
+	promReplicationErrors prometheus.Gauge
 
 	tasksMtx sync.Mutex
 	tasks    activeSideTasks
@@ -299,6 +300,14 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (
 		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"filesystem"})
 
+	j.promReplicationErrors = prometheus.NewGauge(prometheus.GaugeOpts{
+		Namespace:   "zrepl",
+		Subsystem:   "replication",
+		Name:        "filesystem_errors",
+		Help:        "number of filesystems that failed replication in the latest replication attempt, or -1 if the job failed before enumerating the filesystems",
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
+	})
+
 	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot build client")
@@ -323,6 +332,7 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
 	registerer.MustRegister(j.promRepStateSecs)
 	registerer.MustRegister(j.promPruneSecs)
 	registerer.MustRegister(j.promBytesReplicated)
+	registerer.MustRegister(j.promReplicationErrors)
 }
 
 func (j *ActiveSide) Name() string { return j.name.String() }
@@ -455,6 +465,10 @@ func (j *ActiveSide) do(ctx context.Context) {
 		GetLogger(ctx).Info("start replication")
 		repWait(true) // wait blocking
 		repCancel()   // always cancel to free up context resources
+
+		replicationReport := j.tasks.replicationReport()
+		j.promReplicationErrors.Set(float64(replicationReport.GetFailedFilesystemsCountInLatestAttempt()))
+
 		endSpan()
 	}
 

dist/grafana/grafana-prometheus-zrepl.json

@@ -69,6 +69,139 @@
             "title": "Panel Title",
             "type": "text"
         },
+        {
+            "cacheTimeout": null,
+            "colorBackground": true,
+            "colorPostfix": false,
+            "colorPrefix": false,
+            "colorValue": false,
+            "colors": [
+                "#bf1b00",
+                "#508642",
+                "#bf1b00"
+            ],
+            "datasource": "${DS_PROMETHEUS}",
+            "description": "Number of filesystems that failed replications",
+            "format": "none",
+            "gauge": {
+                "maxValue": 100,
+                "minValue": 0,
+                "show": false,
+                "thresholdLabels": false,
+                "thresholdMarkers": true
+            },
+            "gridPos": {
+                "h": 3,
+                "w": 24,
+                "x": 0,
+                "y": 10
+            },
+            "id": 50,
+            "interval": null,
+            "links": [],
+            "mappingType": 1,
+            "mappingTypes": [
+                {
+                    "name": "value to text",
+                    "value": 1
+                },
+                {
+                    "name": "range to text",
+                    "value": 2
+                }
+            ],
+            "maxDataPoints": 100,
+            "nullPointMode": "connected",
+            "nullText": null,
+            "postfix": "",
+            "postfixFontSize": "50%",
+            "prefix": "",
+            "prefixFontSize": "50%",
+            "rangeMaps": [
+                {
+                    "from": "",
+                    "text": "",
+                    "to": ""
+                }
+            ],
+            "repeat": "zrepl_job_name",
+            "repeatDirection": "h",
+            "scopedVars": {
+                "zrepl_job_name": {
+                    "selected": false,
+                    "text": "desktop_to_homesrv",
+                    "value": "desktop_to_homesrv"
+                }
+            },
+            "sparkline": {
+                "fillColor": "rgba(31, 118, 189, 0.18)",
+                "full": true,
+                "lineColor": "rgb(31, 120, 193)",
+                "show": true
+            },
+            "tableColumn": "__name__",
+            "targets": [
+                {
+                    "expr": "zrepl_replication_filesystem_errors{job=\"$prom_job_name\",zrepl_job=\"$zrepl_job_name\"}",
+                    "format": "time_series",
+                    "groupBy": [
+                        {
+                            "params": [
+                                "$__interval"
+                            ],
+                            "type": "time"
+                        },
+                        {
+                            "params": [
+                                "null"
+                            ],
+                            "type": "fill"
+                        }
+                    ],
+                    "instant": true,
+                    "interval": "",
+                    "intervalFactor": 1,
+                    "legendFormat": "",
+                    "orderByTime": "ASC",
+                    "policy": "default",
+                    "refId": "A",
+                    "resultFormat": "time_series",
+                    "select": [
+                        [
+                            {
+                                "params": [
+                                    "value"
+                                ],
+                                "type": "field"
+                            },
+                            {
+                                "params": [],
+                                "type": "mean"
+                            }
+                        ]
+                    ],
+                    "tags": []
+                }
+            ],
+            "thresholds": "0,1",
+            "title": "Failed replications $zrepl_job_name",
+            "transparent": false,
+            "type": "singlestat",
+            "valueFontSize": "80%",
+            "valueMaps": [
+                {
+                    "op": "=",
+                    "text": "All failed",
+                    "value": "-1"
+                },
+                {
+                    "op": "=",
+                    "text": "All OK",
+                    "value": "0"
+                }
+            ],
+            "valueName": "avg"
+        },
         {
             "aliasColors": {},
             "bars": false,
@@ -87,7 +220,7 @@
                 "h": 4,
                 "w": 12,
                 "x": 0,
-                "y": 10
+                "y": 13
             },
             "hiddenSeries": false,
             "id": 48,
@@ -181,7 +314,7 @@
                 "h": 5,
                 "w": 12,
                 "x": 12,
-                "y": 10
+                "y": 13
             },
             "hiddenSeries": false,
             "id": 44,
@@ -273,7 +406,7 @@
                 "h": 5,
                 "w": 12,
                 "x": 0,
-                "y": 14
+                "y": 17
             },
             "hiddenSeries": false,
             "id": 42,
@@ -373,7 +506,7 @@
                 "h": 4,
                 "w": 12,
                 "x": 12,
-                "y": 15
+                "y": 18
             },
             "hiddenSeries": false,
             "id": 22,
@@ -465,7 +598,7 @@
                 "h": 5,
                 "w": 12,
                 "x": 0,
-                "y": 19
+                "y": 22
             },
             "hiddenSeries": false,
             "id": 33,
@@ -573,7 +706,7 @@
                 "h": 5,
                 "w": 12,
                 "x": 12,
-                "y": 19
+                "y": 22
             },
             "hiddenSeries": false,
             "id": 23,
@@ -665,7 +798,7 @@
                 "h": 5,
                 "w": 12,
                 "x": 0,
-                "y": 24
+                "y": 27
             },
             "hiddenSeries": false,
             "id": 41,
@@ -758,7 +891,7 @@
                 "h": 5,
                 "w": 12,
                 "x": 12,
-                "y": 24
+                "y": 27
             },
             "hiddenSeries": false,
             "id": 47,
@@ -787,7 +920,7 @@
             "stack": false,
             "steppedLine": false,
             "targets": [{
-                "expr": "zrepl_endpoint_send_abstractions_cache_entry_count",
+                "expr": "zrepl_endpoint_abstractions_cache_entry_count",
                 "format": "time_series",
                 "intervalFactor": 1,
                 "refId": "A"
@@ -796,7 +929,7 @@
             "timeFrom": null,
             "timeRegions": [],
             "timeShift": null,
-            "title": "send abstractions cache entry count (should not be zero and not grow unboundedly)",
+            "title": "zfs abstractions cache entry count (should not be zero and not grow unboundedly)",
             "tooltip": {
                 "shared": true,
                 "sort": 0,
@@ -850,7 +983,7 @@
                 "h": 5,
                 "w": 12,
                 "x": 0,
-                "y": 29
+                "y": 32
             },
             "hiddenSeries": false,
             "id": 17,
@@ -943,7 +1076,7 @@
                 "h": 5,
                 "w": 12,
                 "x": 0,
-                "y": 34
+                "y": 37
             },
             "hiddenSeries": false,
             "id": 19,
@@ -1044,6 +1177,33 @@
             "tagsQuery": "",
             "type": "query",
             "useTags": false
+        },
+        {
+            "allValue": null,
+            "current": {
+                "text": "All",
+                "value": [
+                    "$__all"
+                ]
+            },
+            "datasource": "${DS_PROMETHEUS}",
+            "definition": "label_values(zrepl_replication_filesystem_errors{job=\"$prom_job_name\"}, zrepl_job)",
+            "hide": 2,
+            "includeAll": true,
+            "label": "Zrepl Job Name",
+            "multi": true,
+            "name": "zrepl_job_name",
+            "options": [],
+            "query": "label_values(zrepl_replication_filesystem_errors{job=\"$prom_job_name\"}, zrepl_job)",
+            "refresh": 1,
+            "regex": "",
+            "skipUrlSync": false,
+            "sort": 1,
+            "tagValuesQuery": "",
+            "tags": [],
+            "tagsQuery": "",
+            "type": "query",
+            "useTags": false
         }]
     },
     "time": {
@@ -1076,6 +1236,6 @@
     },
     "timezone": "",
     "title": "zrepl 0.3",
-    "uid": "etJuvBmGz",
+    "uid": "etQuvBnGz",
-    "version": 6
+    "version": 7
-}
+}

docs/changelog.rst

@@ -65,6 +65,7 @@ Additional changelog:
     The migration will ensure that only those old-format cursors are destroyed that have been superseeded by new-format cursors.
 
 * |feature| New option ``listen_freebind`` (tcp, tls, prometheus listener)
+* |feature| :issue:`341` Prometheus metric for failing replications + corresponding Grafana panel
 * |feature| :issue:`265` transport/tcp: support for CIDR masks in client IP whitelist
 * |feature| documented subcommand to generate ``bash`` and ``zsh`` completions
 * |feature| :issue:`307` ``chrome://trace`` -compatible activity tracing of zrepl daemon activity

docs/configuration/prune.rst

@@ -84,35 +84,82 @@ Policy ``grid``
         - type: grid
           regex: "^zrepl_.*"
           grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
-                │                │
+                │                │               │
-                └─ one hour interval
+                └─ 1 repetition of a one-hour interval with keep=all
-                                 │
+                                 │               │
-                                 └─ 24 adjacent one-hour intervals
+                                 └─ 24 repetitions of a one-hour interval with keep=1
+                                                 │
+                                                 └─ 6 repetitions of a 30-day interval with keep=1
       ...
 
-The retention grid can be thought of as a time-based sieve:
+The retention grid can be thought of as a time-based sieve that thins out snapshots as they get older.
-The ``grid`` field specifies a list of adjacent time intervals:
+The ``grid`` field specifies a list of adjacent time intervals.
-the left edge of the leftmost (first) interval is the ``creation`` date of the youngest snapshot.
-All intervals to its right describe time intervals further in the past.
-
-Each interval carries a maximum number of snapshots to keep.
-It is specified via ``(keep=N)``, where ``N`` is either ``all`` (all snapshots are kept) or a positive integer.
-The default value is **keep=1**.
-
 The following procedure happens during pruning:
 
 #. The list of snapshots is filtered by the regular expression in ``regex``.
    Only snapshots names that match the regex are considered for this rule, all others are not affected.
-#. The filtered list of snapshots is sorted by ``creation``
+#. The filtered list of snapshots is sorted by ``creation``.
-#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot
+#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot.
-#. A list of buckets is created, one for each interval
+#. A list of buckets is created, one for each interval.
-#. The list of snapshots is split up into the buckets.
+#. The snapshots are placed into the bucket that matches their ``creation`` date.
 #. For each bucket
 
    #. the contained snapshot list is sorted by creation.
    #. snapshots from the list, oldest first, are destroyed until the specified ``keep`` count is reached.
    #. all remaining snapshots on the list are kept.
 
+The syntax to describe the list of time intervals ("buckets") is as follows:
+
+::
+
+     Repeat x Duration (keep=all)
+
+* The **duration** specifies the length of the interval.
+* The **keep** count specifies the number of snapshots that fit into the bucket.
+  It can be either a positive integer or ``all`` (all snapshots are kept).
+* The **repeat** count repeats the bucket definition for the specified number of times.
+
+**Example**:
+
+::
+
+          This grid spec produces the following list of adjacent buckets. For the sake of simplicity,
+          we subject all snapshots to the grid pruning policy by settings `regex: .*`.
+
+          `
+           grid: 1x1h(keep=all) | 2x2h | 1x3h
+           regex: .*
+          `
+
+          0h         1h         2h         3h         4h         5h         6h         7h         8h
+          |          |          |          |          |          |          |          |          |
+          |-Bucket 1-|------Bucket 2-------|-------Bucket 3------|------------Bucket 4------------|
+          | keep=all |       keep=1        |        keep=1       |             keep=1             |
+
+
+
+          Let us consider the following set of snapshots @a-zA-C , taken at an interval of ~15min:
+
+
+          |  a  b  c  d  e  f  g  h  i  j  k  l  m  n  o  p  q  r  s  t  u  v  w  x  y  z  A  B  C |
+
+
+          The `grid` algorithm maps them to their respective buckets:
+
+          Bucket 1: a, b, c
+          Bucket 2: d,e,f,g,h,i,j
+          Bucket 3: k,l,m,n,o,p,q,r
+          Bucket 4: q,r,s,t,u,v,w,x,y,z,A,B,C
+
+          It then applies the per-bucket pruning logic described above which resulting in the
+          following list of remaining snapshots.
+
+          |  a  b  c  d                    k                       s                               |
+
+          Note that it only makes sense to grow (not shorten) the interval duration for buckets
+          further in the past since each bucket acts like a low-pass filter for incoming snapshots
+          and adding a less-low-pass-filter after a low-pass one has no effect.
+
 
 .. _prune-keep-last-n:
 

docs/supporters.rst

@@ -32,6 +32,8 @@ We would like to thank the following people and organizations for supporting zre
 
    <div class="fa fa-code" style="width: 1em;"></div>
 
+* |supporter-std| `@jwittlincohen <https://github.com/jwittlincohen>`_
+* |supporter-std| `Michael D. Schmitt <https://waterbendingscroll.dancingdragons.org>`_
 * |supporter-std| `Hans Schulz <https://github.com/schulzh>`_
 * |supporter-std| Henning Kessler
 * |supporter-std| `John Ramsden <https://github.com/johnramsden>`_

logger/datastructures.go

@@ -82,7 +82,7 @@ func (l Level) String() string {
 	case Error:
 		return "error"
 	default:
-		return string(l)
+		return fmt.Sprintf("unknown level %d", l)
 	}
 }
 

replication/report/replication_report.go

@@ -162,3 +162,30 @@ func (f *FilesystemReport) NextStep() *StepReport {
 func (f *StepReport) IsIncremental() bool {
 	return f.Info.From != ""
 }
+
+// Returns, for the latest replication attempt,
+// 0  if there have not been any replication attempts,
+// -1 if the replication failed while enumerating file systems
+// N  if N filesystems could not not be replicated successfully
+func (r *Report) GetFailedFilesystemsCountInLatestAttempt() int {
+
+	if len(r.Attempts) == 0 {
+		return 0
+	}
+
+	a := r.Attempts[len(r.Attempts)-1]
+	switch a.State {
+	case AttemptPlanningError:
+		return -1
+	case AttemptFanOutError:
+		var count int
+		for _, f := range a.Filesystems {
+			if f.Error() != nil {
+				count++
+			}
+		}
+		return count
+	default:
+		return 0
+	}
+}

rpc/dataconn/dataconn_client.go

@@ -66,7 +66,7 @@ type ProtocolError struct {
 }
 
 func (e *ProtocolError) Error() string {
-	return fmt.Sprintf("protocol error: %s", e)
+	return fmt.Sprintf("protocol error: %s", e.cause)
 }
 
 func (c *Client) recv(ctx context.Context, conn *stream.Conn, res proto.Message) error {