Compare commits

..

8 Commits

Author SHA1 Message Date
Christian Schwarz e239d6f633 build: make platformtest-* usable
* use plain `go build` instead of the `go test` + `func TestMain` hack
* add target cover-platform and use that to generate coverage reports
2020-08-21 23:01:35 +02:00
InsanePrawn 0bbe2befce docs: prune: add prune interval visualisation
fixes #122

Co-Authored-By: Christian Schwarz <me@cschwarz.com>

Signed-off-by: InsanePrawn <insane.prawny@gmail.com>
2020-08-21 22:05:05 +02:00
InsanePrawn fa4e048169 readme: fix typo
Signed-off-by: InsanePrawn <insane.prawny@gmail.com>
2020-08-21 22:05:05 +02:00
Christian Schwarz 4f9f21f7f2 logger: fix go-1.15-discovered conversion from int to string
logger/datastructures.go:85:10: conversion from Level (int) to string yields a string of one rune
2020-08-12 21:44:02 +02:00
Christian Schwarz 480176ba2d rpc/dataconn: fix go1.15-discovered recursive Error() method impl
rpc/dataconn/dataconn_client.go:69:9: Sprintf format %s with arg e causes recursive Error method call
2020-08-12 21:41:31 +02:00
Christian Schwarz 1190c0f6d2 docs: supporters: update 2020-08-12 21:38:23 +02:00
Christian Schwarz 720a284db5 dist/grafana: fix endpoint abstractions cache metric panel
fixup of 30cdc14
2020-08-04 01:36:31 +02:00
Hans Schulz 83fdffbcef replication: prometheus metric for number of failed replications in last attempt
- package replication: metric
- Grafana panel
- wiring
- changelog

Signed-off-by: Christian Schwarz <me@cschwarz.com>

closes #341
2020-08-04 01:19:44 +02:00
10 changed files with 314 additions and 47 deletions
+25 -9
View File
@@ -147,11 +147,26 @@ zrepl-bin:
generate-platform-test-list:
$(GO_BUILD) -o $(ARTIFACTDIR)/generate-platform-test-list ./platformtest/tests/gen
test-platform-bin:
COVER_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-cover-$(ZREPL_TARGET_TUPLE)
cover-platform-bin:
$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) \
-c -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
-c -o "$(COVER_PLATFORM_BIN_PATH)" \
-covermode=atomic -cover -coverpkg github.com/zrepl/zrepl/... \
./platformtest/harness
cover-platform:
# do not track dependency on cover-platform-bin to allow build of binary outside of test VM
export _TEST_PLATFORM_CMD="$(COVER_PLATFORM_BIN_PATH) \
-test.coverprofile \"$(ARTIFACTDIR)/platformtest.cover\" \
-test.v \
__DEVEL--i-heard-you-like-tests"; \
$(MAKE) _test-or-cover-platform-impl
TEST_PLATFORM_BIN_PATH := $(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)
test-platform-bin:
$(GO_BUILD) -o "$(TEST_PLATFORM_BIN_PATH)" ./platformtest/harness
test-platform:
export _TEST_PLATFORM_CMD="\"$(TEST_PLATFORM_BIN_PATH)\""; \
$(MAKE) _test-or-cover-platform-impl
ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
@@ -159,14 +174,14 @@ ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
ZREPL_PLATFORMTEST_ARGS :=
test-platform: $(ARTIFACTDIR) # do not track dependency on test-platform-bin to allow build of platformtest outside of test VM
_test-or-cover-platform-impl: $(ARTIFACTDIR)
ifndef _TEST_PLATFORM_CMD
$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
endif
rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
rm -f "$(ARTIFACTDIR)/platformtest.cover"
platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
-test.coverprofile "$(ARTIFACTDIR)/platformtest.cover" \
-test.v \
__DEVEL--i-heard-you-like-tests \
$(_TEST_PLATFORM_CMD) \
-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
@@ -178,10 +193,11 @@ cover-merge: $(ARTIFACTDIR)
cover-html: cover-merge
$(GO) tool cover -html "$(ARTIFACTDIR)/merged.cover" -o "$(ARTIFACTDIR)/merged.cover.html"
test-full:
cover-full:
test "$$(id -u)" = "0" || echo "MUST RUN AS ROOT" 1>&2
$(MAKE) test-go COVER=1
$(MAKE) test-platform
$(MAKE) cover-platform-bin
$(MAKE) cover-platform
$(MAKE) cover-html
##################### DEV TARGETS #####################
+1 -1
View File
@@ -21,7 +21,7 @@ zrepl is a one-stop ZFS backup & replication solution.
## Feature Requests
1. Does you feature request require default values / some kind of configuration?
1. Does your feature request require default values / some kind of configuration?
If so, think of an expressive configuration example.
2. Think of at least one use case that generalizes from your concrete application.
3. Open an issue on GitHub with example conf & use case attached.
+17 -3
View File
@@ -34,9 +34,10 @@ type ActiveSide struct {
prunerFactory *pruner.PrunerFactory
promRepStateSecs *prometheus.HistogramVec // labels: state
promPruneSecs *prometheus.HistogramVec // labels: prune_side
promBytesReplicated *prometheus.CounterVec // labels: filesystem
promRepStateSecs *prometheus.HistogramVec // labels: state
promPruneSecs *prometheus.HistogramVec // labels: prune_side
promBytesReplicated *prometheus.CounterVec // labels: filesystem
promReplicationErrors prometheus.Gauge
tasksMtx sync.Mutex
tasks activeSideTasks
@@ -299,6 +300,14 @@ func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
}, []string{"filesystem"})
j.promReplicationErrors = prometheus.NewGauge(prometheus.GaugeOpts{
Namespace: "zrepl",
Subsystem: "replication",
Name: "filesystem_errors",
Help: "number of filesystems that failed replication in the latest replication attempt, or -1 if the job failed before enumerating the filesystems",
ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
})
j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
if err != nil {
return nil, errors.Wrap(err, "cannot build client")
@@ -323,6 +332,7 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
registerer.MustRegister(j.promRepStateSecs)
registerer.MustRegister(j.promPruneSecs)
registerer.MustRegister(j.promBytesReplicated)
registerer.MustRegister(j.promReplicationErrors)
}
func (j *ActiveSide) Name() string { return j.name.String() }
@@ -455,6 +465,10 @@ func (j *ActiveSide) do(ctx context.Context) {
GetLogger(ctx).Info("start replication")
repWait(true) // wait blocking
repCancel() // always cancel to free up context resources
replicationReport := j.tasks.replicationReport()
j.promReplicationErrors.Set(float64(replicationReport.GetFailedFilesystemsCountInLatestAttempt()))
endSpan()
}
+175 -15
View File
@@ -69,6 +69,139 @@
"title": "Panel Title",
"type": "text"
},
{
"cacheTimeout": null,
"colorBackground": true,
"colorPostfix": false,
"colorPrefix": false,
"colorValue": false,
"colors": [
"#bf1b00",
"#508642",
"#bf1b00"
],
"datasource": "${DS_PROMETHEUS}",
"description": "Number of filesystems that failed replications",
"format": "none",
"gauge": {
"maxValue": 100,
"minValue": 0,
"show": false,
"thresholdLabels": false,
"thresholdMarkers": true
},
"gridPos": {
"h": 3,
"w": 24,
"x": 0,
"y": 10
},
"id": 50,
"interval": null,
"links": [],
"mappingType": 1,
"mappingTypes": [
{
"name": "value to text",
"value": 1
},
{
"name": "range to text",
"value": 2
}
],
"maxDataPoints": 100,
"nullPointMode": "connected",
"nullText": null,
"postfix": "",
"postfixFontSize": "50%",
"prefix": "",
"prefixFontSize": "50%",
"rangeMaps": [
{
"from": "",
"text": "",
"to": ""
}
],
"repeat": "zrepl_job_name",
"repeatDirection": "h",
"scopedVars": {
"zrepl_job_name": {
"selected": false,
"text": "desktop_to_homesrv",
"value": "desktop_to_homesrv"
}
},
"sparkline": {
"fillColor": "rgba(31, 118, 189, 0.18)",
"full": true,
"lineColor": "rgb(31, 120, 193)",
"show": true
},
"tableColumn": "__name__",
"targets": [
{
"expr": "zrepl_replication_filesystem_errors{job=\"$prom_job_name\",zrepl_job=\"$zrepl_job_name\"}",
"format": "time_series",
"groupBy": [
{
"params": [
"$__interval"
],
"type": "time"
},
{
"params": [
"null"
],
"type": "fill"
}
],
"instant": true,
"interval": "",
"intervalFactor": 1,
"legendFormat": "",
"orderByTime": "ASC",
"policy": "default",
"refId": "A",
"resultFormat": "time_series",
"select": [
[
{
"params": [
"value"
],
"type": "field"
},
{
"params": [],
"type": "mean"
}
]
],
"tags": []
}
],
"thresholds": "0,1",
"title": "Failed replications $zrepl_job_name",
"transparent": false,
"type": "singlestat",
"valueFontSize": "80%",
"valueMaps": [
{
"op": "=",
"text": "All failed",
"value": "-1"
},
{
"op": "=",
"text": "All OK",
"value": "0"
}
],
"valueName": "avg"
},
{
"aliasColors": {},
"bars": false,
@@ -87,7 +220,7 @@
"h": 4,
"w": 12,
"x": 0,
"y": 10
"y": 13
},
"hiddenSeries": false,
"id": 48,
@@ -181,7 +314,7 @@
"h": 5,
"w": 12,
"x": 12,
"y": 10
"y": 13
},
"hiddenSeries": false,
"id": 44,
@@ -273,7 +406,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 14
"y": 17
},
"hiddenSeries": false,
"id": 42,
@@ -373,7 +506,7 @@
"h": 4,
"w": 12,
"x": 12,
"y": 15
"y": 18
},
"hiddenSeries": false,
"id": 22,
@@ -465,7 +598,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 19
"y": 22
},
"hiddenSeries": false,
"id": 33,
@@ -573,7 +706,7 @@
"h": 5,
"w": 12,
"x": 12,
"y": 19
"y": 22
},
"hiddenSeries": false,
"id": 23,
@@ -665,7 +798,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 24
"y": 27
},
"hiddenSeries": false,
"id": 41,
@@ -758,7 +891,7 @@
"h": 5,
"w": 12,
"x": 12,
"y": 24
"y": 27
},
"hiddenSeries": false,
"id": 47,
@@ -787,7 +920,7 @@
"stack": false,
"steppedLine": false,
"targets": [{
"expr": "zrepl_endpoint_send_abstractions_cache_entry_count",
"expr": "zrepl_endpoint_abstractions_cache_entry_count",
"format": "time_series",
"intervalFactor": 1,
"refId": "A"
@@ -796,7 +929,7 @@
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "send abstractions cache entry count (should not be zero and not grow unboundedly)",
"title": "zfs abstractions cache entry count (should not be zero and not grow unboundedly)",
"tooltip": {
"shared": true,
"sort": 0,
@@ -850,7 +983,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 29
"y": 32
},
"hiddenSeries": false,
"id": 17,
@@ -943,7 +1076,7 @@
"h": 5,
"w": 12,
"x": 0,
"y": 34
"y": 37
},
"hiddenSeries": false,
"id": 19,
@@ -1044,6 +1177,33 @@
"tagsQuery": "",
"type": "query",
"useTags": false
},
{
"allValue": null,
"current": {
"text": "All",
"value": [
"$__all"
]
},
"datasource": "${DS_PROMETHEUS}",
"definition": "label_values(zrepl_replication_filesystem_errors{job=\"$prom_job_name\"}, zrepl_job)",
"hide": 2,
"includeAll": true,
"label": "Zrepl Job Name",
"multi": true,
"name": "zrepl_job_name",
"options": [],
"query": "label_values(zrepl_replication_filesystem_errors{job=\"$prom_job_name\"}, zrepl_job)",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 1,
"tagValuesQuery": "",
"tags": [],
"tagsQuery": "",
"type": "query",
"useTags": false
}]
},
"time": {
@@ -1076,6 +1236,6 @@
},
"timezone": "",
"title": "zrepl 0.3",
"uid": "etJuvBmGz",
"version": 6
}
"uid": "etQuvBnGz",
"version": 7
}
+1
View File
@@ -65,6 +65,7 @@ Additional changelog:
The migration will ensure that only those old-format cursors are destroyed that have been superseeded by new-format cursors.
* |feature| New option ``listen_freebind`` (tcp, tls, prometheus listener)
* |feature| :issue:`341` Prometheus metric for failing replications + corresponding Grafana panel
* |feature| :issue:`265` transport/tcp: support for CIDR masks in client IP whitelist
* |feature| documented subcommand to generate ``bash`` and ``zsh`` completions
* |feature| :issue:`307` ``chrome://trace`` -compatible activity tracing of zrepl daemon activity
+64 -17
View File
@@ -84,35 +84,82 @@ Policy ``grid``
- type: grid
regex: "^zrepl_.*"
grid: 1x1h(keep=all) | 24x1h | 35x1d | 6x30d
│ │
└─ one hour interval
└─ 24 adjacent one-hour intervals
│ │
└─ 1 repetition of a one-hour interval with keep=all
└─ 24 repetitions of a one-hour interval with keep=1
└─ 6 repetitions of a 30-day interval with keep=1
...
The retention grid can be thought of as a time-based sieve:
The ``grid`` field specifies a list of adjacent time intervals:
the left edge of the leftmost (first) interval is the ``creation`` date of the youngest snapshot.
All intervals to its right describe time intervals further in the past.
Each interval carries a maximum number of snapshots to keep.
It is specified via ``(keep=N)``, where ``N`` is either ``all`` (all snapshots are kept) or a positive integer.
The default value is **keep=1**.
The retention grid can be thought of as a time-based sieve that thins out snapshots as they get older.
The ``grid`` field specifies a list of adjacent time intervals.
The following procedure happens during pruning:
#. The list of snapshots is filtered by the regular expression in ``regex``.
Only snapshots names that match the regex are considered for this rule, all others are not affected.
#. The filtered list of snapshots is sorted by ``creation``
#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot
#. A list of buckets is created, one for each interval
#. The list of snapshots is split up into the buckets.
#. The filtered list of snapshots is sorted by ``creation``.
#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot.
#. A list of buckets is created, one for each interval.
#. The snapshots are placed into the bucket that matches their ``creation`` date.
#. For each bucket
#. the contained snapshot list is sorted by creation.
#. snapshots from the list, oldest first, are destroyed until the specified ``keep`` count is reached.
#. all remaining snapshots on the list are kept.
The syntax to describe the list of time intervals ("buckets") is as follows:
::
Repeat x Duration (keep=all)
* The **duration** specifies the length of the interval.
* The **keep** count specifies the number of snapshots that fit into the bucket.
It can be either a positive integer or ``all`` (all snapshots are kept).
* The **repeat** count repeats the bucket definition for the specified number of times.
**Example**:
::
This grid spec produces the following list of adjacent buckets. For the sake of simplicity,
we subject all snapshots to the grid pruning policy by settings `regex: .*`.
`
grid: 1x1h(keep=all) | 2x2h | 1x3h
regex: .*
`
0h 1h 2h 3h 4h 5h 6h 7h 8h
| | | | | | | | |
|-Bucket 1-|------Bucket 2-------|-------Bucket 3------|------------Bucket 4------------|
| keep=all | keep=1 | keep=1 | keep=1 |
Let us consider the following set of snapshots @a-zA-C , taken at an interval of ~15min:
| a b c d e f g h i j k l m n o p q r s t u v w x y z A B C |
The `grid` algorithm maps them to their respective buckets:
Bucket 1: a, b, c
Bucket 2: d,e,f,g,h,i,j
Bucket 3: k,l,m,n,o,p,q,r
Bucket 4: q,r,s,t,u,v,w,x,y,z,A,B,C
It then applies the per-bucket pruning logic described above which resulting in the
following list of remaining snapshots.
| a b c d k s |
Note that it only makes sense to grow (not shorten) the interval duration for buckets
further in the past since each bucket acts like a low-pass filter for incoming snapshots
and adding a less-low-pass-filter after a low-pass one has no effect.
.. _prune-keep-last-n:
+2
View File
@@ -32,6 +32,8 @@ We would like to thank the following people and organizations for supporting zre
<div class="fa fa-code" style="width: 1em;"></div>
* |supporter-std| `@jwittlincohen <https://github.com/jwittlincohen>`_
* |supporter-std| `Michael D. Schmitt <https://waterbendingscroll.dancingdragons.org>`_
* |supporter-std| `Hans Schulz <https://github.com/schulzh>`_
* |supporter-std| Henning Kessler
* |supporter-std| `John Ramsden <https://github.com/johnramsden>`_
+1 -1
View File
@@ -82,7 +82,7 @@ func (l Level) String() string {
case Error:
return "error"
default:
return string(l)
return fmt.Sprintf("unknown level %d", l)
}
}
+27
View File
@@ -162,3 +162,30 @@ func (f *FilesystemReport) NextStep() *StepReport {
func (f *StepReport) IsIncremental() bool {
return f.Info.From != ""
}
// Returns, for the latest replication attempt,
// 0 if there have not been any replication attempts,
// -1 if the replication failed while enumerating file systems
// N if N filesystems could not not be replicated successfully
func (r *Report) GetFailedFilesystemsCountInLatestAttempt() int {
if len(r.Attempts) == 0 {
return 0
}
a := r.Attempts[len(r.Attempts)-1]
switch a.State {
case AttemptPlanningError:
return -1
case AttemptFanOutError:
var count int
for _, f := range a.Filesystems {
if f.Error() != nil {
count++
}
}
return count
default:
return 0
}
}
+1 -1
View File
@@ -66,7 +66,7 @@ type ProtocolError struct {
}
func (e *ProtocolError) Error() string {
return fmt.Sprintf("protocol error: %s", e)
return fmt.Sprintf("protocol error: %s", e.cause)
}
func (c *Client) recv(ctx context.Context, conn *stream.Conn, res proto.Message) error {