doc: fixup 0.3 changelog 05f1237a6d

docs/publish.sh: only render latest (patch+rc) version for each (major,minor) versio.
docs: GitHub Sponsors link
2020-06-14 18:29:37 +02:00 · 2020-06-14 18:24:20 +02:00 · 2020-06-14 15:26:05 +02:00 · 2020-06-14 15:26:05 +02:00 · 2020-06-14 15:26:05 +02:00 · 2020-06-14 15:26:05 +02:00
33 changed files with 1712 additions and 864 deletions
@@ -59,7 +59,6 @@ jobs:
      - run: ./lazy.sh devsetup

      - run: make zrepl-bin
-      - run: make test-platform-bin
      - run: make vet
      - run: make lint
      - run: make release
@@ -86,7 +85,6 @@ jobs:
            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
            
            # Upload artifacts
-            mkdir -p artifacts/release
            echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
            mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"

@@ -100,30 +98,6 @@ jobs:
              -X POST \
              -d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'

-      - run:
-          shell: /bin/bash -eo pipefail
-          when: always
-          command: |
-            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
-              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
-              exit 0
-            fi
-            set -u # from now on
-
-            # Trigger Platform Tests
-            # ZREPL_BOT_GIT_PRWN_EU_ACCESS_TOKEN from secrets
-            ## Sync this commit to the GitLab
-            git remote add prwn-gitlab "https://zrepl-bot:$ZREPL_BOT_GIT_PRWN_EU_ACCESS_TOKEN@git.prwn.eu/problame/zrepl.git"
-            REMOTE_REF=circleci/$CIRCLE_BRANCH/$CIRCLE_BUILD_NUM
-            git push -f prwn-gitlab "HEAD:$REMOTE_REF"
-            ## Trigger Tests
-            curl -v --request POST \
-              --header "PRIVATE-TOKEN: $ZREPL_BOT_GIT_PRWN_EU_ACCESS_TOKEN" \
-              --header "Content-Type: application/json" \
-              --data '{ "ref":"'"$REMOTE_REF"'", "variables": [ {"key":"ZREPL_MAIN_REPO_COMMIT", "value": "'"$CIRCLE_SHA1"'" }, { "key":"ZREPL_CI_BUILDJOB", "value": "'"$CIRCLE_JOB"'" } ] } ' \
-              "https://git.prwn.eu/api/v4/projects/13/pipeline"
-
-
      - run:
          shell: /bin/bash -euo pipefail
          command: |
@@ -1,3 +1,4 @@
+github: problame
 patreon: zrepl
 liberapay: zrepl
 custom: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
@@ -1,17 +0,0 @@
-build-and-run-platformtests:
-  variables:
-    GIT_STRATEGY: none
-  script:
-    - mkdir artifacts
-    # platformtests are installed by the CI environment
-    - /opt/platformtest-linux-amd64
-      -test.coverprofile artifacts/platformtest.cover
-      -test.v
-      __DEVEL--i-heard-you-like-tests
-      -poolname zreplplatformtest
-      -imagepath /tmp/zreplplatformtest.pool.img
-      -mountpoint /tmp/zreplplatformtest.pool
-  artifacts:
-    paths:
-      - artifacts
-  image: "images:ubuntu/focal/cloud"
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-cd "$(dirname "${BASH_SOURCE[0]}")"
-
-commit="$1"
-build="$2"
-src="$3"
-dst="$4"
-
-[ -n "$commit" -a -n "$build" -a -n "$src" -a -n "$dst" ] || ( echo "arguments must not be empty"; exit 1 )
-
-aws="aws --endpoint-url https://minio.cschwarz.com --no-sign-request"
-
-
-path="s3://zrepl-ci-artifacts/$commit/$build/$src"
-$aws s3 cp "$path" "$dst"
@@ -2,7 +2,8 @@
 [![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
 [![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
 [![Donate via Patreon](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow)](https://www.patreon.com/zrepl)
-[![Donate via Liberapay](https://img.shields.io/liberapay/receives/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
+[![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame)
+[![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
 [![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
 [![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)

@@ -0,0 +1,85 @@
+# This config serves as an example for a local zrepl installation that
+# backups the entire zpool `system` to `backuppool/zrepl/sink`
+#
+# The requirements covered by this setup are described in the zrepl documentation's
+# quick start section which inlines this example.
+#
+# CUSTOMIZATIONS YOU WILL LIKELY WANT TO APPLY:
+# - adjust the name of the production pool `system` in the `filesystems` filter of jobs `snapjob` and `push_to_derive`
+# - adjust the name of the backup pool `backuppool` in the `backuppool_sink` job
+# - adjust the occurences of `myhostname` to the name of the system you are backing up (cannot be easily changed once you start replicating)
+# - make sure the `zrepl_` prefix is not being used by any other zfs tools you might have installed (it likely isn't)
+
+jobs:
+
+# this job takes care of snapshot creation + pruning
+- name: snapjob
+  type: snap
+  filesystems: {
+      "system<": true,
+  }
+  # create snapshots with prefix `zrepl_` every 15 minutes
+  snapshotting:
+    type: periodic
+    interval: 15m
+    prefix: zrepl_
+  pruning:
+    keep:
+    # fade-out scheme for snapshots starting with `zrepl_`
+    # - keep all created in the last hour
+    # - then destroy snapshots such that we keep 24 each 1 hour apart
+    # - then destroy snapshots such that we keep 14 each 1 day apart
+    # - then destroy all older snapshots
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 14x1d
+      regex: "^zrepl_.*"
+    # keep all snapshots that don't have the `zrepl_` prefix
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job pushes to the local sink defined in job `backuppool_sink`.
+# We trigger replication manually from the command line / udev rules using
+#  `zrepl signal wakeup push_to_drive`
+- type: push
+  name: push_to_drive
+  connect:
+    type: local
+    listener_name: backuppool_sink
+    client_identity: myhostname
+  filesystems: {
+      "system<": true
+  }
+  send:
+    encrypted: true
+    # disable incremental step holds so that
+    # - we can yank out the backup drive during replication
+    # - thereby sacrificing resumability
+    # - in exchange for the replicating snapshot not sticking around until we reconnect the backup drive
+    step_holds:
+      disable_incremental: true
+  snapshotting:
+    type: manual
+  pruning:
+    # no-op prune rule on sender (keep all snapshots), job `snapshot` takes care of this
+    keep_sender:
+    - type: regex
+      regex: ".*"
+    # retain 
+    keep_receiver:
+    # longer retention on the backup drive, we have more space there
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 360x1d
+      regex: "^zrepl_.*"
+    # retain all non-zrepl snapshots on the backup drive
+    - type: regex
+      negate: true
+      regex: "^zrepl_.*"
+
+# This job receives from job `push_to_drive` into `backuppool/zrepl/sink/myhostname`
+- type: sink
+  name: backuppool_sink
+  root_fs: "backuppool/zrepl/sink"
+  serve:
+    type: local
+    listener_name: backuppool_sink
@@ -0,0 +1,12 @@
+jobs:
+- name: sink
+  type: sink
+  serve:
+      type: tls
+      listen: ":8888"
+      ca: "/etc/zrepl/prod.crt"
+      cert: "/etc/zrepl/backups.crt"
+      key: "/etc/zrepl/backups.key"
+      client_cns:
+        - "prod"
+  root_fs: "storage/zrepl/sink"
@@ -0,0 +1,28 @@
+jobs:
+- name: prod_to_backups
+  type: push
+  connect:
+    type: tls
+    address: "backups.example.com:8888"
+    ca: /etc/zrepl/backups.crt
+    cert: /etc/zrepl/prod.crt
+    key:  /etc/zrepl/prod.key
+    server_cn: "backups"
+  filesystems: {
+    "zroot/var/db": true,
+    "zroot/usr/home<": true,
+    "zroot/usr/home/paranoid": false
+  }
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+  pruning:
+    keep_sender:
+    - type: not_replicated
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: grid
+      grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d
+      regex: "^zrepl_"
@@ -95,6 +95,7 @@ func Run(ctx context.Context, conf *config.Config) error {
 	}

 	// register global (=non job-local) metrics
+	version.PrometheusRegister(prometheus.DefaultRegisterer)
 	zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
 	trace.RegisterMetrics(prometheus.DefaultRegisterer)
 	endpoint.RegisterMetrics(prometheus.DefaultRegisterer)
@@ -1,595 +0,0 @@
-{
-  "annotations": {
-    "list": [
-      {
-        "$$hashKey": "object:3351",
-        "builtIn": 1,
-        "datasource": "-- Grafana --",
-        "enable": true,
-        "hide": true,
-        "iconColor": "rgba(0, 211, 255, 1)",
-        "name": "Annotations & Alerts",
-        "type": "dashboard"
-      }
-    ]
-  },
-  "editable": true,
-  "gnetId": null,
-  "graphTooltip": 1,
-  "id": 7,
-  "iteration": 1552746418826,
-  "links": [],
-  "panels": [
-    {
-      "content": "# zrepl Prometheus Metrics\n\nzrepl exposes Prometheus metrics and ships with this Grafana dashboard.\nThe exported metrics are suitable for health checks:\n\n* The log should generally be warning & error-free\n  * The `Log Messages that require attention` graph visualizes log message counts indicating problems.\n* The number of goroutines should not grow unboundedly over time.\n  * During replication, the number of goroutines can be way higher than during idle time.\n  * If the goroutine count grows with each replication, there is clearly a goroutine leak. Please open a bug report.\n* The sys memory consumption should not grow unboundedly over time.\n  * Note that the Go runtime pre-allocates some of its heap from the OS.\n  * zrepl actually uses much less memory than allocated from the OS.\n  * Since Go 1.11, Go pre-allocates more aggressively.\n* Monitor that some data is replicated, although that metric does not guarantee that replication was successful.\n\n**In general, note that the exported metrics are not stable unless declared otherwise.**",
-      "gridPos": {
-        "h": 9,
-        "w": 24,
-        "x": 0,
-        "y": 0
-      },
-      "id": 35,
-      "mode": "markdown",
-      "title": "Panel Title",
-      "type": "text"
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 9
-      },
-      "id": 15,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": true,
-      "steppedLine": false,
-      "targets": [
-        {
-          "$$hashKey": "object:3436",
-          "expr": "up{job='$prom_job_name'}",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "zrepl Instances Up",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": "5",
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 9
-      },
-      "id": 22,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "increase(zrepl_daemon_log_entries{job='$prom_job_name',level=~'warn|error'}[$__interval])",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Log Messages that require attention",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 18
-      },
-      "id": 33,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [
-        {
-          "alias": "/replicated bytes in last.*/",
-          "yaxis": 2
-        }
-      ],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "sum(rate(zrepl_replication_bytes_replicated{job='$prom_job_name'}[$__interval])) by (zrepl_job)",
-          "format": "time_series",
-          "hide": false,
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "replication data rate zrepl_job={{zrepl_job}}",
-          "refId": "A"
-        },
-        {
-          "expr": "sum(increase(zrepl_replication_bytes_replicated{job='$prom_job_name'}[10m])) by (zrepl_job)",
-          "format": "time_series",
-          "hide": false,
-          "interval": "",
-          "intervalFactor": 1,
-          "legendFormat": "replicated bytes in last 10min zrepl_job={{zrepl_job}}",
-          "refId": "B"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Replication Data Rate and Volume(integrates last 10min)",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "Bps",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        },
-        {
-          "format": "bytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 18
-      },
-      "id": 23,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "sum(increase(zrepl_daemon_log_entries{job='$prom_job_name',zrepl_job=~\"^[^_].*\"}[$__interval])) by (instance,zrepl_job)",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Log Activity (without internal jobs)",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 0,
-        "y": 27
-      },
-      "id": 17,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "$$hashKey": "object:3535",
-          "expr": "go_memstats_sys_bytes{job='$prom_job_name'}",
-          "format": "time_series",
-          "hide": false,
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "Memory Allocated by the Go runtime from the OS (should not grow unboundedly)",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "bytes",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    },
-    {
-      "aliasColors": {},
-      "bars": false,
-      "dashLength": 10,
-      "dashes": false,
-      "datasource": null,
-      "fill": 1,
-      "gridPos": {
-        "h": 9,
-        "w": 12,
-        "x": 12,
-        "y": 27
-      },
-      "id": 19,
-      "legend": {
-        "avg": false,
-        "current": false,
-        "max": false,
-        "min": false,
-        "show": true,
-        "total": false,
-        "values": false
-      },
-      "lines": true,
-      "linewidth": 1,
-      "links": [],
-      "nullPointMode": "null",
-      "percentage": false,
-      "pointradius": 5,
-      "points": false,
-      "renderer": "flot",
-      "seriesOverrides": [],
-      "spaceLength": 10,
-      "stack": false,
-      "steppedLine": false,
-      "targets": [
-        {
-          "expr": "go_goroutines{job='$prom_job_name'}",
-          "format": "time_series",
-          "intervalFactor": 1,
-          "refId": "A"
-        }
-      ],
-      "thresholds": [],
-      "timeFrom": null,
-      "timeShift": null,
-      "title": "number of goroutines (should not grow unboundedly)",
-      "tooltip": {
-        "shared": true,
-        "sort": 0,
-        "value_type": "individual"
-      },
-      "type": "graph",
-      "xaxis": {
-        "buckets": null,
-        "mode": "time",
-        "name": null,
-        "show": true,
-        "values": []
-      },
-      "yaxes": [
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": "0",
-          "show": true
-        },
-        {
-          "format": "short",
-          "label": null,
-          "logBase": 1,
-          "max": null,
-          "min": null,
-          "show": true
-        }
-      ]
-    }
-  ],
-  "refresh": "1m",
-  "schemaVersion": 16,
-  "style": "dark",
-  "tags": [],
-  "templating": {
-    "list": [
-      {
-        "allValue": null,
-        "current": {
-          "text": "zrepl",
-          "value": "zrepl"
-        },
-        "datasource": "prometheus",
-        "hide": 0,
-        "includeAll": false,
-        "label": "Prometheus Job Name",
-        "multi": false,
-        "name": "prom_job_name",
-        "options": [],
-        "query": "label_values(up, job)",
-        "refresh": 1,
-        "regex": "",
-        "sort": 1,
-        "tagValuesQuery": "",
-        "tags": [],
-        "tagsQuery": "",
-        "type": "query",
-        "useTags": false
-      }
-    ]
-  },
-  "time": {
-    "from": "now-2d",
-    "to": "now"
-  },
-  "timepicker": {
-    "refresh_intervals": [
-      "5s",
-      "10s",
-      "30s",
-      "1m",
-      "5m",
-      "15m",
-      "30m",
-      "1h",
-      "2h",
-      "1d"
-    ],
-    "time_options": [
-      "5m",
-      "15m",
-      "1h",
-      "6h",
-      "12h",
-      "24h",
-      "2d",
-      "7d",
-      "30d"
-    ]
-  },
-  "timezone": "",
-  "title": "zrepl 0.1",
-  "uid": "xTljn4qmk",
-  "version": 6
-}
@@ -51,7 +51,7 @@ Additional changelog:
 * |break| |break_config| **more restrictive job names than in prior zrepl versions**
  Starting with this version, job names are going to be embedded into ZFS holds and bookmark names (see :ref:`here<replication-cursor-and-last-received-hold>` and :ref:`here<step-holds-and-bookmarks>`).
  Therefore you might need to adjust your job names.
-  **Note that jobs** :issue:`cannot be renamed easily<327>` **once you start using zrepl 0.3.**
+  **Note that jobs** :issue:`cannot be renamed easily` **once you start using zrepl 0.3.**
 * |break| |mig| replication cursor representation changed

  * zrepl now manages the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` per job-filesystem tuple instead of a single replication cursor per filesystem.
@@ -62,11 +62,11 @@ Additional changelog:
    The migration will ensure that only those old-format cursors are destroyed that have been superseeded by new-format cursors.

 * |feature| New option ``listen_freebind`` (tcp, tls, prometheus listener)
-* |feature| :issue:`<265>` transport/tcp: support for CIDR masks in client IP whitelist
+* |feature| :issue:`265` transport/tcp: support for CIDR masks in client IP whitelist
 * |feature| documented subcommand to generate ``bash`` and ``zsh`` completions
-* |feature| :issue:`<307>` ``chrome://trace`` -compatible activity tracing of zrepl daemon activity
+* |feature| :issue:`307` ``chrome://trace`` -compatible activity tracing of zrepl daemon activity
 * |feature| logging: trace IDs for better log entry correlation with concurrent replication jobs
-* |feature| experimental environment variable for parallel replication (see :issue:`<306>` )
+* |feature| experimental environment variable for parallel replication (see :issue:`306` )
 * |bugfix| missing logger context vars in control connection handlers
 * |bugfix| improved error messages on ``zfs send`` errors
 * |bugfix| |docs| snapshotting: clarify sync-up behavior and warn about filesystems
@@ -80,7 +80,7 @@ Additional changelog:
 .. NOTE::
   |  zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
   |  You can support maintenance and feature development through one of the following services:
-   |  |Donate via Patreon| |Donate via Liberapay| |Donate via PayPal|
+   |  |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
   |  Note that PayPal processing fees are relatively high for small donations.
   |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.

@@ -12,7 +12,7 @@ The following paths are considered:
 The ``zrepl configcheck`` subcommand can be used to validate the configuration.
 The command will output nothing and exit with zero status code if the configuration is valid.
 The error messages vary in quality and usefulness: please report confusing config errors to the tracking :issue:`155`.
-Full example configs such as in the :ref:`tutorial` or the :sampleconf:`/` directory might also be helpful.
+Full example configs such as in the :ref:`quick-start guides <quickstart-toc>` or the :sampleconf:`/` directory might also be helpful.
 However, copy-pasting examples is no substitute for reading documentation!

 Config File Structure
@@ -36,6 +36,7 @@ Filesystems matched by ``filesystems`` that are not encrypted are not sent and w

 If ``encryption=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.

+.. _job-send-option-step-holds-disable-incremental:

 ``step_holds.disable_incremental`` option
 -----------------------------------------
@@ -44,9 +45,15 @@ The ``step_holds.disable_incremental`` variable controls whether the creation of
 The default value is ``false``.

 Disabling step holds has the disadvantage that steps :ref:`might not be resumable <step-holds-and-bookmarks>` if interrupted.
+Non-resumability means that replication progress is no longer monotonic which might result in a replication setup that never makes progress if mid-step interruptions are too frequent (e.g. frequent network outages).
+
 However, the advantage and :issue:`reason for existence <288>` of this flag is that it allows the pruner to delete snapshots of interrupted replication steps
 which is useful if replication happens so rarely (or fails so frequently) that the amount of disk space exclusively referenced by the step's snapshots becomes intolerable.

+.. NOTE::
+
+   When setting this flag to ``true``, existing step holds for the job will be destroyed on the next replication attempt.
+
 .. _job-recv-options:

 Recv Options
@@ -0,0 +1,76 @@
+#!/usr/bin/env python3
+
+import subprocess
+import re
+
+output = subprocess.run(["git", "tag", "-l"], capture_output=True, check=True, text=True)
+tagRE = re.compile(r"^v(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-rc(?P<rc>\d+))?$")
+class Tag:
+    orig: str
+    major: int
+    minor: int
+    patch: int
+    rc: int
+
+    def __str__(self):
+        return self.orig
+
+    def __repr__(self):
+        return self.orig
+
+tags = []
+for line in output.stdout.split("\n"):
+    m = tagRE.match(line)
+    if not m:
+        continue
+    m = m.groupdict()
+
+    t = Tag()
+    t.orig = line
+    t.major = int(m["major"])   
+    t.minor = int(m["minor"])   
+    t.patch = int(m["patch"])
+    t.rc = int(m["rc"] if m["rc"] is not None else 0)
+
+    tags.append(t)
+
+by_major_minor = {}
+
+for tag in tags:
+    key = (tag.major, tag.minor)
+    l = by_major_minor.get(key, [])
+    l.append(tag)
+    by_major_minor[key] = l
+
+latest_by_major_minor = []
+for (mm, l) in by_major_minor.items():
+    # sort ascending by patch-level (rc's weigh less)
+    l.sort(key=lambda tag: (tag.patch, int(tag.rc == 0), tag.rc))
+    latest_by_major_minor.append(l[-1])
+latest_by_major_minor.sort(key=lambda tag: (tag.major, tag.minor))
+
+# print(by_major_minor)
+# print(latest_by_major_minor)
+
+cmdline = []
+
+for latest_patch in latest_by_major_minor:
+    cmdline.append("--whitelist-tags")
+    cmdline.append(f"^{re.escape(latest_patch.orig)}$")
+
+# we want to render the latest non-rc version as the default page
+# (latest_by_major_minor is already sorted)
+default_version = latest_by_major_minor[-1]
+for tag in reversed(latest_by_major_minor):
+    if tag.rc == 0:
+        default_version = tag
+        break
+
+cmdline.extend(["--root-ref", f"{default_version}"])
+cmdline.extend(["--banner-main-ref", f"{default_version}"])
+cmdline.extend(["--show-banner"])
+cmdline.extend(["--sort", "semver"])
+
+cmdline.extend(["--whitelist-branches", "master"])
+
+print(" ".join(cmdline))
@@ -11,10 +11,12 @@
   :target: https://zrepl.github.io
 .. |Donate via PayPal| image:: https://img.shields.io/badge/donate-paypal-yellow.svg
   :target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
-.. |Donate via Liberapay| image:: https://img.shields.io/liberapay/receives/zrepl.svg?logo=liberapay
+.. |Donate via Liberapay| image:: https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay
   :target: https://liberapay.com/zrepl/donate
 .. |Donate via Patreon| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow
   :target: https://www.patreon.com/zrepl
+.. |Donate via GitHub Sponsors| image:: https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow
+   :target: https://github.com/sponsors/problame
 .. |Twitter| image:: https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social
   :target: https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl

@@ -25,4 +27,8 @@
 .. |recv-options| replace:: :ref:`recv options<job-recv-options>`
 .. |snapshotting-spec| replace:: :ref:`snapshotting specification <job-snapshotting-spec>`
 .. |pruning-spec| replace:: :ref:`pruning specification <prune>`
-.. |filter-spec| replace:: :ref:`filter specification<pattern-filter>`
+.. |filter-spec| replace:: :ref:`filter specification<pattern-filter>`
+
+.. |br| raw:: html
+
+   <br/>
@@ -5,7 +5,7 @@

 .. include:: global.rst.inc

-|GitHub license| |Language: Go| |Twitter| |Donate via Patreon| |Donate via Liberapay| |Donate via PayPal| 
+|GitHub license| |Language: Go| |Twitter| |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|


 zrepl - ZFS replication
@@ -44,7 +44,7 @@ zrepl - ZFS replication
 Getting started
 ~~~~~~~~~~~~~~~

-The :ref:`10 minutes tutorial setup <tutorial>` gives you a first impression.
+The :ref:`10 minute quick-start guides <quickstart-toc>` give you a first impression.

 Main Features
 ~~~~~~~~~~~~~
@@ -125,7 +125,7 @@ Table of Contents
   :maxdepth: 2
   :caption: Contents:

-   tutorial
+   quickstart
   installation
   configuration
   usage
@@ -7,7 +7,7 @@ Installation

 .. TIP::

-    Note: check out the :ref:`tutorial` if you want a first impression of zrepl.
+    Note: check out the :ref:`quick-start guides <quickstart-toc>` if you want a first impression of zrepl.

 .. toctree::

@@ -42,4 +42,4 @@ Either way, all build results are located in the ``artifacts/`` directory.
 .. NOTE::

    It is your job to install the appropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
-    Otherwise, the examples in the :ref:`tutorial` may need to be adjusted.
+    Otherwise, the examples in the :ref:`quick-start guides <quickstart-toc>` may need to be adjusted.
@@ -126,7 +126,7 @@ Modify the ``/usr/local/etc/zrepl/zrepl.yml`` configuration file.

 .. TIP::

-    Note: check out the :ref:`tutorial` for examples of a ``sink`` job.
+    Note: check out the :ref:`quick-start guides <quickstart-toc>` for examples of a ``sink`` job.

 Now ``zrepl`` can be started.

@@ -5,4 +5,4 @@ What next?

 Read the :ref:`configuration chapter<configuration_toc>` and then continue with the :ref:`usage chapter<usage>`.

-**Reminder**: If you want a quick introduction, please read the :ref:`tutorial`.
+**Reminder**: If you want a quick introduction, please read the :ref:`quick-start guides <quickstart-toc>`.
@@ -51,11 +51,10 @@ git rm -rf .
 popd

 echo "building site"
+
 set -e
 sphinx-versioning build \
-   --show-banner \
-   --whitelist-branches '^master$' \
-   --whitelist-tags '(v)?\d+\.[1-9][0-9]*.\d+$' \
+   $(python3 gen-sphinx-versioning-flags.py) \
   docs ./public_git \
   -- -c sphinxconf # older conf.py throw errors because they used
                    # version = subprocess.show_output(["git", "describe"])
@@ -0,0 +1,87 @@
+.. include:: global.rst.inc
+
+.. _quickstart-toc:
+
+***********************
+Quick Start by Use Case
+***********************
+
+The goal of this quick-start guide is to give you an impression of how zrepl can accomodate your use case.
+
+Install zrepl
+=============
+
+Follow the :ref:`OS-specific installation instructions <installation_toc>` and come back here.
+
+Overview Of How zrepl Works
+============================
+
+Check out the :ref:`overview section <job-overview>` to get a rough idea of what you are going to configure in the next step, then come back here.
+
+Configuration Examples
+======================
+
+zrepl is configured through a YAML configuration file in ``/etc/zrepl/zrepl.yml``.
+We have prepared example use cases that show-case typical deployments and different functionality of zrepl.
+We encourage you to read through all of the examples to get an idea of what zrepl has to offer, and how you can mix-and-match configurations for your use case.
+Keep the :ref:`full config documentation <configuration_toc>` handy if a config snippet is unclear.
+
+**Example Use Cases**
+
+.. toctree::
+   :titlesonly:
+
+   quickstart/continuous_server_backup
+   quickstart/backup_to_external_disk
+
+Use ``zrepl configcheck`` to validate your configuration.
+No output indicates that everything is fine.
+
+.. NOTE::
+
+   Please open an issue on GitHub if your use case for zrepl is significantly different from those listed above.
+   Or even better, write it up in the same style as above and open a PR!
+
+.. _quickstart-apply-config:
+
+Apply Configuration Changes
+===========================
+
+We hope that you have found a configuration that fits your use case.
+Use ``zrepl configcheck`` once again to make sure the config is correct (output indicates that everything is fine).
+Then restart the zrepl daemon on all systems involved in the replication, likely using ``service zrepl restart`` or ``systemctl restart zrepl``.
+
+Watch it Work
+=============
+
+Run ``zrepl status`` on the active side of the replication setup to monitor snaphotting, replication and pruning activity.
+To re-trigger replication (snapshots are separate!), use ``zrepl signal wakeup JOBNAME``.
+(refer to the example use case document if you are uncertain which job you want to wake up).
+
+You can also use basic UNIX tools to inspect see what's going on.
+If you like tmux, here is a handy script that works on FreeBSD: ::
+
+    pkg install gnu-watch tmux
+    tmux new -s zrepl -d
+    tmux split-window -t zrepl "tail -f /var/log/messages"
+    tmux split-window -t zrepl "gnu-watch 'zfs list -t snapshot -o name,creation -s creation'"
+    tmux split-window -t zrepl "zrepl status"
+    tmux select-layout -t zrepl tiled
+    tmux attach -t zrepl
+
+The Linux equivalent might look like this: ::
+
+    # make sure tmux is installed & let's assume you use systemd + journald
+    tmux new -s zrepl -d
+    tmux split-window -t zrepl  "journalctl -f -u zrepl.service"
+    tmux split-window -t zrepl "watch 'zfs list -t snapshot -o name,creation -s creation'"
+    tmux split-window -t zrepl "zrepl status"
+    tmux select-layout -t zrepl tiled
+    tmux attach -t zrepl
+
+What Next?
+==========
+
+* Read more about :ref:`configuration format, options & job types <configuration_toc>`
+* Configure :ref:`logging <logging>` \& :ref:`monitoring <monitoring>`.
+
@@ -0,0 +1,36 @@
+.. include:: ../global.rst.inc
+
+.. _quickstart-backup-to-external-disk:
+
+Local Snapshots + Offline Backup to an External Disk
+====================================================
+
+This config example shows how we can use zrepl to make periodic snapshots of our local workstation and back it up to a zpool on an external disk which we occassionally connect.
+
+The local snapshots should be taken every 15 minutes for pain-free recovery from CLI disasters (``rm -rf /`` and the like).
+However, we do not want to keep the snapshots around for very long because our workstation is a little tight on disk space.
+Thus, we only keep one hour worth of high-resolution snapshots, then fade them out to one per hour for a day (24 hours), then one per day for 14 days.
+
+At the end of each work day, we connect our external disk that serves as our workstation's local offline backup.
+We want zrepl to inspect the filesystems and snapshots on the external pool, figure out which snapshots were created since the last time we connected the external disk, and use incremental replication to efficiently mirror our workstation to our backup disk.
+Afterwards, we want to clean up old snapshots on the backup pool: we want to keep all snapshots younger than one hour, 24 for each hor of the first day, then 360 daily backups.
+
+A few additional requirements:
+
+* Snapshot creation and pruning on our workstation should happen in the background, without interaction from our side.
+* However, we want to explicitly trigger replication via the command line.
+* We want to use OpenZFS native encryption to protect our data on the external disk.
+  It is absolutely critical that only encrypted data leaves our workstation.
+  **zrepl should provide an easy config knob for this and prevent replication of unencrypted datasets to the external disk.**
+* We want to be able to put off the backups for more than three weeks, i.e., longer than the lifetime of the automatically created snapshots on our workstation.
+  **zrepl should use bookmarks and holds to achieve this goal**.
+* When we yank out the drive during replication and go on a long vacation, we do *not* want the partially replicated snapshot to stick around as it would hold on to too much disk space over time.
+  Therefore, we want zrepl to deviate from its :ref:`default step-hold behavior <step-holds-and-bookmarks>` and sacrifice resumability, but nonetheless retain the ability to do incremental replication once we return from our vacation.
+  **zrepl should provide an easy config knob to disable step holds for incremental replication**.
+
+The following config snippet implements the setup described above.
+You will likely want to customize some aspects mentioned in the top comment in the file.
+
+.. literalinclude:: ../../config/samples/quickstart_backup_to_external_disk.yml
+
+:ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.
@@ -0,0 +1,97 @@
+.. include:: ../global.rst.inc
+
+.. _quickstart-continuous-replication:
+
+Continuous Backup of a Server
+=============================
+
+This config example shows how we can backup our ZFS-based server to another machine using a zrepl push job.
+
+* Production server ``prod`` with filesystems to back up:
+
+  * ``zroot/var/db``
+  * ``zroot/usr/home`` and all its child filesystems
+  * **except** ``zroot/usr/home/paranoid`` belonging to a user doing backups themselves
+
+* Backup server ``backups`` with
+
+  * Filesystem ``storage/zrepl/sink/prod`` + children dedicated to backups of ``prod``
+
+Our backup solution should fulfill the following requirements:
+
+* Periodically snapshot the filesystems on ``prod`` *every 10 minutes*
+* Incrementally replicate these snapshots to ``storage/zrepl/sink/prod/*`` on ``backups``
+* Keep only very few snapshots on ``prod`` to save disk space
+* Keep a fading history (24 hourly, 30 daily, 6 monthly) of snapshots on ``backups``
+* The network is untrusted - zrepl should use TLS to protect its communication and our data.
+
+Analysis
+--------
+
+We can model this situation as two jobs:
+
+* A **push job** on ``prod``
+
+  * Creates the snapshots
+  * Keeps a short history of local snapshots to enable incremental replication to ``backups``
+  * Connects to the ``zrepl daemon`` process on ``backups``
+  * Pushes snapshots ``backups``
+  * Prunes snapshots on ``backups`` after replication is complete
+
+* A **sink job** on ``backups``
+
+  * Accepts connections & responds to requests from ``prod``
+  * Limits client ``prod`` access to filesystem sub-tree ``storage/zrepl/sink/prod``
+
+Generate TLS Certificates
+-------------------------
+
+We use the :ref:`TLS client authentication transport <transport-tcp+tlsclientauth>` to protect our data on the wire.
+To get things going quickly, we skip setting up a CA and generate two self-signed certificates as described :ref:`here <transport-tcp+tlsclientauth-2machineopenssl>`.
+For convenience, we generate the key pairs on our local machine and distribute them using ssh:
+
+.. code-block:: bash
+   :emphasize-lines: 6,13
+
+   openssl req -x509 -sha256 -nodes \
+      -newkey rsa:4096 \
+      -days 365 \
+      -keyout backups.key \
+      -out backups.crt
+   # ... and use "backups" as Common Name (CN)
+
+   openssl req -x509 -sha256 -nodes \
+      -newkey rsa:4096 \
+      -days 365 \
+      -keyout prod.key \
+      -out prod.crt
+   # ... and use "prod" as Common Name (CN)
+
+   ssh root@backups "mkdir /etc/zrepl"
+   scp  backups.key backups.crt prod.crt root@backups:/etc/zrepl
+
+   ssh root@prod "mkdir /etc/zrepl"
+   scp  prod.key prod.crt backups.crt root@prod:/etc/zrepl
+
+Note that alternative transports exist, e.g. via :ref:`TCP without TLS <transport-tcp>` or :ref:`ssh <transport-ssh+stdinserver>`.
+
+Configure server ``prod``
+-------------------------
+
+We define a **push job** named ``prod_to_backups`` in ``/etc/zrepl/zrepl.yml`` on host ``prod`` :
+
+.. literalinclude:: ../../config/samples/quickstart_continuous_server_backup_sender.yml
+
+.. _tutorial-configure-prod:
+
+Configure server ``backups``
+----------------------------
+
+We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`` on host ``backups`` :
+
+.. literalinclude:: ../../config/samples/quickstart_continuous_server_backup_receiver.yml
+
+Go Back To Quickstart Guide
+---------------------------
+
+:ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.
@@ -2,7 +2,7 @@

 .. _supporters:

-|Donate via Patreon| |Donate via Liberapay| |Donate via PayPal|
+|Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|

 zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
 You can support maintenance and feature development through one of the services listed above.
@@ -1,177 +0,0 @@
-.. include:: global.rst.inc
-
-.. _tutorial:
-
-Tutorial
-========
-
-
-This tutorial shows how zrepl can be used to implement a ZFS-based push backup.
-We assume the following scenario:
-
-* Production server ``prod`` with filesystems to back up:
-
-  * ``zroot/var/db``
-  * ``zroot/usr/home`` and all its child filesystems
-  * **except** ``zroot/usr/home/paranoid`` belonging to a user doing backups themselves
-
-* Backup server ``backups`` with
-
-  * Filesystem ``storage/zrepl/sink/prod`` + children dedicated to backups of ``prod``
-
-Our backup solution should fulfill the following requirements:
-
-* Periodically snapshot the filesystems on ``prod`` *every 10 minutes*
-* Incrementally replicate these snapshots to ``storage/zrepl/sink/prod/*`` on ``backups``
-* Keep only very few snapshots on ``prod`` to save disk space
-* Keep a fading history (24 hourly, 30 daily, 6 monthly) of snapshots on ``backups``
-
-Analysis
--------
-
-We can model this situation as two jobs:
-
-* A **push job** on ``prod``
-
-  * Creates the snapshots
-  * Keeps a short history of local snapshots to enable incremental replication to ``backups``
-  * Connects to the ``zrepl daemon`` process on ``backups``
-  * Pushes snapshots ``backups``
-  * Prunes snapshots on ``backups`` after replication is complete
-
-* A **sink job** on ``backups``
-
-  * Accepts connections & responds to requests from ``prod``
-  * Limits client ``prod`` access to filesystem sub-tree ``storage/zrepl/sink/prod``
-
-Install zrepl
-------------
-
-Follow the :ref:`OS-specific installation instructions <installation_toc>` and come back here.
-
-Generate TLS Certificates
-------------------------
-
-We use the :ref:`TLS client authentication transport <transport-tcp+tlsclientauth>` to protect our data on the wire.
-To get things going quickly, we skip setting up a CA and generate two self-signed certificates as described :ref:`here <transport-tcp+tlsclientauth-2machineopenssl>`.
-For convenience, we generate the key pairs on our local machine and distribute them using ssh:
-
-.. code-block:: bash
-   :emphasize-lines: 6,13
-
-   openssl req -x509 -sha256 -nodes \
-      -newkey rsa:4096 \
-      -days 365 \
-      -keyout backups.key \
-      -out backups.crt
-   # ... and use "backups" as Common Name (CN)
-
-   openssl req -x509 -sha256 -nodes \
-      -newkey rsa:4096 \
-      -days 365 \
-      -keyout prod.key \
-      -out prod.crt
-   # ... and use "prod" as Common Name (CN)
-
-   ssh root@backups "mkdir /etc/zrepl"
-   scp  backups.key backups.crt prod.crt root@backups:/etc/zrepl
-
-   ssh root@prod "mkdir /etc/zrepl"
-   scp  prod.key prod.crt backups.crt root@prod:/etc/zrepl
-
-
-Configure server ``prod``
-------------------------
-
-We define a **push job** named ``prod_to_backups`` in ``/etc/zrepl/zrepl.yml`` on host ``prod`` : ::
-
-    jobs:
-    - name: prod_to_backups
-      type: push 
-      connect:
-        type: tls
-        address: "backups.example.com:8888"
-        ca: /etc/zrepl/backups.crt
-        cert: /etc/zrepl/prod.crt
-        key:  /etc/zrepl/prod.key
-        server_cn: "backups"
-      filesystems: {
-        "zroot/var/db": true,
-        "zroot/usr/home<": true,
-        "zroot/usr/home/paranoid": false
-      }
-      snapshotting:
-        type: periodic
-        prefix: zrepl_
-        interval: 10m
-      pruning:
-        keep_sender:
-        - type: not_replicated
-        - type: last_n
-          count: 10
-        keep_receiver:
-        - type: grid
-          grid: 1x1h(keep=all) | 24x1h | 30x1d | 6x30d
-          regex: "^zrepl_"
-
-.. _tutorial-configure-prod:
-
-Configure server ``backups``
----------------------------
-
-We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`` on host ``backups`` : ::
-
-    jobs:
-    - name: sink
-      type: sink 
-      serve:
-         type: tls
-         listen: ":8888"
-         ca: "/etc/zrepl/prod.crt"
-         cert: "/etc/zrepl/backups.crt"
-         key: "/etc/zrepl/backups.key"
-         client_cns:
-           - "prod"
-      root_fs: "storage/zrepl/sink"
-
-
-Apply Configuration Changes
---------------------------
-
-We use ``zrepl configcheck`` to catch any configuration errors: no output indicates that everything is fine.
-If that is the case, restart the zrepl daemon on **both** ``prod`` and ``backups`` using ``service zrepl restart`` or ``systemctl restart zrepl``.
-
-
-Watch it Work
-------------
-
-Run ``zrepl status`` on ``prod`` to monitor the replication and pruning activity.
-To re-trigger replication (snapshots are separate!), use ``zrepl signal wakeup prod_to_backups`` on ``prod``.
-
-If you like tmux, here is a handy script that works on FreeBSD: ::
-
-    pkg install gnu-watch tmux
-    tmux new -s zrepl -d
-    tmux split-window -t zrepl "tail -f /var/log/messages"
-    tmux split-window -t zrepl "gnu-watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_'"
-    tmux split-window -t zrepl "zrepl status"
-    tmux select-layout -t zrepl tiled
-    tmux attach -t zrepl
-
-The Linux equivalent might look like this: ::
-
-    # make sure tmux is installed & let's assume you use systemd + journald
-    tmux new -s zrepl -d
-    tmux split-window -t zrepl  "journalctl -f -u zrepl.service"
-    tmux split-window -t zrepl "watch 'zfs list -t snapshot -o name,creation -s creation | grep zrepl_'"
-    tmux split-window -t zrepl "zrepl status"
-    tmux select-layout -t zrepl tiled
-    tmux attach -t zrepl
-
-Summary
-------
-
-Congratulations, you have a working push backup. Where to go next?
-
-* Read more about :ref:`configuration format, options & job types <configuration_toc>`
-* Configure :ref:`logging <logging>` \& :ref:`monitoring <monitoring>`.
@@ -38,7 +38,7 @@ CLI Overview
    * - ``zrepl migrate``
      - | perform on-disk state / ZFS property migrations
        | (see :ref:`changelog <changelog>` for details)
-    * - ``zrepl zfs-abstractions``
+    * - ``zrepl zfs-abstraction``
      - list and remove zrepl's abstractions on top of ZFS, e.g. holds and step bookmarks (see :ref:`overview <replication-cursor-and-last-received-hold>` )

 .. _usage-zrepl-daemon:
@@ -282,7 +282,10 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
 			}
 			for _, staleVersion := range obsoleteAbs {
 				for _, mustLiveVersion := range mustLiveVersions {
-					if zfs.FilesystemVersionEqualIdentity(mustLiveVersion, staleVersion.GetFilesystemVersion()) {
+					isSendArg := zfs.FilesystemVersionEqualIdentity(mustLiveVersion, staleVersion.GetFilesystemVersion())
+					isStepHoldWeMightHaveCreatedWithCurrentValueOf_takeStepHolds :=
+						takeStepHolds && staleVersion.GetType() == AbstractionStepHold
+					if isSendArg && isStepHoldWeMightHaveCreatedWithCurrentValueOf_takeStepHolds {
 						panic(fmt.Sprintf("impl error: %q would be destroyed because it is considered stale but it is part of of sendArgs=%s", mustLiveVersion.String(), pretty.Sprint(sendArgs)))
 					}
 				}
@@ -351,6 +354,7 @@ func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*p
 			return &pdu.SendCompletedRes{}, err
 		}
 	} else {
+		sendAbstractionsCacheSingleton.Put(toReplicationCursor)
 		log(ctx).WithField("to_cursor", toReplicationCursor.String()).Info("successfully created `to` replication cursor")
 	}

@@ -86,7 +86,7 @@ func (s *sendAbstractionsCache) InvalidateFSCache(fs string) {
 // - only fetches on-disk abstractions once, but every time from the in-memory store
 //
 // That means that for precise results, all abstractions created by the endpoint must be .Put into this cache.
-func (s *sendAbstractionsCache) GetAndDeleteByJobIDAndFS(ctx context.Context, jobID JobID, fs string) (ret []Abstraction) {
+func (s *sendAbstractionsCache) GetAndDeleteByJobIDAndFS(ctx context.Context, jobID JobID, fs string, keep func(a Abstraction) bool) (ret []Abstraction) {
 	defer s.mtx.Lock().Unlock()
 	defer trace.WithSpanFromStackUpdateCtx(&ctx)()
 	var zeroJobId JobID
@@ -104,7 +104,7 @@ func (s *sendAbstractionsCache) GetAndDeleteByJobIDAndFS(ctx context.Context, jo
 	for _, a := range s.abstractions {
 		aJobId := *a.GetJobID()
 		aFS := a.GetFS()
-		if aJobId == jobID && aFS == fs {
+		if aJobId == jobID && aFS == fs && !keep(a) {
 			ret = append(ret, a)
 		} else {
 			remaining = append(remaining, a)
@@ -180,14 +180,7 @@ func (s *sendAbstractionsCache) TryBatchDestroy(ctx context.Context, jobId JobID

 	defer trace.WithSpanFromStackUpdateCtx(&ctx)()

-	allSendStepAbstractions := s.GetAndDeleteByJobIDAndFS(ctx, jobId, fs)
-
-	var obsoleteAbs []Abstraction
-	for _, a := range allSendStepAbstractions {
-		if !keep(a) {
-			obsoleteAbs = append(obsoleteAbs, a)
-		}
-	}
+	obsoleteAbs := s.GetAndDeleteByJobIDAndFS(ctx, jobId, fs, keep)

 	if check != nil {
 		check(obsoleteAbs)
@@ -18,6 +18,7 @@ var Cases = []Case{BatchDestroy,
 	ReceiveForceRollbackWorksUnencrypted,
 	ReplicationIncrementalCleansUpStaleAbstractionsWithCacheOnSecondReplication,
 	ReplicationIncrementalCleansUpStaleAbstractionsWithoutCacheOnSecondReplication,
+	ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledButStepHoldsExist,
 	ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed,
 	ReplicationIsResumableFullSend__DisableIncrementalStepHolds_False,
 	ReplicationIsResumableFullSend__DisableIncrementalStepHolds_True,
@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"sort"

 	"github.com/kr/pretty"
 	"github.com/stretchr/testify/require"
@@ -182,7 +183,7 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
 	require.NoError(ctx, err)
 	require.Contains(ctx, holds, expectRjidHoldTag)

-	// create artifical stale replication cursors
+	// create artifical stale replication cursors & step holds
 	createArtificalStaleAbstractions := func(jobId endpoint.JobID) []endpoint.Abstraction {
 		snap2Cursor, err := endpoint.CreateReplicationCursor(ctx, sfs, snap2, jobId) // no shadow
 		require.NoError(ctx, err)
@@ -407,3 +408,146 @@ func implReplicationIsResumableFullSend(ctx *platformtest.Context, disableIncrem
 	_ = fsversion(ctx, rfs, "@3")

 }
+
+func ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledButStepHoldsExist(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		CREATEROOT
+		+  "sender"
+		+  "receiver"
+		R  zfs create -p "${ROOTDS}/receiver/${ROOTDS}"
+	`)
+
+	sjid := endpoint.MustMakeJobID("sender-job")
+	rjid := endpoint.MustMakeJobID("receiver-job")
+
+	sfs := ctx.RootDataset + "/sender"
+	rfsRoot := ctx.RootDataset + "/receiver"
+
+	// fully replicate snapshots @1
+	{
+		mustSnapshot(ctx, sfs+"@1")
+		rep := replicationInvocation{
+			sjid:                        sjid,
+			rjid:                        rjid,
+			sfs:                         sfs,
+			rfsRoot:                     rfsRoot,
+			disableIncrementalStepHolds: false,
+		}
+		rfs := rep.ReceiveSideFilesystem()
+		report := rep.Do(ctx)
+		ctx.Logf("\n%s", pretty.Sprint(report))
+		// assert this worked (not the main subject of the test)
+		_ = fsversion(ctx, rfs, "@1")
+	}
+
+	// create a large snapshot @2
+	{
+		sfsmp, err := zfs.ZFSGetMountpoint(ctx, sfs)
+		require.NoError(ctx, err)
+		require.True(ctx, sfsmp.Mounted)
+		writeDummyData(path.Join(sfsmp.Mountpoint, "dummy.data"), 1<<22)
+		mustSnapshot(ctx, sfs+"@2")
+	}
+	snap2sfs := fsversion(ctx, sfs, "@2")
+
+	// partially replicate snapshots @2 with step holds enabled
+	// to effect a step-holds situation
+	{
+		rep := replicationInvocation{
+			sjid:                        sjid,
+			rjid:                        rjid,
+			sfs:                         sfs,
+			rfsRoot:                     rfsRoot,
+			disableIncrementalStepHolds: false, // !
+			interceptSender: func(e *endpoint.Sender) logic.Sender {
+				return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
+			},
+		}
+		rfs := rep.ReceiveSideFilesystem()
+		report := rep.Do(ctx)
+		ctx.Logf("\n%s", pretty.Sprint(report))
+		// assert this partial receive worked
+		_, err := zfs.ZFSGetFilesystemVersion(ctx, rfs+"@2")
+		ctx.Logf("%T %s", err, err)
+		_, notFullyReceived := err.(*zfs.DatasetDoesNotExist)
+		require.True(ctx, notFullyReceived)
+		// assert step holds are in place
+		abs, absErrs, err := endpoint.ListAbstractions(ctx, endpoint.ListZFSHoldsAndBookmarksQuery{
+			FS: endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+				FS: &sfs,
+			},
+			Concurrency: 1,
+			JobID:       &sjid,
+			What:        endpoint.AbstractionTypeSet{endpoint.AbstractionStepHold: true},
+		})
+		require.NoError(ctx, err)
+		require.Empty(ctx, absErrs)
+		require.Len(ctx, abs, 2)
+		sort.Slice(abs, func(i, j int) bool {
+			return abs[i].GetCreateTXG() < abs[j].GetCreateTXG()
+		})
+		require.True(ctx, zfs.FilesystemVersionEqualIdentity(abs[0].GetFilesystemVersion(), fsversion(ctx, sfs, "@1")))
+		require.True(ctx, zfs.FilesystemVersionEqualIdentity(abs[1].GetFilesystemVersion(), fsversion(ctx, sfs, "@2")))
+	}
+
+	//
+	// end of test setup
+	//
+
+	// retry replication with incremental step holds disabled
+	// - replication should not fail due to holds-related stuff
+	// - replication should fail intermittently due to partial sender being fully read
+	// - the partial sender is 1/4th the length of the stream, thus expect
+	//   successful replication after 5 more attempts
+	rep := replicationInvocation{
+		sjid:                        sjid,
+		rjid:                        rjid,
+		sfs:                         sfs,
+		rfsRoot:                     rfsRoot,
+		disableIncrementalStepHolds: true, // !
+		interceptSender: func(e *endpoint.Sender) logic.Sender {
+			return &PartialSender{Sender: e, failAfterByteCount: 1 << 20}
+		},
+	}
+	rfs := rep.ReceiveSideFilesystem()
+	for i := 0; ; i++ {
+		require.True(ctx, i < 5)
+		report := rep.Do(ctx)
+		ctx.Logf("retry run=%v\n%s", i, pretty.Sprint(report))
+		_, err := zfs.ZFSGetFilesystemVersion(ctx, rfs+"@2")
+		if err == nil {
+			break
+		}
+	}
+
+	// assert replication worked
+	fsversion(ctx, rfs, "@2")
+
+	// assert no step holds exist
+	abs, absErrs, err := endpoint.ListAbstractions(ctx, endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS: endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &sfs,
+		},
+		Concurrency: 1,
+		JobID:       &sjid,
+		What:        endpoint.AbstractionTypeSet{endpoint.AbstractionStepHold: true},
+	})
+	require.NoError(ctx, err)
+	require.Empty(ctx, absErrs)
+	require.Len(ctx, abs, 0)
+
+	// assert that the replication cursor bookmark exists
+	abs, absErrs, err = endpoint.ListAbstractions(ctx, endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS: endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &sfs,
+		},
+		Concurrency: 1,
+		JobID:       &sjid,
+		What:        endpoint.AbstractionTypeSet{endpoint.AbstractionReplicationCursorBookmarkV2: true},
+	})
+	require.NoError(ctx, err)
+	require.Empty(ctx, absErrs)
+	require.Len(ctx, abs, 1)
+	require.True(ctx, zfs.FilesystemVersionEqualIdentity(abs[0].GetFilesystemVersion(), snap2sfs))
+}
@@ -3,6 +3,8 @@ package version
 import (
 	"fmt"
 	"runtime"
+
+	"github.com/prometheus/client_golang/prometheus"
 )

 var (
@@ -29,3 +31,21 @@ func (i *ZreplVersionInformation) String() string {
 	return fmt.Sprintf("zrepl version=%s GOOS=%s GOARCH=%s Compiler=%s",
 		i.Version, i.RuntimeGOOS, i.RuntimeGOARCH, i.RUNTIMECompiler)
 }
+
+var prometheusMetric = prometheus.NewUntypedFunc(
+	prometheus.UntypedOpts{
+		Namespace: "zrepl",
+		Subsystem: "version",
+		Name:      "daemon",
+		Help:      "zrepl daemon version",
+		ConstLabels: map[string]string{
+			"raw":          zreplVersion,
+			"version_info": NewZreplVersionInformation().String(),
+		},
+	},
+	func() float64 { return 1 },
+)
+
+func PrometheusRegister(r prometheus.Registerer) {
+	r.MustRegister(prometheusMetric)
+}
Author	SHA1	Message	Date
Christian Schwarz	8e1937fe75	doc: fixup 0.3 changelog `05f1237a6d`	2020-06-14 18:29:37 +02:00
Christian Schwarz	073514fc21	docs/publish.sh: only render latest (patch+rc) version for each (major,minor) versio.	2020-06-14 18:24:20 +02:00
Christian Schwarz	dab222d95f	docs: GitHub Sponsors link	2020-06-14 15:26:05 +02:00
Christian Schwarz	a827894274	docs: add backup-to-external-disk quick-start guide and convert existing tutorial to quick-start guide refs #219 fixes #329	2020-06-14 15:26:05 +02:00
Christian Schwarz	9a8d813d14	docs: fix typo in cli help for zfs-abstraction subcommand	2020-06-14 15:26:05 +02:00
Christian Schwarz	e391fa94f9	dist/grafana: update grafana dashboard - uses version metric for 'instances up' - displays active task count - displays send abstractions cache entry count - in general, graphs have a shorter y axis for better overview fixes #332	2020-06-14 15:26:05 +02:00
Christian Schwarz	509185dfbe	prometheus: expose zrepl version as const metric	2020-06-14 15:26:05 +02:00
Christian Schwarz	4b1b7a8561	envconst: queryable report of resolved variables + integration inot zrepl status --raw fixes #299 refs #186	2020-06-14 15:26:05 +02:00
Christian Schwarz	b330ccca5d	transport/ssh: bump go-netssh version to fix ssh client process leaks fixes #322	2020-06-14 15:26:05 +02:00
Christian Schwarz	05f1237a6d	docs: 0.3 changelog	2020-06-14 15:26:05 +02:00
Christian Schwarz	1c270b7e39	add option to disable step holds for incremental sends This is a stop-gap solution until we re-write the pruner to support rules for removing step holds. Note that disabling step holds for incremental sends does not affect zrepl's guarantee that incremental replication is always possible: Suppose you yank the external drive during an incremental @from -> @to step: * restarting that step or future incrementals @from -> @to_later` will be possible because the replication cursor bookmark points to @from until the step is complete * resuming @from -> @to will work as long as the pruner on your internal pool doesn't come around to destroy @to. * in that case, the replication algorithm should determine that the resumable state on the receiving side isuseless because @to no longer exists on the sending side, and consequently clear it, and restart an incremental step @from -> @to_later refs #288	2020-06-14 15:26:05 +02:00
Christian Schwarz	1b39e9d03c	docs: update & extend replication overview wrt step holds + bookmarks	2020-06-14 15:21:36 +02:00
Christian Schwarz	655a2e5404	docs/configuration/overview.rst: fix wrong headline hierarchy	2020-06-14 15:21:36 +02:00
Christian Schwarz	9c80eea045	docs: update supporters	2020-06-14 15:21:36 +02:00
Christian Schwarz	175ad1dd0b	zfs: ZFSListFilesystemVersions: remove handling of io.ErrUnexpectedEOF ZFSListChan returns (*DatasetDoesNotExist) for the case mentioned in the comment	2020-06-14 15:21:36 +02:00
Christian Schwarz	728e97700f	zfs: fix error message formatting for send args validation	2020-06-14 15:21:36 +02:00
Christian Schwarz	94a0fbf953	[#321 ] platformtest: add test for zfs.ZFSHolds	2020-06-14 15:21:36 +02:00
Christian Schwarz	b056e7b2b9	[#321 ] endpoint: ListAbstractions: acutally emit one Abstraction per matching hold	2020-06-14 15:21:36 +02:00
Christian Schwarz	6e927f20f9	[#321 ] platformtest: minimal integration tests for package replication # Conflicts: # platformtest/tests/generated_cases.go	2020-06-14 15:21:36 +02:00
Christian Schwarz	301f163a44	[#321 ] platformtest: generate test case list + coverage tooling	2020-06-14 15:21:36 +02:00
Christian Schwarz	474652ea51	[#321 ] platformtest: fix test ListFilesystemVersionsZeroExistIsNotAnError	2020-06-14 15:21:36 +02:00
Christian Schwarz	1bc731e782	[#316 ] endpoint: delete unreachable code	2020-06-14 15:21:36 +02:00
Christian Schwarz	292b85b5ef	[#316 ] endpoint / replication protocol: more robust step-holds and replication cursor management - drop HintMostRecentCommonAncestor rpc call - it is wrong to put faith into the active side of the replication to always make that call (we might not trust it, ref pull setup) - clean up step holds + step bookmarks + replication cursor bookmarks on send RPC instead - this makes it symmetric with Receive RPC - use a cache (endpoint.sendAbstractionsCache) to avoid the cost of listing the on-disk endpoint abstractions state on every step The "create" methods for endpoint abstractions (CreateReplicationCursor, HoldStep) are now fully idempotent and return an Abstraction. Notes about endpoint.sendAbstractionsCache: - fills lazily from disk state on first `Get` operation - fill from disk is generally only attempted once - unless the `ListAbstractions` fails, in which case the fill from disk is retried on next `Get` (the current `Get` will observe a subset of the actual on-disk abstractions) - the `Invalidate` method is called - it is a global (zrepl process-wide) cache fixes #316	2020-06-14 15:21:36 +02:00