docs: changelog for 0.6.1

address updated golangci-lint errors: S1011 (gosimple)
should replace loop with `copy.outs[level] = append(copy.outs[level], os.outs[level]...)` (gosimple)
2023-09-10 11:13:14 +00:00 · 2023-09-10 11:12:46 +00:00 · 2023-09-10 11:12:46 +00:00 · 2023-09-10 11:12:46 +00:00 · 2023-09-10 10:19:23 +00:00 · 2023-09-10 10:12:20 +00:00
43 changed files with 3581 additions and 2257 deletions
@@ -1,5 +1,8 @@
 version: 2.1
-
+orbs:
+  # NB: 1.7.2 is not the Go version, but the Orb version
+  # https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache
+  go: circleci/go@1.7.2
 commands:
  setup-home-local-bin:
    steps:
@@ -31,17 +34,6 @@ commands:
      # The need for this was required for cimg/go:1.12, but let's future proof this here and now.
      - run: sudo apt-get install -y git ca-certificates

-  restore-cache-gomod:
-    steps:
-      - restore_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-  save-cache-gomod:
-    steps:
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
-
  install-godep:
    steps:
      - apt-update-and-install-common-deps
@@ -55,94 +47,41 @@ commands:
      - invoke-lazy-sh:
          subcommand: docdep

-  download-and-install-minio-client:
-    steps:
-      - setup-home-local-bin
-      - restore_cache:
-          key: minio-client-v2
-      - run:
-          shell: /bin/bash -eo pipefail
-          command: |
-            if which mc; then exit 0; fi
-            sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
-              -o "$HOME/.local/bin/mc"
-            sudo chmod +x "$HOME/.local/bin/mc"
-      - save_cache:
-          key: minio-client-v2
-          paths:
-            - "$HOME/.local/bin/mc"
-
-  upload-minio:
+  docs-publish-sh:
    parameters:
-      src:
-        type: string
-      dst:
-        type: string
+      push:
+        type: boolean
    steps:
+      - checkout
      - run:
-          shell: /bin/bash -eo pipefail
-          when: always
          command: |
-            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
-              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
-              exit 0
-            fi
-            set -u # from now on
+            git config --global user.email "zreplbot@cschwarz.com"
+            git config --global user.name "zrepl-github-io-ci"

-            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
-
-            # keep in sync with set-github-minio-status
-            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
-
-            # Upload artifacts
-            mkdir -p ./artifacts
-            mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
-
-  set-github-minio-status:
-    parameters:
-      context:
-        type: string
-      description:
-        type: string
-      minio-dst:
-        type: string
-    steps:
+      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
+      - run: ssh-add -D
+      # the default circleci ssh config only additional ssh keys for Host !github.com
      - run:
-          shell: /bin/bash -eo pipefail
          command: |
-            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
-              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
-              exit 0
-            fi
-            set -u # from now on
+            cat > ~/.ssh/config \<<EOF
+            Host *
+              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
+            EOF
+      - add_ssh_keys:
+          fingerprints:
+            # deploy key for zrepl.github.io
+            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"

-            # keep in sync with with upload-minio command
-            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
-            # Push Artifact Link to GitHub
-            REPO="zrepl/zrepl"
-            COMMIT="${CIRCLE_SHA1}"
-            JOB_NAME="${CIRCLE_JOB}"
-            CONTEXT="<<parameters.context>>"
-            DESCRIPTION="<<parameters.description>>"
-            TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
-            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
-              -H "Content-Type: application/json" \
-              -H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
-              -X POST \
-              -d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
-
-
-  trigger-pipeline:
-    parameters:
-      body_no_shell_subst:
-        type: string
-    steps:
-      - run: |
-          curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
-            -H 'Content-Type: application/json' \
-            -H 'Accept: application/json' \
-            -H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
-            --data '<<parameters.body_no_shell_subst>>'
+      # caller must install-docdep
+      - when:
+          condition: << parameters.push >>
+          steps:
+            - run: bash -x docs/publish.sh -c -a -P
+      - when:
+          condition:
+            not: << parameters.push >>
+          steps:
+            - run: bash -x docs/publish.sh -c -a

 parameters:
  do_ci:
@@ -155,7 +94,7 @@ parameters:

  release_docker_baseimage_tag:
    type: string
-    default: "1.19"
+    default: "1.21"

 workflows:
  version: 2
@@ -165,19 +104,19 @@ workflows:
    jobs:
      - quickcheck-docs
      - quickcheck-go: &quickcheck-go-smoketest
-          name: quickcheck-go-amd64-linux-1.19
-          goversion: &latest-go-release "1.19"
+          name: quickcheck-go-amd64-linux-1.21
+          goversion: &latest-go-release "1.21"
          goos: linux
          goarch: amd64
      - test-go-on-latest-go-release:
          goversion: *latest-go-release
      - quickcheck-go:
          requires:
-            - quickcheck-go-amd64-linux-1.19 #quickcheck-go-smoketest.name
+            - quickcheck-go-amd64-linux-1.21 #quickcheck-go-smoketest.name
          matrix: &quickcheck-go-matrix
            alias: quickcheck-go-matrix
            parameters:
-              goversion: [*latest-go-release, "1.18"]
+              goversion: [*latest-go-release, "1.20"]
              goos: ["linux", "freebsd"]
              goarch: ["amd64", "arm64"]
            exclude:
@@ -185,6 +124,14 @@ workflows:
              - goversion: *latest-go-release
                goos: linux
                goarch: amd64
+      - platformtest:
+          matrix:
+            parameters:
+              goversion: [*latest-go-release]
+              goos: ["linux"]
+              goarch: ["amd64"]
+          requires:
+            - quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >>

  release:
    when: << pipeline.parameters.do_release >>
@@ -202,20 +149,7 @@ workflows:
            - release-deb
            - release-rpm

-  periodic:
-    triggers:
-      - schedule:
-          cron: "00 17 * * *"
-          filters:
-            branches:
-              only:
-                - master
-                - stable
-                - problame/circleci-build
-    jobs:
-      - periodic-full-pipeline-run
-
-  zrepl.github.io:
+  publish-zrepl.github.io:
    jobs:
      - publish-zrepl-github-io:
          filters:
@@ -226,20 +160,15 @@ workflows:
 jobs:
  quickcheck-docs:
    docker:
-      - image: cimg/base:2020.08
+      - image: cimg/base:2023.09
    steps:
      - checkout
      - install-docdep
+      # do the current docs build
      - run: make docs
-
-      - download-and-install-minio-client
-      - upload-minio:
-          src: artifacts
-          dst: ""
-      - set-github-minio-status:
-          context: artifacts/${CIRCLE_JOB}
-          description: artifacts of CI job ${CIRCLE_JOB}
-          minio-dst: ""
+      # does the publish.sh script still work?
+      - docs-publish-sh:
+          push: false

  quickcheck-go:
    parameters:
@@ -258,12 +187,13 @@ jobs:
    steps:
      - checkout

+      - go/load-cache:
+          key: quickcheck-<<parameters.goversion>>
      - install-godep
-
-      - restore-cache-gomod
      - run: go mod download
      - run: cd build && go mod download
-      - save-cache-gomod
+      - go/save-cache:
+          key: quickcheck-<<parameters.goversion>>

      - run: make formatcheck
      - run: make generate-platform-test-list
@@ -271,17 +201,34 @@ jobs:
      - run: make vet
      - run: make lint

-      - download-and-install-minio-client
      - run: rm -f artifacts/generate-platform-test-list
      - store_artifacts:
          path: artifacts
-      - upload-minio:
-          src: artifacts
-          dst: ""
-      - set-github-minio-status:
-          context: artifacts/${CIRCLE_JOB}
-          description: artifacts of CI job ${CIRCLE_JOB}
-          minio-dst: ""
+      - persist_to_workspace:
+          root: .
+          paths: [.]
+
+  platformtest:
+    parameters:
+      goversion:
+        type: string
+      goos:
+        type: string
+      goarch:
+        type: string
+    machine:
+      image: ubuntu-2204:current
+    resource_class: medium
+    environment:
+      GOOS: <<parameters.goos>>
+      GOARCH: <<parameters.goarch>>
+    steps:
+      - attach_workspace:
+          at: .
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y zfsutils-linux
+      - run: sudo zfs version
+      - run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH"

  test-go-on-latest-go-release:
    parameters:
@@ -291,9 +238,11 @@ jobs:
      - image: cimg/go:<<parameters.goversion>>
    steps:
      - checkout
-      - restore-cache-gomod
+      - go/load-cache:
+          key: make-test-go
      - run: make test-go
-      # don't save-cache-gomod here, test-go doesn't pull all the dependencies
+      - go/save-cache:
+          key: make-test-go

  release-build:
    machine:
@@ -336,46 +285,12 @@ jobs:
          at: .
      - store_artifacts:
          path: artifacts
-      - download-and-install-minio-client
-      - upload-minio:
-          src: artifacts
-          dst: ""
-      - set-github-minio-status:
-          context: artifacts/release
-          description: CI-generated release artifacts
-          minio-dst: ""
-
-  periodic-full-pipeline-run:
-    docker:
-      - image: cimg/base:2020.08
-    steps:
-      - trigger-pipeline:
-          body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'

  publish-zrepl-github-io:
    docker:
-      - image: cimg/python:3.7
+      - image: cimg/base:2023.09
    steps:
      - checkout
-      - invoke-lazy-sh:
-          subcommand: docdep
-      - run:
-          command: |
-            git config --global user.email "zreplbot@cschwarz.com"
-            git config --global user.name "zrepl-github-io-ci"
-
-      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
-      - run: ssh-add -D
-      # the default circleci ssh config only additional ssh keys for Host !github.com
-      - run:
-          command: |
-            cat > ~/.ssh/config \<<EOF
-            Host *
-              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
-            EOF
-      - add_ssh_keys:
-          fingerprints:
-            # deploy key for zrepl.github.io
-            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
-
-      - run: bash -x docs/publish.sh -c -a
+      - install-docdep
+      - docs-publish-sh:
+          push: true
@@ -28,7 +28,7 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
 GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
 GOLANGCI_LINT := golangci-lint
 GOCOVMERGE := gocovmerge
-RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.19
+RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.21
 RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)

 ifneq ($(GOARM),)
@@ -2,12 +2,18 @@ FROM !SUBSTITUTED_BY_MAKEFILE

 RUN apt-get update && apt-get install -y \
    python3-pip \
+    python3-venv \
    unzip \
    gawk

 ADD build.installprotoc.bash ./
 RUN bash build.installprotoc.bash

+# setup venv
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 ADD lazy.sh /tmp/lazy.sh
 ADD docs/requirements.txt /tmp/requirements.txt
 ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt
@@ -4,10 +4,37 @@ go 1.12

 require (
 	github.com/alvaroloes/enumer v1.1.1
-	github.com/golangci/golangci-lint v1.50.1
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/breml/bidichk v0.2.6 // indirect
+	github.com/breml/errchkjson v0.3.5 // indirect
+	github.com/chavacava/garif v0.1.0 // indirect
+	github.com/daixiang0/gci v0.11.1 // indirect
+	github.com/golangci/golangci-lint v1.54.2
+	github.com/golangci/revgrep v0.5.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/jgautheron/goconst v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mgechev/revive v1.3.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/polyfloyd/go-errorlint v1.4.5 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
+	github.com/spf13/viper v1.16.0 // indirect
+	github.com/stretchr/objx v0.5.1 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tetafro/godot v1.4.15 // indirect
 	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
-	golang.org/x/tools v0.2.0
-	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
-	google.golang.org/protobuf v1.28.0
+	github.com/xen0n/gosmopolitan v1.2.2 // indirect
+	gitlab.com/bosi/decorder v0.4.1 // indirect
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.25.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/tools v0.13.0
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
+	google.golang.org/protobuf v1.31.0
+	mvdan.cc/unparam v0.0.0-20230815095028-f7c6fb1088f0 // indirect
 )
@@ -374,13 +374,15 @@ func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *by
 			eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
 		}

-		t.Write("Progress: ")
-		t.DrawBar(50, replicated, expected, changeCount)
-		t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
-		if eta != 0 {
-			t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+		if !latest.State.IsTerminal() {
+			t.Write("Progress: ")
+			t.DrawBar(50, replicated, expected, changeCount)
+			t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
+			if eta != 0 {
+				t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+			}
+			t.Newline()
 		}
-		t.Newline()
 		if containsInvalidSizeEstimates {
 			t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
 			t.Newline()
@@ -493,15 +495,17 @@ func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFun
 	}

 	// global progress bar
-	progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
-	t.Write("Progress: ")
-	t.Write("[")
-	t.Write(stringbuilder.Times("=", progress))
-	t.Write(">")
-	t.Write(stringbuilder.Times("-", 80-progress))
-	t.Write("]")
-	t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
-	t.Newline()
+	if !state.IsTerminal() {
+		progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
+		t.Write("Progress: ")
+		t.Write("[")
+		t.Write(stringbuilder.Times("=", progress))
+		t.Write(">")
+		t.Write(stringbuilder.Times("-", 80-progress))
+		t.Write("]")
+		t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
+		t.Newline()
+	}

 	sort.SliceStable(all, func(i, j int) bool {
 		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1
@@ -65,7 +65,6 @@ type ActiveJob struct {
 	Name               string                `yaml:"name"`
 	Connect            ConnectEnum           `yaml:"connect"`
 	Pruning            PruningSenderReceiver `yaml:"pruning"`
-	Debug              JobDebugSettings      `yaml:"debug,optional"`
 	Replication        *Replication          `yaml:"replication,optional,fromdefaults"`
 	ConflictResolution *ConflictResolution   `yaml:"conflict_resolution,optional,fromdefaults"`
 }
@@ -75,17 +74,15 @@ type ConflictResolution struct {
 }

 type PassiveJob struct {
-	Type  string           `yaml:"type"`
-	Name  string           `yaml:"name"`
-	Serve ServeEnum        `yaml:"serve"`
-	Debug JobDebugSettings `yaml:"debug,optional"`
+	Type  string    `yaml:"type"`
+	Name  string    `yaml:"name"`
+	Serve ServeEnum `yaml:"serve"`
 }

 type SnapJob struct {
 	Type         string            `yaml:"type"`
 	Name         string            `yaml:"name"`
 	Pruning      PruningLocal      `yaml:"pruning"`
-	Debug        JobDebugSettings  `yaml:"debug,optional"`
 	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
 }
@@ -480,14 +477,6 @@ type GlobalStdinServer struct {
 	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
 }

-type JobDebugSettings struct {
-	Conn *struct {
-		ReadDump  string `yaml:"read_dump"`
-		WriteDump string `yaml:"write_dump"`
-	} `yaml:"conn,optional"`
-	RPCLog bool `yaml:"rpc_log,optional,default=false"`
-}
-
 type HookList []HookEnum

 type HookEnum struct {
@@ -160,6 +160,14 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 	return
 }

+func (m DatasetMapFilter) UserSpecifiedDatasets() (datasets zfs.UserSpecifiedDatasetsSet) {
+	datasets = make(zfs.UserSpecifiedDatasetsSet)
+	for i := range m.entries {
+		datasets[m.entries[i].path.ToString()] = true
+	}
+	return
+}
+
 // Construct a new filter-only DatasetMapFilter from a mapping
 // The new filter allows exactly those paths that were not forbidden by the mapping.
 func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
@@ -199,6 +199,16 @@ const (
 	Done
 )

+// Returns true in case the State is a terminal state(PlanErr, ExecErr, Done)
+func (s State) IsTerminal() bool {
+	switch s {
+	case PlanErr, ExecErr, Done:
+		return true
+	default:
+		return false
+	}
+}
+
 type updater func(func(*Pruner))

 func (p *Pruner) Prune() {
@@ -0,0 +1,23 @@
+#!/sbin/openrc-run
+
+command='/usr/local/bin/zrepl'
+command_args='daemon'
+command_background='true'
+pidfile="/run/${RC_SVCNAME}.pid"
+output_log="/var/log/${RC_SVCNAME}.log"
+error_log="/var/log/${RC_SVCNAME}.log"
+
+zrepl_runtime_dir='/var/run/zrepl'
+
+start() {
+	mkdir -p "$zrepl_runtime_dir/stdinserver"
+	chmod -R 0700 "$zrepl_runtime_dir"
+	default_start
+}
+
+stop() {
+	rm -rf "$zrepl_runtime_dir"
+	default_stop
+}
+
+# vi: noet sw=8 sts=0
@@ -10,11 +10,11 @@ BUILDDIR      = _build

 # Put it first so that "make" without argument is like "make help".
 help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" -c sphinxconf $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

 .PHONY: help Makefile

 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" -c sphinxconf $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
@@ -0,0 +1,43 @@
+/* https://github.com/sphinx-contrib/sphinxcontrib-versioning/blob/0b56210959c6b21bbb730072e81876491b4e1371/sphinxcontrib/versioning/_static/banner.css */
+
+.scv-banner {
+    padding: 3px;
+    border-radius: 2px;
+    font-size: 80%;
+    text-align: center;
+    color: white;
+    background: #d40 linear-gradient(-45deg,
+        rgba(255, 255, 255, 0.2) 0%,
+        rgba(255, 255, 255, 0.2) 25%,
+        transparent 25%,
+        transparent 50%,
+        rgba(255, 255, 255, 0.2) 50%,
+        rgba(255, 255, 255, 0.2) 75%,
+        transparent 75%,
+        transparent
+    );
+    background-size: 28px 28px;
+}
+.scv-banner > a {
+    color: white;
+}
+
+
+.scv-sphinx_rtd_theme {
+    background-color: #2980B9;
+}
+
+
+.scv-bizstyle {
+    background-color: #336699;
+}
+
+
+.scv-classic {
+    text-align: center !important;
+}
+
+
+.scv-traditional {
+    text-align: center !important;
+}
@@ -0,0 +1,17 @@
+{% extends "!page.html" %}
+{% block body %}
+{% if current_version and latest_version and current_version != latest_version %}
+<p class="scv-banner scv-sphinx_rtd_theme">
+  <strong>
+    {% if current_version.is_released %}
+    You're reading an old version of this documentation.
+    If you want up-to-date information, please have a look at <a href="{{ vpathto(latest_version.name) }}">{{latest_version.name}}</a>.
+    {% else %}
+    You're reading the documentation for a development version.
+    For the latest released version, please have a look at <a href="{{ vpathto(latest_version.name) }}">{{latest_version.name}}</a>.
+    {% endif %}
+  </strong>
+</p>
+{% endif %}
+{{ super() }}
+{% endblock %}%
@@ -0,0 +1,27 @@
+{%- if current_version %}
+<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    <span class="fa fa-book"> Other Versions</span>
+    v: {{ current_version.name }}
+    <span class="fa fa-caret-down"></span>
+  </span>
+  <div class="rst-other-versions">
+    {%- if versions.tags %}
+    <dl>
+      <dt>Tags</dt>
+      {%- for item in versions.tags %}
+      <dd><a href="{{ item.url }}">{{ item.name }}</a></dd>
+      {%- endfor %}
+    </dl>
+    {%- endif %}
+    {%- if versions.branches %}
+    <dl>
+      <dt>Branches</dt>
+      {%- for item in versions.branches %}
+      <dd><a href="{{ item.url }}">{{ item.name }}</a></dd>
+      {%- endfor %}
+    </dl>
+    {%- endif %}
+  </div>
+</div>
+{%- endif %}
@@ -40,6 +40,27 @@ Those changes will likely come with some breakage in the config.
 However, I want to avoid breaking **use cases** that are satisfied by the current design.
 There will be beta/RC releases to give users a chance to evaluate.

+0.6.1
+-----
+
+* |feature| add metric to detect filesystems rules that don't match any local dataset (thanks, `@gmekicaxcient <https://github.com/gmekicaxcient>`_).
+* |bugfix| ``zrepl status``: hide progress bar once all filesystems reach terminal state (thanks, `@0x3333 <https://github.com/0x3333>`_).
+* |bugfix| handling of tenative cursor presence if protection strategy doesn't use it (:issue:`714`).
+* |docs| address setup with two or more external disks (thanks, `@se-jaeger <https://github.com/se-jaeger>`_).
+* |docs| document ``replication`` and ``conflict_resolution`` options (thanks, `@InsanePrawn <https://github.com/InsanePrawn>`_).
+* |docs| docs: talks: add note on keep_bookmarks option (thanks, `@skirmess <https://github.com/skirmess>`_).
+* |maint| dist: add openrc service file (thanks, `@gramosg <https://github.com/gramosg>`_).
+* |maint| grafana: update dashboard to Grafana 9.3.6.
+* |maint| run platform tests as part of CI.
+* |maint| build: upgrade to Go 1.21 and update golangci-lint; minimum Go version for builds is now 1.20
+
+.. NOTE::
+  |  zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
+  |  You can support maintenance and feature development through one of the following services:
+  |  |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
+  |  Note that PayPal processing fees are relatively high for small donations.
+  |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
+
 0.6
 ---

@@ -111,14 +132,6 @@ Note to all users: please read up on the following OpenZFS bugs, as you might be

 Finally, I'd like to point you to the `GitHub discussion <https://github.com/zrepl/zrepl/discussions/547>`_ about which bugfixes and features should be prioritized in zrepl 0.6 and beyond!

-.. NOTE::
-  |  zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
-  |  You can support maintenance and feature development through one of the following services:
-  |  |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
-  |  Note that PayPal processing fees are relatively high for small donations.
-  |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
-
-
 0.4.0
 -----

@@ -1 +0,0 @@
-sphinxconf/conf.py
@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# zrepl documentation build configuration file, created by
+# sphinx-quickstart on Wed Nov  8 22:28:10 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.todo',
+    'sphinx.ext.githubpages',
+    'sphinx.ext.extlinks',
+    "sphinx_multiversion",
+]
+
+# suppress_warnings = ['image.nonlocal_uri']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['./_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'zrepl'
+copyright = '2017-2023, Christian Schwarz'
+author = 'Christian Schwarz'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+#version = set by sphinxcontrib-versioning
+# The full version, including alpha/beta/rc tags.
+#release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'en'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+html_css_files = [
+    'banner.css',
+]
+
+html_logo = '_static/zrepl.svg'
+
+html_context = {
+  # https://github.com/rtfd/sphinx_rtd_theme/issues/205
+  # Add 'Edit on Github' link instead of 'View page source'
+  "display_github": True,
+  "github_user": "zrepl",
+  "github_repo": "zrepl",
+  "github_version": "master",
+  "conf_py_path": "/docs/",
+  "source_suffix": source_suffix,
+}
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'zrepldoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'zrepl.tex', 'zrepl Documentation',
+     'Christian Schwarz', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'zrepl', 'zrepl Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'zrepl', 'zrepl Documentation',
+     author, 'zrepl', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for the extlinks extension -----------------------------------
+# http://www.sphinx-doc.org/en/stable/ext/extlinks.html
+extlinks = {
+    'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #%s'),
+    'repomasterlink':('https://github.com/zrepl/zrepl/blob/master/%s', '%s'),
+    'sampleconf':('https://github.com/zrepl/zrepl/blob/master/config/samples%s', 'config/samples%s'),
+    'commit':('https://github.com/zrepl/zrepl/commit/%s', 'commit %s'),
+        }
+
@@ -1,5 +1,6 @@
 .. include:: ../global.rst.inc

+.. _conflict_resolution-options:

 Conflict Resolution Options
 ===========================
@@ -30,6 +30,10 @@ Job Type ``push``
      - |snapshotting-spec|
    * - ``pruning``
      - |pruning-spec|
+    * - ``replication``
+      - |replication-options|
+    * - ``conflict_resolution``
+      - |conflict-resolution-options|

 Example config: :sampleconf:`/push.yml`

@@ -81,6 +85,10 @@ Job Type ``pull``
        | ``manual`` disables periodic pulling, replication then only happens on :ref:`wakeup <cli-signal-wakeup>`.
    * - ``pruning``
      - |pruning-spec|
+    * - ``replication``
+      - |replication-options|
+    * - ``conflict_resolution``
+      - |conflict-resolution-options|

 Example config: :sampleconf:`/pull.yml`

@@ -44,27 +44,3 @@ Interval & duration fields in job definitions, pruning configurations, etc. must

    var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
    // s = second, m = minute, h = hour, d = day, w = week (7 days)
-
-Super-Verbose Job Debugging
---------------------------
-
-You have probably landed here because you opened an issue on GitHub and some developer told you to do this...
-So just read the annotated comments ;)
-
-::
-
-    job:
-    - name: ...
-      ...
-     # JOB DEBUGGING OPTIONS
-      # should be equal for all job types, but each job implements the debugging itself
-      debug:
-        conn: # debug the io.ReadWriteCloser connection
-          read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
-          write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
-        rpc: # debug the RPC protocol implementation
-          log: true # log output from rpc layer to the job log
-
-.. ATTENTION::
-
-    Connection dumps will almost certainly contain your or other's private data. Do not share it in a bug report.
@@ -1,5 +1,6 @@
 .. include:: ../global.rst.inc

+.. _replication-options:

 Replication Options
 ===================
@@ -197,11 +197,8 @@ Mount behaviour
 * ``canmount``
 * ``overlay``

-Note: inheriting or overriding the ``mountpoint`` property on ZVOLs fails in ``zfs recv``.
-This is an `issue in OpenZFS <https://github.com/openzfs/zfs/issues/11416>`_ .
-As a workaround, consider creating separate zrepl jobs for your ZVOL and filesystem datasets.
-Please comment at zrepl :issue:`430` if you encounter this issue and/or would like zrepl to automatically work around it.
-
+Note: Before `OpenZFS 2.0.5 <https://github.com/openzfs/zfs/issues/11416>`_, inheriting or overriding the ``mountpoint`` property on ZVOLs fails in ``zfs recv``.
+If you are on such an older version, consider creating separate zrepl jobs for your ZVOL and filesystem datasets.

 Systemd
 -------
@@ -26,6 +26,8 @@
 .. |connect-transport| replace:: :ref:`connect specification<transport>`
 .. |send-options| replace:: :ref:`send options<job-send-options>`, e.g. for encrypted sends
 .. |recv-options| replace:: :ref:`recv options<job-recv-options>`
+.. |replication-options| replace:: :ref:`replication options<replication-options>`
+.. |conflict-resolution-options| replace:: :ref:`conflict resolution options<conflict_resolution-options>`
 .. |snapshotting-spec| replace:: :ref:`snapshotting specification <job-snapshotting-spec>`
 .. |pruning-spec| replace:: :ref:`pruning specification <prune>`
 .. |filter-spec| replace:: :ref:`filter specification<pattern-filter>`
@@ -15,3 +15,5 @@ Talks & Presentations
  `Event <https://wiki.freebsd.org/DevSummit/201709>`__
  )

+  * Note: The remarks on ``keep_bookmarks`` are irrelevant as of zrepl 0.1 which introduced the zrepl-managed replication cursor bookmark.
+    Read the `Overview <overview-how-replication-works>`_ section to learn more.
@@ -3,7 +3,8 @@ set -euo pipefail

 NON_INTERACTIVE=false
 DO_CLONE=false
-while getopts "ca" arg; do
+PUSH=false
+while getopts "caP" arg; do
    case "$arg" in
        "a")
            NON_INTERACTIVE=true
@@ -11,6 +12,9 @@ while getopts "ca" arg; do
        "c")
            DO_CLONE=true
            ;;
+        "P")
+            PUSH=true
+            ;;
        *)
            echo "invalid option '-$arg'"
            exit 1
@@ -26,8 +30,8 @@ checkout_repo_msg() {
    echo "clone ${GHPAGESREPO} to ${PUBLICDIR}:"
 }

-if ! type sphinx-versioning >/dev/null; then
-    echo "install sphinx-versioning and come back"
+if ! type sphinx-multiversion >/dev/null; then
+    echo "install sphinx-multiversion and come back"
    exit 1
 fi

@@ -49,11 +53,11 @@ else
    read -r
 fi

-pushd "$PUBLICDIR" 
+pushd "$PUBLICDIR"

 echo "verify we're in the GitHub pages repo..."
 git remote get-url origin | grep -E "^${GHPAGESREPO}\$"
-if [ "$?" -ne "0" ] ;then 
+if [ "$?" -ne "0" ] ;then
    checkout_repo_msg
    echo "finished checkout, please run again"
    exit 1
@@ -73,22 +77,15 @@ popd

 echo "building site"

-flags="$(python3 gen-sphinx-versioning-flags.py)"
-set -e
-sphinx-versioning build \
-   $flags \
-   docs ./public_git \
-   -- -c sphinxconf # older conf.py throw errors because they used
-                    # version = subprocess.show_output(["git", "describe"])
-                    # which fails when building with sphinxcontrib-versioning
-set +e
+python3 run-sphinx-multiversion.py . ./public_git
+

 CURRENT_COMMIT=$(git rev-parse HEAD)
 git status --porcelain
 if [[ "$(git status --porcelain)" != "" ]]; then
-    CURRENT_COMMIT="${CURRENT_COMMIT}(dirty)" 
+    CURRENT_COMMIT="${CURRENT_COMMIT}(dirty)"
 fi
-COMMIT_MSG="sphinx-versioning render from publish.sh - $(date -u) - ${CURRENT_COMMIT}"
+COMMIT_MSG="render from publish.sh - $(date -u) - ${CURRENT_COMMIT}"

 pushd "$PUBLICDIR"

@@ -100,6 +97,11 @@ if [ "$(git status --porcelain)" != "" ]; then
 else
    echo "nothing to commit"
 fi
-echo "pushing to GitHub pages repo"
-git push origin master
+
+if $PUSH; then
+    echo "pushing to GitHub pages repo"
+    git push origin master
+else
+    echo "not pushing to GitHub pages repo, set -P flag to push"
+fi

@@ -33,4 +33,17 @@ You will likely want to customize some aspects mentioned in the top comment in t

 .. literalinclude:: ../../config/samples/quickstart_backup_to_external_disk.yml

+
+Offline Backups with two (or more) External Disks
+-------------------------------------------------
+
+It can be desirable to have multiple disk-based backups of the same machine.
+To accomplish this,
+
+* create one zpool per external HDD, each with a unique name, and
+* define a pair of ``push`` and ``sink`` job **for each** of these zpools, each with a unique ``name``, ``listener_name``, and ``root_fs``.
+
+The unique names ensure that the jobs don't step on each others' toes when managing :ref:`zrepl's ZFS abstractions <zrepl-zfs-abstractions>` . 
+
+
 :ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.
@@ -1,30 +1,24 @@
-alabaster==0.7.12
-attrs==19.1.0
-Babel==2.7.0
-certifi==2019.6.16
-chardet==3.0.4
-Click==7.0
-colorclass==2.2.0
-docutils==0.15.2
-idna==2.8
-imagesize==1.1.0
-Jinja2==2.10.1
-MarkupSafe==1.1.1
-packaging==19.1
-Pygments==2.4.2
-pyparsing==2.4.2
-pytz==2019.2
-requests==2.22.0
-six==1.12.0
-snowballstemmer==1.9.1
-Sphinx==1.8.5
-sphinx-rtd-theme==0.4.3
-sphinxcontrib-applehelp==1.0.1
-sphinxcontrib-devhelp==1.0.1
-sphinxcontrib-htmlhelp==1.0.2
+alabaster==0.7.13
+Babel==2.12.1
+certifi==2023.7.22
+charset-normalizer==3.2.0
+docutils==0.18.1
+idna==3.4
+imagesize==1.4.1
+Jinja2==3.1.2
+MarkupSafe==2.1.3
+packaging==23.1
+Pygments==2.16.1
+requests==2.31.0
+snowballstemmer==2.2.0
+Sphinx==7.2.5
+sphinx-multiversion @ git+https://github.com/zrepl/sphinx-multiversion/@52c915d7ad898d9641ec48c8bbccb7d4f079db93
+sphinx-rtd-theme==1.3.0
+sphinxcontrib-applehelp==1.0.7
+sphinxcontrib-devhelp==1.0.5
+sphinxcontrib-htmlhelp==2.0.4
+sphinxcontrib-jquery==4.1
 sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.2
-sphinxcontrib-serializinghtml==1.1.3
-git+https://github.com/rwblair/sphinxcontrib-versioning.git@7e3885a389a809e17ea55261316b7b0e98dbf98f#egg=sphinxcontrib-versioning
-sphinxcontrib-websupport==1.1.2
-urllib3==1.25.3
+sphinxcontrib-qthelp==1.0.6
+sphinxcontrib-serializinghtml==1.1.9
+urllib3==2.0.4
@@ -1,7 +1,15 @@
 #!/usr/bin/env python3

+from pathlib import Path
 import subprocess
 import re
+import argparse
+import distutils
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument("docsroot")
+argparser.add_argument("outdir")
+args = argparser.parse_args()

 output = subprocess.run(["git", "tag", "-l"], capture_output=True, check=True, text=True)
 tagRE = re.compile(r"^v(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-rc(?P<rc>\d+))?$")
@@ -27,8 +35,8 @@ for line in output.stdout.split("\n"):

    t = Tag()
    t.orig = line
-    t.major = int(m["major"])   
-    t.minor = int(m["minor"])   
+    t.major = int(m["major"])
+    t.minor = int(m["minor"])
    t.patch = int(m["patch"])
    t.rc = int(m["rc"] if m["rc"] is not None else 0)

@@ -49,26 +57,16 @@ for (mm, l) in by_major_minor.items():
    latest_by_major_minor.append(l[-1])
 latest_by_major_minor.sort(key=lambda tag: (tag.major, tag.minor))

-# print(by_major_minor)
-# print(latest_by_major_minor)
-
-cmdline = []
-
-for latest_patch in latest_by_major_minor:
-    cmdline.append("--whitelist-tags")
-    cmdline.append(f"^{re.escape(latest_patch.orig)}$")
-
-# we want flexibility to update docs for the latest stable release
-# => we have a branch for that, called `stable` which we move manually
-# TODO: in the future, have f"stable-{latest_by_major_minor[-1]}"
-default_version = "stable"
-cmdline.extend(["--whitelist-branches", default_version])
-
-cmdline.extend(["--root-ref", f"{default_version}"])
-cmdline.extend(["--banner-main-ref", f"{default_version}"])
-cmdline.extend(["--show-banner"])
-cmdline.extend(["--sort", "semver"])
-
-cmdline.extend(["--whitelist-branches", "master"])
-
-print(" ".join(cmdline))
+cmdline = [
+    "sphinx-multiversion",
+    "-D", "smv_tag_whitelist=^({})$".format("|".join([re.escape(tag.orig) for tag in latest_by_major_minor])),
+    "-D", "smv_branch_whitelist=^(master|stable)$",
+    "-D", "smv_remote_whitelist=^.*$",
+    "-D", "smv_latest_version=stable",
+    "-D", r"smv_released_pattern=^refs/(tags|heads|remotes/[^/]+)/(?!master).*$", # treat everything except master as released, that way, the banner message makes sense
+    # "--dump-metadata", # for debugging
+    args.docsroot,
+    args.outdir,
+]
+print(cmdline)
+subprocess.run(cmdline, check=True)
@@ -1,182 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# zrepl documentation build configuration file, created by
-# sphinx-quickstart on Wed Nov  8 22:28:10 2017.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
-
-
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = ['sphinx.ext.todo',
-    'sphinx.ext.githubpages',
-    'sphinx.ext.extlinks']
-
-suppress_warnings = ['image.nonlocal_uri']
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['../_templates']
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = 'zrepl'
-copyright = '2017-2019, Christian Schwarz'
-author = 'Christian Schwarz'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-#version = set by sphinxcontrib-versioning
-# The full version, including alpha/beta/rc tags.
-#release = version
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = True
-
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = 'sphinx_rtd_theme'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['../_static']
-
-
-html_logo = '../_static/zrepl.svg'
-
-html_context = {
-  # https://github.com/rtfd/sphinx_rtd_theme/issues/205
-  # Add 'Edit on Github' link instead of 'View page source'
-  "display_github": True, 
-  "github_user": "zrepl",
-  "github_repo": "zrepl",
-  "github_version": "master",
-  "conf_py_path": "/docs/",
-  "source_suffix": source_suffix,
-}
-
-# -- Options for HTMLHelp output ------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'zrepldoc'
-
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (master_doc, 'zrepl.tex', 'zrepl Documentation',
-     'Christian Schwarz', 'manual'),
-]
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'zrepl', 'zrepl Documentation',
-     [author], 1)
-]
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (master_doc, 'zrepl', 'zrepl Documentation',
-     author, 'zrepl', 'One line description of project.',
-     'Miscellaneous'),
-]
-
-
-# -- Options for the extlinks extension -----------------------------------
-# http://www.sphinx-doc.org/en/stable/ext/extlinks.html
-extlinks = {
-    'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #'),
-    'repomasterlink':('https://github.com/zrepl/zrepl/blob/master/%s', ''),
-    'sampleconf':('https://github.com/zrepl/zrepl/blob/master/config/samples%s', 'config/samples'),
-    'commit':('https://github.com/zrepl/zrepl/commit/%s', 'commit '),
-        }
-
@@ -32,6 +32,7 @@ We would like to thank the following people and organizations for supporting zre

   <div class="fa fa-code" style="width: 1em;"></div>

+* |supporter-std| `Max Christian Pohle <https://coderonline.de>`_
 * |supporter-gold| Prominic.NET, Inc.
 * |supporter-std| Torsten Blum
 * |supporter-gold| Cyberiada GmbH
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"strings"

 	"github.com/kr/pretty"
 	"github.com/pkg/errors"
@@ -233,6 +234,21 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
 	//
 	// Note further that a resuming send, due to the idempotent nature of func CreateReplicationCursor and HoldStep,
 	// will never lose its step holds because we just (idempotently re-)created them above, before attempting the cleanup.
+	destroyTypes := AbstractionTypeSet{
+		AbstractionStepHold:                           true,
+		AbstractionTentativeReplicationCursorBookmark: true,
+	}
+	// The replication planner can also pick an endpoint zfs abstraction as FromVersion.
+	// Keep it, so that the replication will succeed.
+	//
+	// NB: there is no abstraction for snapshots, so, we only need to check bookmarks.
+	if sendArgs.FromVersion != nil && sendArgs.FromVersion.IsBookmark() {
+		dp, err := zfs.NewDatasetPath(sendArgs.FS)
+		if err != nil {
+			panic(err) // sendArgs is validated, this shouldn't happen
+		}
+		liveAbs = append(liveAbs, destroyTypes.ExtractBookmark(dp, sendArgs.FromVersion))
+	}
 	func() {
 		ctx, endSpan := trace.WithSpan(ctx, "cleanup-stale-abstractions")
 		defer endSpan()
@@ -245,35 +261,45 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
 			return keep
 		}
 		check := func(obsoleteAbs []Abstraction) {
-			// last line of defense: check that we don't destroy the incremental `from` and `to`
-			// if we did that, we might be about to blow away the last common filesystem version between sender and receiver
-			mustLiveVersions := []zfs.FilesystemVersion{sendArgs.ToVersion}
-			if sendArgs.FromVersion != nil {
-				mustLiveVersions = append(mustLiveVersions, *sendArgs.FromVersion)
+			// Ensure that we don't delete `From` or `To`.
+			// Regardless of whether they are in AbstractionTypeSet or not.
+			// And produce a nice error message in case we do, to aid debugging the resulting panic.
+			//
+			// This is especially important for `From`. We could break incremental replication
+			// if we deleted the last common filesystem version between sender and receiver.
+			type Problem struct {
+				sendArgsWhat string
+				fullpath     string
+				obsoleteAbs  Abstraction
 			}
-			for _, staleVersion := range obsoleteAbs {
-				for _, mustLiveVersion := range mustLiveVersions {
-					isSendArg := zfs.FilesystemVersionEqualIdentity(mustLiveVersion, staleVersion.GetFilesystemVersion())
-					stepHoldBasedGuaranteeStrategy := false
-					k := replicationGuaranteeStrategy.Kind()
-					switch k {
-					case ReplicationGuaranteeKindResumability:
-						stepHoldBasedGuaranteeStrategy = true
-					case ReplicationGuaranteeKindIncremental:
-					case ReplicationGuaranteeKindNone:
-					default:
-						panic(fmt.Sprintf("this is supposed to be an exhaustive match, got %v", k))
-					}
-					isSnapshot := mustLiveVersion.IsSnapshot()
-					if isSendArg && (!isSnapshot || stepHoldBasedGuaranteeStrategy) {
-						panic(fmt.Sprintf("impl error: %q would be destroyed because it is considered stale but it is part of of sendArgs=%s", mustLiveVersion.String(), pretty.Sprint(sendArgs)))
+			problems := make([]Problem, 0)
+			checkFullpaths := make(map[string]string, 2)
+			checkFullpaths["ToVersion"] = sendArgs.ToVersion.FullPath(sendArgs.FS)
+			if sendArgs.FromVersion != nil {
+				checkFullpaths["FromVersion"] = sendArgs.FromVersion.FullPath(sendArgs.FS)
+			}
+			for _, a := range obsoleteAbs {
+				for what, fullpath := range checkFullpaths {
+					if a.GetFullPath() == fullpath && a.GetType().IsSnapshotOrBookmark() {
+						problems = append(problems, Problem{
+							sendArgsWhat: what,
+							fullpath:     fullpath,
+							obsoleteAbs:  a,
+						})
 					}
 				}
 			}
-		}
-		destroyTypes := AbstractionTypeSet{
-			AbstractionStepHold:                           true,
-			AbstractionTentativeReplicationCursorBookmark: true,
+			if len(problems) == 0 {
+				return
+			}
+			var msg strings.Builder
+			fmt.Fprintf(&msg, "cleaning up send stale would destroy send args:\n")
+			fmt.Fprintf(&msg, "  SendArgs: %s\n", pretty.Sprint(sendArgs))
+			for _, check := range problems {
+				fmt.Fprintf(&msg, "would delete %s %s because it was deemed an obsolete abstraction: %s\n",
+					check.sendArgsWhat, check.fullpath, check.obsoleteAbs)
+			}
+			panic(msg.String())
 		}
 		abstractionsCacheSingleton.TryBatchDestroy(ctx, s.jobId, sendArgs.FS, destroyTypes, keep, check)
 	}()
@@ -428,6 +454,7 @@ func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, _ io.ReadCloser

 type FSFilter interface { // FIXME unused
 	Filter(path *zfs.DatasetPath) (pass bool, err error)
+	UserSpecifiedDatasets() zfs.UserSpecifiedDatasetsSet
 }

 // FIXME: can we get away without error types here?
@@ -587,6 +614,12 @@ func (f subroot) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 	return p.HasPrefix(f.localRoot) && !p.Equal(f.localRoot), nil
 }

+func (f subroot) UserSpecifiedDatasets() zfs.UserSpecifiedDatasetsSet {
+	return zfs.UserSpecifiedDatasetsSet{
+		f.localRoot.ToString(): true,
+	}
+}
+
 func (f subroot) MapToLocal(fs string) (*zfs.DatasetPath, error) {
 	p, err := zfs.NewDatasetPath(fs)
 	if err != nil {
@@ -89,6 +89,8 @@ func ReplicationGuaranteeFromKind(k ReplicationGuaranteeKind) ReplicationGuarant

 type ReplicationGuaranteeNone struct{}

+func (g ReplicationGuaranteeNone) String() string { return "none" }
+
 func (g ReplicationGuaranteeNone) Kind() ReplicationGuaranteeKind {
 	return ReplicationGuaranteeKindNone
 }
@@ -107,6 +109,8 @@ func (g ReplicationGuaranteeNone) SenderPostRecvConfirmed(ctx context.Context, j

 type ReplicationGuaranteeIncremental struct{}

+func (g ReplicationGuaranteeIncremental) String() string { return "incremental" }
+
 func (g ReplicationGuaranteeIncremental) Kind() ReplicationGuaranteeKind {
 	return ReplicationGuaranteeKindIncremental
 }
@@ -144,6 +148,8 @@ func (g ReplicationGuaranteeIncremental) SenderPostRecvConfirmed(ctx context.Con

 type ReplicationGuaranteeResumability struct{}

+func (g ReplicationGuaranteeResumability) String() string { return "resumability" }
+
 func (g ReplicationGuaranteeResumability) Kind() ReplicationGuaranteeKind {
 	return ReplicationGuaranteeKindResumability
 }
@@ -31,7 +31,7 @@ const (
 	AbstractionReplicationCursorBookmarkV2        AbstractionType = "replication-cursor-bookmark-v2"
 )

-var AbstractionTypesAll = map[AbstractionType]bool{
+var AbstractionTypesAll = AbstractionTypeSet{
 	AbstractionStepHold:                           true,
 	AbstractionLastReceivedHold:                   true,
 	AbstractionTentativeReplicationCursorBookmark: true,
@@ -181,6 +181,38 @@ func (s AbstractionTypeSet) Validate() error {
 	return nil
 }

+// Use the `BookmarkExtractor()` method of each abstraction type in this set
+// to try extract an abstraction from the given FilesystemVersion.
+//
+// Abstraction types in this set that don't have a bookmark extractor are skipped.
+//
+// Panics if more than one abstraction type matches.
+func (s AbstractionTypeSet) ExtractBookmark(dp *zfs.DatasetPath, v *zfs.FilesystemVersion) Abstraction {
+	matched := make(AbstractionTypeSet, 1)
+	var matchedAbs Abstraction
+	for absType := range s {
+		extractor := absType.BookmarkExtractor()
+		if extractor == nil {
+			continue
+		}
+		abstraction := extractor(dp, *v)
+		if abstraction != nil {
+			matched[absType] = true
+			matchedAbs = abstraction
+		}
+	}
+	if len(matched) == 0 {
+		return nil
+	}
+	if len(matched) == 1 {
+		if matchedAbs == nil {
+			panic("loop above should always set matchedAbs if there is a match")
+		}
+		return matchedAbs
+	}
+	panic(fmt.Sprintf("abstraction types extractors should not overlap: %s", matched))
+}
+
 type BookmarkExtractor func(fs *zfs.DatasetPath, v zfs.FilesystemVersion) Abstraction

 // returns nil if the abstraction type is not bookmark-based
@@ -238,6 +270,23 @@ func (t AbstractionType) BookmarkNamer() func(fs string, guid uint64, jobId JobI
 	}
 }

+func (t AbstractionType) IsSnapshotOrBookmark() bool {
+	switch t {
+	case AbstractionTentativeReplicationCursorBookmark:
+		return true
+	case AbstractionReplicationCursorBookmarkV1:
+		return true
+	case AbstractionReplicationCursorBookmarkV2:
+		return true
+	case AbstractionStepHold:
+		return false
+	case AbstractionLastReceivedHold:
+		return false
+	default:
+		panic(fmt.Sprintf("unimpl: %q", t))
+	}
+}
+
 type ListZFSHoldsAndBookmarksQuery struct {
 	FS ListZFSHoldsAndBookmarksQueryFilesystemFilter
 	// What abstraction types should match (any contained in the set)
@@ -150,9 +150,7 @@ func NewOutlets() *Outlets {
 func (os *Outlets) DeepCopy() (copy *Outlets) {
 	copy = NewOutlets()
 	for level := range os.outs {
-		for i := range os.outs[level] {
-			copy.outs[level] = append(copy.outs[level], os.outs[level][i])
-		}
+		copy.outs[level] = append(copy.outs[level], os.outs[level]...)
 	}
 	return copy
 }
@@ -7,7 +7,8 @@ RUN apt-get update && apt-get install -y \
    dh-exec \
 	binutils-aarch64-linux-gnu \
 	binutils-arm-linux-gnueabihf  \
-	binutils-i686-linux-gnu
+	binutils-i686-linux-gnu \
+	binutils-x86-64-linux-gnu

 RUN mkdir -p /build/src && chmod -R 0777 /build

@@ -4,6 +4,21 @@ export ZREPL_MOCK_ZFS_COMMAND_LOG="$1"
 shift
 export ZREPL_MOCK_ZFS_PATH="$1"
 shift
-export PATH="$(dirname "${BASH_SOURCE[0]}" )":"$PATH"
+dirname="$(dirname "${BASH_SOURCE[0]}")"
+# If we invoke this script from the top zrepl source tree, like so:
+#    ./platformtest/logmockzfs/logzfsenv ...
+# then dirname is relative, i.e., ./platformtest/logmockzfs.
+# If we put that relative dir in PATH, then Go >= 1.19 will refuse to
+# exec the `./platformtest/logmockzfs/zfs` wrapper script if it finds
+# it via PATH lookup. For Example:
+#   cmd := exec.Command("zfs")
+#   err := cmd.Run()
+# will fail with an error that errors.Is(err, exec.ErrDot), message:
+#   cannot run executable found relative to current directory
+# The solution is to use an abspath.
+# Learn more at https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
+# and https://go-review.googlesource.com/c/go/+/381374
+absdirname="$(readlink -e "$dirname")"
+export PATH="$absdirname":"$PATH"
 args=("$@")
 exec "${args[@]}"
@@ -20,6 +20,7 @@ var Cases = []Case{BatchDestroy,
 	ReplicationIncrementalCleansUpStaleAbstractionsWithCacheOnSecondReplication,
 	ReplicationIncrementalCleansUpStaleAbstractionsWithoutCacheOnSecondReplication,
 	ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledButStepHoldsExist,
+	ReplicationIncrementalHandlesFromVersionEqTentativeCursorCorrectly,
 	ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed,
 	ReplicationInitialAll,
 	ReplicationInitialFail,
@@ -248,7 +248,10 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
 		require.NoError(ctx, err)
 		snap2Hold, err := endpoint.HoldStep(ctx, sfs, snap2, jobId) // no shadow
 		require.NoError(ctx, err)
-		return []endpoint.Abstraction{snap2Cursor, snap1Hold, snap2Hold}
+		// create artificial tentative cursor
+		snap3TentativeCursor, err := endpoint.CreateTentativeReplicationCursor(ctx, sfs, snap3, jobId)
+		require.NoError(ctx, err)
+		return []endpoint.Abstraction{snap2Cursor, snap1Hold, snap2Hold, snap3TentativeCursor}
 	}
 	createArtificalStaleAbstractions(sjid)
 	ojidSendAbstractions := createArtificalStaleAbstractions(ojid)
@@ -333,21 +336,29 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
 		require.NoError(ctx, err)
 		snap2OjidCursorName, err := endpoint.ReplicationCursorBookmarkName(sfs, snap2.Guid, ojid)
 		require.NoError(ctx, err)
+		snap3SjidTentativeCursorName, err := endpoint.TentativeReplicationCursorBookmarkName(sfs, snap3.Guid, sjid)
+		require.NoError(ctx, err)
+		snap3OjidTentativeCursorName, err := endpoint.TentativeReplicationCursorBookmarkName(sfs, snap3.Guid, ojid)
+		require.NoError(ctx, err)
 		var bmNames []string
 		for _, bm := range sBms {
 			bmNames = append(bmNames, bm.Name)
 		}

 		if invalidateCacheBeforeSecondReplication {
-			require.Len(ctx, sBms, 3)
+			require.Len(ctx, sBms, 4)
 			require.Contains(ctx, bmNames, snap5SjidCursorName)
 			require.Contains(ctx, bmNames, snap2OjidCursorName)
+			require.Contains(ctx, bmNames, snap3OjidTentativeCursorName)
 			require.Contains(ctx, bmNames, "2")
 		} else {
-			require.Len(ctx, sBms, 4)
+			require.Len(ctx, sBms, 6)
+			ctx.Logf("%s", pretty.Sprint(sBms))
 			require.Contains(ctx, bmNames, snap5SjidCursorName)
 			require.Contains(ctx, bmNames, snap2SjidCursorName)
 			require.Contains(ctx, bmNames, snap2OjidCursorName)
+			require.Contains(ctx, bmNames, snap3SjidTentativeCursorName)
+			require.Contains(ctx, bmNames, snap3OjidTentativeCursorName)
 			require.Contains(ctx, bmNames, "2")
 		}
 	}
@@ -370,6 +381,84 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte

 }

+func ReplicationIncrementalHandlesFromVersionEqTentativeCursorCorrectly(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		CREATEROOT
+		+  "sender"
+		+  "sender@1"
+		+  "receiver"
+		R  zfs create -p "${ROOTDS}/receiver/${ROOTDS}"
+	`)
+
+	sjid := endpoint.MustMakeJobID("sender-job")
+	rjid := endpoint.MustMakeJobID("receiver-job")
+
+	sfs := ctx.RootDataset + "/sender"
+	rfsRoot := ctx.RootDataset + "/receiver"
+
+	rep := replicationInvocation{
+		sjid:    sjid,
+		rjid:    rjid,
+		sfs:     sfs,
+		rfsRoot: rfsRoot,
+		// It doesn't really matter what guarantee we use here, as the second replication will configure another.
+		// But, in the real world, the only way for a stale tentative cursor to appear is if the guarantee is set to
+		// incremental replication and we crash before converting the tentative cursor into a regular cursor.
+		guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication),
+	}
+
+	// Do initial replication to set up the test.
+	rep1 := rep.Do(ctx)
+	ctx.Logf("\n%s", pretty.Sprint(rep1))
+	sfsDs := mustDatasetPath(sfs)
+	snap1_sender := mustGetFilesystemVersion(ctx, sfs+"@1")
+	snap1_replicationCursor_name, err := endpoint.ReplicationCursorBookmarkName(sfs, snap1_sender.Guid, sjid)
+	require.NoError(ctx, err)
+	snap1_replicationCursor := mustGetFilesystemVersion(ctx, sfs+"#"+snap1_replicationCursor_name)
+
+	// The second replication will be done with a guarantee kind that doesn't create tentative cursors by itself.
+	// So, it would generally be right to clean up any tentative cursors on sfs since they're stale abstractions.
+	// However, if the cursor is used as the `from` version in any send step, we must not destroy it, as that
+	// would break incremental replication.
+	// NB: we only need to test the first step as all subsequent steps will be snapshot->snapshot.
+	rep.guarantee = pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing)
+	// create the artificial cursor
+	snap1_tentativeCursor, err := endpoint.CreateTentativeReplicationCursor(ctx, sfs, snap1_sender, sjid)
+	require.NoError(ctx, err)
+	endpoint.AbstractionsCacheInvalidate(sfs)
+	// remove other bookmarks of snap1, and snap1 itself, to force the replication planner to use the tentative cursor
+	err = zfs.ZFSDestroyFilesystemVersion(ctx, sfsDs, &snap1_sender)
+	require.NoError(ctx, err)
+	err = zfs.ZFSDestroyFilesystemVersion(ctx, sfsDs, &snap1_replicationCursor)
+	require.NoError(ctx, err)
+	versions, err := zfs.ZFSListFilesystemVersions(ctx, sfsDs, zfs.ListFilesystemVersionsOptions{})
+	require.NoError(ctx, err)
+	require.Len(ctx, versions, 1)
+	require.Equal(ctx, versions[0].Guid, snap1_tentativeCursor.GetFilesystemVersion().Guid)
+	// create another snapshot so that replication does one incremental step `tentative_cursor` -> `@2`
+	mustSnapshot(ctx, sfs+"@2")
+	mustGetFilesystemVersion(ctx, sfs+"@2")
+	// do the replication
+	rep2 := rep.Do(ctx)
+	ctx.Logf("\n%s", pretty.Sprint(rep2))
+
+	// Ensure that the tentative cursor was used.
+	require.Len(ctx, rep2.Attempts, 1)
+	require.Equal(ctx, rep2.Attempts[0].State, report.AttemptDone)
+	require.Len(ctx, rep2.Attempts[0].Filesystems, 1)
+	require.Nil(ctx, rep2.Attempts[0].Filesystems[0].Error())
+	require.Len(ctx, rep2.Attempts[0].Filesystems[0].Steps, 1)
+	require.EqualValues(ctx, rep2.Attempts[0].Filesystems[0].CurrentStep, 1)
+	require.Len(ctx, rep2.Attempts[0].Filesystems[0].Steps, 1)
+	require.Equal(ctx, rep2.Attempts[0].Filesystems[0].Steps[0].Info.From, snap1_tentativeCursor.GetFilesystemVersion().RelName())
+
+	// Ensure that the tentative cursor was destroyed as part of SendPost.
+	_, err = zfs.ZFSGetFilesystemVersion(ctx, snap1_replicationCursor.FullPath(sfs))
+	_, ok := err.(*zfs.DatasetDoesNotExist)
+	require.True(ctx, ok)
+}
+
 type PartialSender struct {
 	*endpoint.Sender
 	failAfterByteCount int64
@@ -192,3 +192,14 @@ func (r *Report) GetFailedFilesystemsCountInLatestAttempt() int {
 		return 0
 	}
 }
+
+// Returns true in case the AttemptState is a terminal
+// state(AttemptPlanningError, AttemptFanOutError, AttemptDone)
+func (a AttemptState) IsTerminal() bool {
+	switch a {
+	case AttemptPlanningError, AttemptFanOutError, AttemptDone:
+		return true
+	default:
+		return false
+	}
+}
@@ -107,34 +107,31 @@ func (c *Conn) RenewWriteDeadline() error {
 	return c.SetWriteDeadline(time.Now().Add(c.idleTimeout))
 }

-func (c *Conn) Read(p []byte) (n int, err error) {
+func (c *Conn) Read(p []byte) (n int, _ error) {
 	n = 0
-	err = nil
 restart:
 	if err := c.renewReadDeadline(); err != nil {
 		return n, err
 	}
 	var nCurRead int
-	nCurRead, err = c.Wire.Read(p[n:])
+	nCurRead, err := c.Wire.Read(p[n:])
 	n += nCurRead
 	if netErr, ok := err.(net.Error); ok && netErr.Timeout() && nCurRead > 0 {
-		err = nil
 		goto restart
 	}
 	return n, err
 }

-func (c *Conn) Write(p []byte) (n int, err error) {
+func (c *Conn) Write(p []byte) (n int, _ error) {
 	n = 0
 restart:
 	if err := c.RenewWriteDeadline(); err != nil {
 		return n, err
 	}
 	var nCurWrite int
-	nCurWrite, err = c.Wire.Write(p[n:])
+	nCurWrite, err := c.Wire.Write(p[n:])
 	n += nCurWrite
 	if netErr, ok := err.(net.Error); ok && netErr.Timeout() && nCurWrite > 0 {
-		err = nil
 		goto restart
 	}
 	return n, err
@@ -144,17 +141,16 @@ restart:
 // but is guaranteed to use the writev system call if the wrapped Wire
 // support it.
 // Note the Conn does not support writev through io.Copy(aConn, aNetBuffers).
-func (c *Conn) WritevFull(bufs net.Buffers) (n int64, err error) {
+func (c *Conn) WritevFull(bufs net.Buffers) (n int64, _ error) {
 	n = 0
 restart:
 	if err := c.RenewWriteDeadline(); err != nil {
 		return n, err
 	}
 	var nCurWrite int64
-	nCurWrite, err = io.Copy(c.Wire, &bufs)
+	nCurWrite, err := io.Copy(c.Wire, &bufs)
 	n += nCurWrite
 	if netErr, ok := err.(net.Error); ok && netErr.Timeout() && nCurWrite > 0 {
-		err = nil
 		goto restart
 	}
 	return n, err
@@ -3,12 +3,20 @@ package zfs
 import (
 	"context"
 	"fmt"
+
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )

 type DatasetFilter interface {
 	Filter(p *DatasetPath) (pass bool, err error)
+	// The caller owns the returned set.
+	// Implementations should return a copy.
+	UserSpecifiedDatasets() UserSpecifiedDatasetsSet
 }

+// A set of dataset names that the user specified in the configuration file.
+type UserSpecifiedDatasetsSet map[string]bool
+
 // Returns a DatasetFilter that does not filter (passes all paths)
 func NoFilter() DatasetFilter {
 	return noFilter{}
@@ -18,7 +26,8 @@ type noFilter struct{}

 var _ DatasetFilter = noFilter{}

-func (noFilter) Filter(p *DatasetPath) (pass bool, err error) { return true, nil }
+func (noFilter) Filter(p *DatasetPath) (pass bool, err error)    { return true, nil }
+func (noFilter) UserSpecifiedDatasets() UserSpecifiedDatasetsSet { return nil }

 func ZFSListMapping(ctx context.Context, filter DatasetFilter) (datasets []*DatasetPath, err error) {
 	res, err := ZFSListMappingProperties(ctx, filter, nil)
@@ -61,6 +70,7 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert

 	go ZFSListChan(ctx, rchan, properties, nil, "-r", "-t", "filesystem,volume")

+	unmatchedUserSpecifiedDatasets := filter.UserSpecifiedDatasets()
 	datasets = make([]ZFSListMappingPropertiesResult, 0)
 	for r := range rchan {

@@ -74,6 +84,8 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert
 			return
 		}

+		delete(unmatchedUserSpecifiedDatasets, path.ToString())
+
 		pass, filterErr := filter.Filter(path)
 		if filterErr != nil {
 			return nil, fmt.Errorf("error calling filter: %s", filterErr)
@@ -87,5 +99,9 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert

 	}

+	jobid := zfscmd.GetJobIDOrDefault(ctx, "__nojobid")
+	metric := prom.ZFSListUnmatchedUserSpecifiedDatasetCount.WithLabelValues(jobid)
+	metric.Add(float64(len(unmatchedUserSpecifiedDatasets)))
+
 	return
 }
@@ -3,10 +3,11 @@ package zfs
 import "github.com/prometheus/client_golang/prometheus"

 var prom struct {
-	ZFSListFilesystemVersionDuration *prometheus.HistogramVec
-	ZFSSnapshotDuration              *prometheus.HistogramVec
-	ZFSBookmarkDuration              *prometheus.HistogramVec
-	ZFSDestroyDuration               *prometheus.HistogramVec
+	ZFSListFilesystemVersionDuration          *prometheus.HistogramVec
+	ZFSSnapshotDuration                       *prometheus.HistogramVec
+	ZFSBookmarkDuration                       *prometheus.HistogramVec
+	ZFSDestroyDuration                        *prometheus.HistogramVec
+	ZFSListUnmatchedUserSpecifiedDatasetCount *prometheus.GaugeVec
 }

 func init() {
@@ -34,6 +35,15 @@ func init() {
 		Name:      "destroy_duration",
 		Help:      "Duration it took to destroy a dataset",
 	}, []string{"dataset_type", "filesystem"})
+	prom.ZFSListUnmatchedUserSpecifiedDatasetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfs",
+		Name:      "list_unmatched_user_specified_dataset_count",
+		Help: "When evaluating a DatsetFilter against zfs list output, this counter " +
+			"is incremented for every DatasetFilter rule that did not match any " +
+			"filesystem name in the zfs list output. Monitor for increases to detect filesystem " +
+			"filter rules that have no effect because they don't match any local filesystem.",
+	}, []string{"jobid"})
 }

 func PrometheusRegister(registry prometheus.Registerer) error {
@@ -49,5 +59,8 @@ func PrometheusRegister(registry prometheus.Registerer) error {
 	if err := registry.Register(prom.ZFSDestroyDuration); err != nil {
 		return err
 	}
+	if err := registry.Register(prom.ZFSListUnmatchedUserSpecifiedDatasetCount); err != nil {
+		return err
+	}
 	return nil
 }
@@ -19,6 +19,10 @@ func WithJobID(ctx context.Context, jobID string) context.Context {
 	return context.WithValue(ctx, contextKeyJobID, jobID)
 }

+func GetJobIDOrDefault(ctx context.Context, def string) string {
+	return getJobIDOrDefault(ctx, def)
+}
+
 func getJobIDOrDefault(ctx context.Context, def string) string {
 	ret, ok := ctx.Value(contextKeyJobID).(string)
 	if !ok {
Author	SHA1	Message	Date
Christian Schwarz	d8d1d25ec2	docs: changelog for 0.6.1	2023-09-10 11:13:14 +00:00
Christian Schwarz	d02d7e5e1d	address updated golangci-lint errors: S1011 (gosimple) should replace loop with `copy.outs[level] = append(copy.outs[level], os.outs[level]...)` (gosimple)	2023-09-10 11:12:46 +00:00
Christian Schwarz	39f8ff62f0	address updated golangci-lint errors: ineffectual assignment to err (ineffassign)	2023-09-10 11:12:46 +00:00
Christian Schwarz	9a434b0e54	go1.21: update golangci-lint (current version panics on go 1.21) Command used: ``` cd build go get -u github.com/golangci/golangci-lint/cmd/golangci-lint go mod tidy ``` Further, golangci-lint requires go 1.20 to build, so, use that as the lowest version in the CI.	2023-09-10 11:12:46 +00:00
Christian Schwarz	b5053d2659	build: use Go 1.21	2023-09-10 10:19:23 +00:00
Christian Schwarz	0fe2ac6b90	debian packaging: make it work on non-x86_64 hosts (arm64 builder, specifically)	2023-09-10 10:12:20 +00:00
Christian Schwarz	95c924968a	circleci: migrate to scheduled pipelines https://circleci.com/docs/migrate-scheduled-workflows-to-scheduled-pipelines/	2023-09-09 11:55:04 +00:00
Christian Schwarz	523a3bb26b	build: address breakage by golang:1.19 Docker image switching to bookworm Debian bookworm apparently _requires_ pip to be used in venv, at least when we use it inside the build.Dockerfile. So, do that.	2023-09-09 11:55:04 +00:00
Christian Schwarz	96396b2e86	circleci: fixup `bc92660`: docs/publish.sh script -P option didn't work forgot to add it to getopt	2023-09-09 11:08:03 +00:00
Sven Kirmess	8749f0bd3d	docs: talks: add note on keep_bookmarks option (#687 ) Mention that the keep_bookmarks option was removed since the talks were given.	2023-09-09 10:47:28 +00:00
Christian Schwarz	bc92660e09	circleci: ensure docs/publish.sh works as part of pre-merge `ci` workflow (#736 )	2023-09-09 12:41:06 +02:00
Christian Schwarz	8b0637ddcc	docs: switch to sphinx-multiversion for multi-versioned docs (#734 ) The sphinxcontrib-versioning seems unmaintainted and I can't get the fork that we used before this PR working on Python 3.10. The situation wrt maintenance doesn't seem much better for sphinx-multiversion, but, at least I could get it to work with current sphinx versions. The main problem with sphinx-multiversion is that it doesn't render anything at `/`. I.e., `https://zrepl.github.io/configuration.html` will be 404. That's different from `sphinxcontrib-versioning`, and thus switching to sphinx-multiversion would break URLs. We host on GitHub pages and don't control the webserver, so, we can't use webserver-level redirects to keep the URLs working. We could create JS-level redirects, or `http-equiv`, but that's ugly as well. The simplest solution was to fork sphinx-multiversion and hard-code zrepl's specific needs into that fork. The fork is based off v0.2.4 and pinned via requirements.txt. Here are its unique commits: https://github.com/Holzhaus/sphinx-multiversion/compare/master...zrepl:sphinx-multiversion:zrepl We should revisit `sphinx-polyversion` in the future once its docs improve. See https://github.com/Holzhaus/sphinx-multiversion/issues/88#issuecomment-1606221194 This PR updates the various Python packages, as I couldn't get sphinx-multiversion to work with the (very old) versions that were pinned in `requirements.txt` prior to this PR. This PR's `requirements.txt` is from a clean Python 3.10 venv on Ubuntu 22.10 after running ``` pip install sphinx sphinx-rtd-theme pip install 'git+https://github.com/zrepl/sphinx-multiversion/@52c915d7ad898d9641ec48c8bbccb7d4f079db93#egg=sphinx_multiversion' ```	2023-09-09 12:21:25 +02:00
Christian Schwarz	bbdc6f5465	fix handling of tenative cursor presence if protection strategy doesn't use it (#714 ) Before this PR, we would panic in the `check` phase of `endpoint.Send()`'s `TryBatchDestroy` call in the following cases: the current protection strategy does NOT produce a tentative replication cursor AND * `FromVersion` is a tentative cursor bookmark * `FromVersion` is a snapshot, and there exists a tentative cursor bookmark for that snapshot * `FromVersion` is a bookmark != tentative cursor bookmark, but there exists a tentative cursor bookmark for the same snapshot as the `FromVersion` bookmark In those cases, the `check` concluded that we would delete `FromVersion`. It came to that conclusion because the tentative cursor isn't part of `obsoleteAbs` if the protection strategy doesn't produce a tentative replication cursor. The scenarios above can happen if the user changes the protection strategy from "with tentative cursor" to one "without tentative replication cursor", while there is a tentative replication cursor on disk. The workaround was to rename the tentative cursor. In all cases above, `TryBatchDestroy` would have destroyed the tentative cursor. In case 1, that would fail the `Send` step and potentially break replication if the cursor is the last common bookmark. The `check` conclusion was correct. In cases 2 and 3, deleting the tentative cursor would have been fine because `FromVersion` was a different entity than the tentative cursor. So, destroying the tentative cursor would be the right call. The solution in this PR is as follows: * add the `FromVersion` to the `liveAbs` set of live abstractions * rewrite the `check` closure to use the full dataset path (`fullpath`) to identify the concrete ZFS object instead of the `zfs.FilesystemVersionEqualIdentity`, which is only identified by matching GUID. * Holds have no dataset path and are not the `FromVersion` in any case, so disregard them. fixes #666	2023-07-04 20:21:48 +02:00
Goran Mekic	bc5e1ede04	metric to detect filesystems rules that don't match any local dataset (#653 ) This PR adds a Prometheus counter called `zrepl_zfs_list_unmatched_user_specified_dataset_count`. Monitor for increases of the counter to detect filesystem filter rules that have no effect because they don't match any local filesystem. An example use case for this is the following story: 1. Someone sets up zrepl with `filesystems` filter for `zroot/pg14<`. 2. During the upgrade to Postgres 15, they rename the dataset to `zroot/pg15`, but forget to update the zrepl `filesystems` filter. 3. zrepl will not snapshot / replicate the `zroot/pg15<` datasets. Since `filesystems` rules are always evaluated on the side that has the datasets, we can smuggle this functionality into the `zfs` module's `ZFSList` function that is used by all jobs with a `filesystems` filter. Dashboard changes: - histogram with increase in $__interval, one row per job - table with increase in $__range - explainer text box, so, people know what the previous two are about We had to re-arrange some panels, hence the Git diff isn't great. closes https://github.com/zrepl/zrepl/pull/653 Co-authored-by: Christian Schwarz <me@cschwarz.com> Co-authored-by: Goran Mekić <meka@tilda.center>	2023-05-02 22:13:52 +02:00
Tercio Filho	2b3daaf9f1	zrepl status: hide progress bar once all filesystems reach terminal state (#674 ) * Added `IsTerminal` method * Made rendering of progress bar conditional based on IsTerminal	2023-05-02 19:28:56 +02:00
Sebastian Jäger	2b3df7e342	docs: address setup with two or more external disks (#695 )	2023-05-02 18:57:26 +02:00
Christian Schwarz	5e4d4188f4	circleci: use orb circlci/go for module caching	2023-02-26 13:08:05 +01:00
Christian Schwarz	1e8ffe4486	circleci: run platform tests in CircleCI	2023-02-26 13:08:05 +01:00
Christian Schwarz	59389b84a2	platformtest: fix logmockzfs wrapper script / make test-platform for Go 1.19 See the comment in the script. refs https://github.com/golang/go/issues/53962 used by make test-platform breaks the test on Go 1.19	2023-02-26 13:08:05 +01:00
Christian Schwarz	4fae0bb68e	grafana: update dashboard to Grafana 9.3.6 ... by importing the old version of the dashboard JSON into Grafana 9.3.6, then re-exporting it.	2023-02-26 11:28:57 +01:00
Guillermo Ramos	9777a441e9	dist: add openrc service file closes https://github.com/zrepl/zrepl/pull/664	2023-01-27 23:59:45 +01:00
InsanePrawn	1a72edea5d	docs/jobs: add replication- conflict_resolution-options to active job types	2023-01-26 00:09:28 +01:00
Christian Schwarz	96db636582	build: circleci: don't trigger periodic full pipeline build for problame/circleci-build	2023-01-08 12:35:59 +01:00
Christian Schwarz	190ab7c08d	build: circleci: stop using minio for artifact storage CircleCI artifacts are available publicly. And regarding expiration of artifacts, it doesn't really matter because I delete minio artifacts after 30d as well.	2022-12-30 14:24:23 +01:00
Christian Schwarz	6be133f55d	remove unused JobDebugSettings along with docs For this kind of debugging, we switched to env vars a while ago. For example, ZREPL_RPC_DEBUG. I don't think we have a substitute for the RPCLog stuff. However, NetConnLogger is still in the codebase. obsoletes https://github.com/zrepl/zrepl/pull/661	2022-12-22 18:13:45 +01:00
Christian Schwarz	5ffd470596	docs: update comment on overriding mountpoint properties during zfs recv of ZVOLs fixes https://github.com/zrepl/zrepl/issues/430	2022-12-10 12:53:24 +01:00
Christian Schwarz	2119dc40ab	docs: update supporters list	2022-12-10 12:00:57 +01:00
Christian Schwarz	0df1c4cdcc	docs: changelog: move donation banner to 0.6 release	2022-11-01 09:57:24 +01:00