build: fix rpm builds, broken by ef9a63b: support package revisions

fixup of 9d5c892023

.circleci/config.yml

@@ -1,5 +1,8 @@
 version: 2.1
-
+orbs:
+  # NB: 1.7.2 is not the Go version, but the Orb version
+  # https://circleci.com/developer/orbs/orb/circleci/go#usage-go-modules-cache
+  go: circleci/go@1.7.2
 commands:
   setup-home-local-bin:
     steps:
@@ -24,18 +27,12 @@ commands:
 
   apt-update-and-install-common-deps:
     steps:
-      - run: sudo apt update && sudo apt install gawk make
+      - run: sudo apt-get update
-
+      - run: sudo apt-get install -y gawk make
-  restore-cache-gomod:
+      # CircleCI doesn't update its cimg/go images.
-    steps:
+      # So, need to update manually to get up-to-date trust chains.
-      - restore_cache:
+      # The need for this was required for cimg/go:1.12, but let's future proof this here and now.
-          key: go-mod-v4-{{ checksum "go.sum" }}
+      - run: sudo apt-get install -y git ca-certificates
-  save-cache-gomod:
-    steps:
-      - save_cache:
-          key: go-mod-v4-{{ checksum "go.sum" }}
-          paths:
-            - "/go/pkg/mod"
 
   install-godep:
     steps:
@@ -50,94 +47,41 @@ commands:
       - invoke-lazy-sh:
           subcommand: docdep
 
-  download-and-install-minio-client:
+  docs-publish-sh:
-    steps:
-      - setup-home-local-bin
-      - restore_cache:
-          key: minio-client-v2
-      - run:
-          shell: /bin/bash -eo pipefail
-          command: |
-            if which mc; then exit 0; fi
-            sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
-              -o "$HOME/.local/bin/mc"
-            sudo chmod +x "$HOME/.local/bin/mc"
-      - save_cache:
-          key: minio-client-v2
-          paths:
-            - "$HOME/.local/bin/mc"
-
-  upload-minio:
     parameters:
-      src:
+      push:
-        type: string
+        type: boolean
-      dst:
-        type: string
     steps:
+      - checkout
       - run:
-          shell: /bin/bash -eo pipefail
-          when: always
           command: |
-            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
+            git config --global user.email "zreplbot@cschwarz.com"
-              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
+            git config --global user.name "zrepl-github-io-ci"
-              exit 0
-            fi
-            set -u # from now on
 
-            mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
+      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
-
+      - run: ssh-add -D
-            # keep in sync with set-github-minio-status
+      # the default circleci ssh config only additional ssh keys for Host !github.com
-            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
-
-            # Upload artifacts
-            mkdir -p ./artifacts
-            mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
-
-  set-github-minio-status:
-    parameters:
-      context:
-        type: string
-      description:
-        type: string
-      minio-dst:
-        type: string
-    steps:
       - run:
-          shell: /bin/bash -eo pipefail
           command: |
-            if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
+            cat > ~/.ssh/config \<<EOF
-              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
+            Host *
-              exit 0
+              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
-            fi
+            EOF
-            set -u # from now on
+      - add_ssh_keys:
+          fingerprints:
+            # deploy key for zrepl.github.io
+            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
 
-            # keep in sync with with upload-minio command
+      # caller must install-docdep
-            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
+      - when:
-            # Push Artifact Link to GitHub
+          condition: << parameters.push >>
-            REPO="zrepl/zrepl"
+          steps:
-            COMMIT="${CIRCLE_SHA1}"
+            - run: bash -x docs/publish.sh -c -a -P
-            JOB_NAME="${CIRCLE_JOB}"
+      - when:
-            CONTEXT="<<parameters.context>>"
+          condition:
-            DESCRIPTION="<<parameters.description>>"
+            not: << parameters.push >>
-            TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
+          steps:
-            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
+            - run: bash -x docs/publish.sh -c -a
-              -H "Content-Type: application/json" \
-              -H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
-              -X POST \
-              -d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
-
-
-  trigger-pipeline:
-    parameters:
-      body_no_shell_subst:
-        type: string
-    steps:
-      - run: |
-          curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
-            -H 'Content-Type: application/json' \
-            -H 'Accept: application/json' \
-            -H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
-            --data '<<parameters.body_no_shell_subst>>'
 
 parameters:
   do_ci:
@@ -150,7 +94,7 @@ parameters:
 
   release_docker_baseimage_tag:
     type: string
-    default: "1.17"
+    default: "1.21"
 
 workflows:
   version: 2
@@ -160,19 +104,19 @@ workflows:
     jobs:
       - quickcheck-docs
       - quickcheck-go: &quickcheck-go-smoketest
-          name: quickcheck-go-amd64-linux-1.17
+          name: quickcheck-go-amd64-linux-1.21
-          goversion: &latest-go-release "1.17"
+          goversion: &latest-go-release "1.21"
           goos: linux
           goarch: amd64
       - test-go-on-latest-go-release:
           goversion: *latest-go-release
       - quickcheck-go:
           requires:
-            - quickcheck-go-amd64-linux-1.17 #quickcheck-go-smoketest.name
+            - quickcheck-go-amd64-linux-1.21 #quickcheck-go-smoketest.name
           matrix: &quickcheck-go-matrix
             alias: quickcheck-go-matrix
             parameters:
-              goversion: [*latest-go-release, "1.12"]
+              goversion: [*latest-go-release, "1.20"]
               goos: ["linux", "freebsd"]
               goarch: ["amd64", "arm64"]
             exclude:
@@ -180,10 +124,14 @@ workflows:
               - goversion: *latest-go-release
                 goos: linux
                 goarch: amd64
-              # not supported by Go 1.12
+      - platformtest:
-              - goversion: "1.12"
+          matrix:
-                goos: freebsd
+            parameters:
-                goarch: arm64
+              goversion: [*latest-go-release]
+              goos: ["linux"]
+              goarch: ["amd64"]
+          requires:
+            - quickcheck-go-<< matrix.goarch >>-<< matrix.goos >>-<< matrix.goversion >>
 
   release:
     when: << pipeline.parameters.do_release >>
@@ -201,20 +149,7 @@ workflows:
             - release-deb
             - release-rpm
 
-  periodic:
+  publish-zrepl.github.io:
-    triggers:
-      - schedule:
-          cron: "00 17 * * *"
-          filters:
-            branches:
-              only:
-                - master
-                - stable
-                - problame/circleci-build
-    jobs:
-      - periodic-full-pipeline-run
-
-  zrepl.github.io:
     jobs:
       - publish-zrepl-github-io:
           filters:
@@ -225,20 +160,15 @@ workflows:
 jobs:
   quickcheck-docs:
     docker:
-      - image: cimg/base:2020.08
+      - image: cimg/base:2023.09
     steps:
       - checkout
       - install-docdep
+      # do the current docs build
       - run: make docs
-
+      # does the publish.sh script still work?
-      - download-and-install-minio-client
+      - docs-publish-sh:
-      - upload-minio:
+          push: false
-          src: artifacts
-          dst: ""
-      - set-github-minio-status:
-          context: artifacts/${CIRCLE_JOB}
-          description: artifacts of CI job ${CIRCLE_JOB}
-          minio-dst: ""
 
   quickcheck-go:
     parameters:
@@ -249,7 +179,7 @@ jobs:
       goarch:
         type: string
     docker:
-      - image: circleci/golang:<<parameters.goversion>>
+      - image: cimg/go:<<parameters.goversion>>
     environment:
       GOOS: <<parameters.goos>>
       GOARCH: <<parameters.goarch>>
@@ -257,41 +187,62 @@ jobs:
     steps:
       - checkout
 
-      - restore-cache-gomod
+      - go/load-cache:
+          key: quickcheck-<<parameters.goversion>>
+      - install-godep
       - run: go mod download
       - run: cd build && go mod download
-      - save-cache-gomod
+      - go/save-cache:
+          key: quickcheck-<<parameters.goversion>>
 
-      - install-godep
       - run: make formatcheck
       - run: make generate-platform-test-list
       - run: make zrepl-bin test-platform-bin
       - run: make vet
       - run: make lint
 
-      - download-and-install-minio-client
       - run: rm -f artifacts/generate-platform-test-list
       - store_artifacts:
           path: artifacts
-      - upload-minio:
+      - persist_to_workspace:
-          src: artifacts
+          root: .
-          dst: ""
+          paths: [.]
-      - set-github-minio-status:
+
-          context: artifacts/${CIRCLE_JOB}
+  platformtest:
-          description: artifacts of CI job ${CIRCLE_JOB}
+    parameters:
-          minio-dst: ""
+      goversion:
+        type: string
+      goos:
+        type: string
+      goarch:
+        type: string
+    machine:
+      image: ubuntu-2204:current
+    resource_class: medium
+    environment:
+      GOOS: <<parameters.goos>>
+      GOARCH: <<parameters.goarch>>
+    steps:
+      - attach_workspace:
+          at: .
+      - run: sudo apt-get update
+      - run: sudo apt-get install -y zfsutils-linux
+      - run: sudo zfs version
+      - run: sudo make test-platform GOOS="$GOOS" GOARCH="$GOARCH"
 
   test-go-on-latest-go-release:
     parameters:
       goversion:
         type: string
     docker:
-      - image: circleci/golang:<<parameters.goversion>>
+      - image: cimg/go:<<parameters.goversion>>
     steps:
       - checkout
-      - restore-cache-gomod
+      - go/load-cache:
+          key: make-test-go
       - run: make test-go
-      # don't save-cache-gomod here, test-go doesn't pull all the dependencies
+      - go/save-cache:
+          key: make-test-go
 
   release-build:
     machine:
@@ -332,48 +283,15 @@ jobs:
     steps:
       - attach_workspace:
           at: .
+      - run: make wrapup-and-checksum
       - store_artifacts:
           path: artifacts
-      - download-and-install-minio-client
-      - upload-minio:
-          src: artifacts
-          dst: ""
-      - set-github-minio-status:
-          context: artifacts/release
-          description: CI-generated release artifacts
-          minio-dst: ""
-
-  periodic-full-pipeline-run:
-    docker:
-      - image: cimg/base:2020.08
-    steps:
-      - trigger-pipeline:
-          body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
 
   publish-zrepl-github-io:
     docker:
-      - image: cimg/python:3.7
+      - image: cimg/base:2023.09
     steps:
       - checkout
-      - invoke-lazy-sh:
+      - install-docdep
-          subcommand: docdep
+      - docs-publish-sh:
-      - run:
+          push: true
-          command: |
-            git config --global user.email "zreplbot@cschwarz.com"
-            git config --global user.name "zrepl-github-io-ci"
-
-      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
-      - run: ssh-add -D
-      # the default circleci ssh config only additional ssh keys for Host !github.com
-      - run:
-          command: |
-            cat > ~/.ssh/config \<<EOF
-            Host *
-              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
-            EOF
-      - add_ssh_keys:
-          fingerprints:
-            # deploy key for zrepl.github.io
-            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
-
-      - run: bash -x docs/publish.sh -c -a

.circleci/download_artifacts.py

@@ -0,0 +1,84 @@
+import argparse
+from pathlib import Path
+import re
+
+import requests
+
+import time
+
+import os
+import argparse
+from pathlib import Path
+
+
+circle_token = os.environ.get('CIRCLE_TOKEN')
+if not circle_token:
+    raise ValueError('CIRCLE_TOKEN environment variable not set')
+
+parser = argparse.ArgumentParser(description='Download artifacts from CircleCI')
+parser.add_argument('build_num', type=str, help='Build number')
+parser.add_argument('dst', type=Path, help='Destination directory')
+parser.add_argument('--prefix', type=str, default='', help='Filter for prefix')
+parser.add_argument('--match', type=str, default='.*', help='Only include paths matching the given regex')
+
+args = parser.parse_args()
+
+res = requests.get(
+    f"https://circleci.com/api/v1.1/project/github/zrepl/zrepl/{args.build_num}/artifacts",
+    headers={
+        "Circle-Token": circle_token,
+    },
+)
+res.raise_for_status()
+
+# https://circleci.com/docs/api/v1/index.html#artifacts-of-a-job
+# [ {
+#   "path" : "raw-test-output/go-test-report.xml",
+#   "pretty_path" : "raw-test-output/go-test-report.xml",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test-report.xml"
+# }, {
+#   "path" : "raw-test-output/go-test.out",
+#   "pretty_path" : "raw-test-output/go-test.out",
+#   "node_index" : 0,
+#   "url" : "https://24-88881093-gh.circle-artifacts.com/0/raw-test-output/go-test.out"
+# } ]
+res = res.json()
+
+for artifact in res:
+    if not artifact["pretty_path"].startswith(args.prefix):
+        continue
+    if not re.match(args.match, artifact["pretty_path"]):
+        continue
+    stripped = artifact["pretty_path"][len(args.prefix):]
+    print(f"Downloading {artifact['pretty_path']} to {args.dst / stripped}")
+    artifact_rel = Path(stripped)
+    artifact_dst = args.dst / artifact_rel
+    artifact_dst.parent.mkdir(parents=True, exist_ok=True)
+
+    res = requests.get(
+        artifact["url"],
+        headers={
+            "Circle-Token": circle_token,
+        },
+        stream=True,
+    )
+    res.raise_for_status()
+
+    total_size = int(res.headers.get("Content-Length", 0))
+    block_size = 128 * 1024
+    with open(artifact_dst, "wb") as f:
+        progress = 0
+        start_time = time.time()
+        for chunk in res.iter_content(chunk_size=block_size):
+            f.write(chunk)
+            progress += len(chunk)
+            percent = progress / total_size * 100
+            elapsed_time = time.time() - start_time
+            if elapsed_time >= 5:
+                print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)", end="\r")
+                start_time = time.time()
+        print(f"Downloaded {progress}/{total_size} bytes ({percent:.2f}%)")
+    print("Download complete!")
+
+print("All files downloaded")

Makefile

@@ -1,5 +1,5 @@
 .PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
-.PHONY: release bins-all release-noarch
+.PHONY: release release-noarch
 .DEFAULT_GOAL := zrepl-bin
 
 ARTIFACTDIR := artifacts
@@ -14,13 +14,15 @@ ifndef _ZREPL_VERSION
     endif
 endif
 
+ZREPL_PACKAGE_RELEASE := 1
+
 GO := go
 GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
 GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
 GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
 GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
 GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
-GO_ENV_VARS := GO111MODULE=on
+GO_ENV_VARS := GO111MODULE=on CGO_ENABLED=0
 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
 GO_MOD_READONLY := -mod=readonly
 GO_EXTRA_BUILDFLAGS :=
@@ -28,7 +30,7 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
 GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
 GOLANGCI_LINT := golangci-lint
 GOCOVMERGE := gocovmerge
-RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.17
+RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.21
 RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
 
 ifneq ($(GOARM),)
@@ -50,21 +52,20 @@ printvars:
 release: clean
 	# no cross-platform support for target test
 	$(MAKE) test-go
-	$(MAKE) bins-all
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="vet"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="lint"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="zrepl-bin"
+	$(MAKE) _run_make_foreach_target_tuple RUN_MAKE_FOREACH_TARGET_TUPLE_ARG="test-platform-bin"
 	$(MAKE) noarch
-	$(MAKE) wrapup-and-checksum
-	$(MAKE) check-git-clean
-ifeq (SIGN, 1)
-	$(MAKE) sign
-endif
-	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
 
 release-docker: $(ARTIFACTDIR)
 	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
 	docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
 	docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
 		zrepl_release \
-		make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make release \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
 
 debs-docker:
 	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
@@ -81,9 +82,14 @@ rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion
 	$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
 	rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
 	mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
-	mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
+	for d in BUILD BUILDROOT RPMS SOURCES SPECS SRPMS; do \
-	sed "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+		mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)/$$d"; \
-	    packaging/rpm/zrepl.spec >  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
+	done
+	sed \
+		-e "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+		-e "s/^Release:.*/Release:          $(ZREPL_PACKAGE_RELEASE)/g" \
+	    packaging/rpm/zrepl.spec \
+		>  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
 
 	# see /usr/lib/rpm/platform
 ifeq ($(GOARCH),amd64)
@@ -110,13 +116,16 @@ rpm-docker:
 	docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
 	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
 		zrepl_rpm_pkg \
-		make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make rpm \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
+
 
 deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
 
 	cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
 	sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
-	VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
+	VERSION="$(subst -,.,$(_ZREPL_VERSION))-$(ZREPL_PACKAGE_RELEASE)"; \
 		export VERSION="$${VERSION#v}"; \
 		sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
 
@@ -134,11 +143,19 @@ endif
 
 deb-docker:
 	docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
+	# Use a small open file limit to make fakeroot work. If we don't
+	# specify it, docker daemon will use its file limit. I don't know
+	# what changed (Docker, its systemd service, its Go version). But I
+	# observed fakeroot iterating close(i) up to i > 1000000, which costs
+	# a good amount of CPU time and makes the build slow.
 	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		--ulimit nofile=1024:1024 \
 		zrepl_debian_pkg \
-		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+		make deb \
+			GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM) \
+			ZREPL_VERSION=$(ZREPL_VERSION) ZREPL_PACKAGE_RELEASE=$(ZREPL_PACKAGE_RELEASE)
 
-# expects `release` target to have run before
+# expects `release`, `deb` & `rpm` targets to have run before
 NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
 wrapup-and-checksum:
 	rm -f $(NOARCH_TARBALL)
@@ -155,7 +172,7 @@ wrapup-and-checksum:
 		config/samples
 	rm -rf "$(ARTIFACTDIR)/release"
 	mkdir -p "$(ARTIFACTDIR)/release"
-	cp -l $(ARTIFACTDIR)/zrepl-* \
+	cp -l $(ARTIFACTDIR)/zrepl* \
 		$(ARTIFACTDIR)/platformtest-* \
 		"$(ARTIFACTDIR)/release"
 	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
@@ -175,39 +192,42 @@ check-git-clean:
 
 tag-release:
 	test -n "$(ZREPL_TAG_VERSION)" || exit 1
-	git tag -u E27CA5FC -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
+	git tag -u '328A6627FA98061D!' -m "$(ZREPL_TAG_VERSION)" "$(ZREPL_TAG_VERSION)"
 
 sign:
-	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+	gpg -u '328A6627FA98061D!' \
 		--armor \
 		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
 
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"
 
-##################### BINARIES #####################
+download-circleci-release:
-.PHONY: bins-all lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
+	rm -rf "$(ARTIFACTDIR)"
+	mkdir -p "$(ARTIFACTDIR)/release"
+	python3 .circleci/download_artifacts.py --prefix 'artifacts/release/' "$(BUILD_NUM)" "$(ARTIFACTDIR)/release"
 
-BINS_ALL_TARGETS := zrepl-bin test-platform-bin vet lint
+##################### MULTI-ARCH HELPERS #####################
-GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
+
-bins-all:
+_run_make_foreach_target_tuple:
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
+	if [ "$(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG)" = "" ]; then \
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
+		echo "RUN_MAKE_FOREACH_TARGET_TUPLE_ARG must be set"; \
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm	  GOARM=7
+		exit 1; \
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=arm64
+	fi
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=386
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=arm	  GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=freebsd   GOARCH=arm64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=amd64
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=arm64
-ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=arm     GOARM=7
-	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=linux     GOARCH=386
-else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=darwin    GOARCH=amd64
-	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=solaris   GOARCH=amd64
-else
+	$(MAKE) $(RUN_MAKE_FOREACH_TARGET_TUPLE_ARG) GOOS=illumos   GOARCH=amd64
-	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
+
-endif
+##################### REGULAR TARGETS #####################
+.PHONY: lint test-go test-platform cover-merge cover-html vet zrepl-bin test-platform-bin generate-platform-test-list
 
 lint:
 	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...

build.Dockerfile

@@ -2,12 +2,18 @@ FROM !SUBSTITUTED_BY_MAKEFILE
 
 RUN apt-get update && apt-get install -y \
     python3-pip \
+    python3-venv \
     unzip \
     gawk
 
 ADD build.installprotoc.bash ./
 RUN bash build.installprotoc.bash
 
+# setup venv
+ENV VIRTUAL_ENV=/opt/venv
+RUN python3 -m venv $VIRTUAL_ENV
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
 ADD lazy.sh /tmp/lazy.sh
 ADD docs/requirements.txt /tmp/requirements.txt
 ENV ZREPL_LAZY_DOCS_REQPATH=/tmp/requirements.txt

build/go.mod

@@ -4,13 +4,37 @@ go 1.12
 
 require (
 	github.com/alvaroloes/enumer v1.1.1
-	github.com/golangci/golangci-lint v1.35.2
+	github.com/breml/bidichk v0.2.6 // indirect
-	github.com/golangci/misspell v0.3.4 // indirect
+	github.com/breml/errchkjson v0.3.5 // indirect
-	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/chavacava/garif v0.1.0 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
+	github.com/daixiang0/gci v0.11.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/golangci/golangci-lint v1.54.2
+	github.com/golangci/revgrep v0.5.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/jgautheron/goconst v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/mgechev/revive v1.3.3 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/polyfloyd/go-errorlint v1.4.5 // indirect
+	github.com/prometheus/client_golang v1.16.0 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/rivo/uniseg v0.4.4 // indirect
+	github.com/ryanrolds/sqlclosecheck v0.5.1 // indirect
+	github.com/spf13/viper v1.16.0 // indirect
+	github.com/stretchr/objx v0.5.1 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tetafro/godot v1.4.15 // indirect
 	github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad
-	golang.org/x/tools v0.0.0-20210105210202-9ed45478a130
+	github.com/xen0n/gosmopolitan v1.2.2 // indirect
-	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0 // indirect
+	gitlab.com/bosi/decorder v0.4.1 // indirect
-	google.golang.org/protobuf v1.25.0
+	go.uber.org/multierr v1.11.0 // indirect
+	go.uber.org/zap v1.25.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/tools v0.13.0
+	google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0
+	google.golang.org/protobuf v1.31.0
+	mvdan.cc/unparam v0.0.0-20230815095028-f7c6fb1088f0 // indirect
 )

client/status/viewmodel/render.go

@@ -374,13 +374,15 @@ func renderReplicationReport(t *stringbuilder.B, rep *report.Report, history *by
 			eta = time.Duration((float64(expected)-float64(replicated))/float64(rate)) * time.Second
 		}
 
-		t.Write("Progress: ")
+		if !latest.State.IsTerminal() {
-		t.DrawBar(50, replicated, expected, changeCount)
+			t.Write("Progress: ")
-		t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
+			t.DrawBar(50, replicated, expected, changeCount)
-		if eta != 0 {
+			t.Write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinaryUint(replicated), ByteCountBinaryUint(expected), ByteCountBinary(rate)))
-			t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+			if eta != 0 {
+				t.Write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+			}
+			t.Newline()
 		}
-		t.Newline()
 		if containsInvalidSizeEstimates {
 			t.Write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
 			t.Newline()
@@ -493,15 +495,17 @@ func renderPrunerReport(t *stringbuilder.B, r *pruner.Report, fsfilter FilterFun
 	}
 
 	// global progress bar
-	progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
+	if !state.IsTerminal() {
-	t.Write("Progress: ")
+		progress := int(math.Round(80 * float64(completedDestroyCount) / float64(totalDestroyCount)))
-	t.Write("[")
+		t.Write("Progress: ")
-	t.Write(stringbuilder.Times("=", progress))
+		t.Write("[")
-	t.Write(">")
+		t.Write(stringbuilder.Times("=", progress))
-	t.Write(stringbuilder.Times("-", 80-progress))
+		t.Write(">")
-	t.Write("]")
+		t.Write(stringbuilder.Times("-", 80-progress))
-	t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
+		t.Write("]")
-	t.Newline()
+		t.Printf(" %d/%d snapshots", completedDestroyCount, totalDestroyCount)
+		t.Newline()
+	}
 
 	sort.SliceStable(all, func(i, j int) bool {
 		return strings.Compare(all[i].Filesystem, all[j].Filesystem) == -1

config/config.go

@@ -6,8 +6,6 @@ import (
 	"log/syslog"
 	"os"
 	"reflect"
-	"regexp"
-	"strconv"
 	"time"
 
 	"github.com/pkg/errors"
@@ -67,7 +65,6 @@ type ActiveJob struct {
 	Name               string                `yaml:"name"`
 	Connect            ConnectEnum           `yaml:"connect"`
 	Pruning            PruningSenderReceiver `yaml:"pruning"`
-	Debug              JobDebugSettings      `yaml:"debug,optional"`
 	Replication        *Replication          `yaml:"replication,optional,fromdefaults"`
 	ConflictResolution *ConflictResolution   `yaml:"conflict_resolution,optional,fromdefaults"`
 }
@@ -77,17 +74,15 @@ type ConflictResolution struct {
 }
 
 type PassiveJob struct {
-	Type  string           `yaml:"type"`
+	Type  string    `yaml:"type"`
-	Name  string           `yaml:"name"`
+	Name  string    `yaml:"name"`
-	Serve ServeEnum        `yaml:"serve"`
+	Serve ServeEnum `yaml:"serve"`
-	Debug JobDebugSettings `yaml:"debug,optional"`
 }
 
 type SnapJob struct {
 	Type         string            `yaml:"type"`
 	Name         string            `yaml:"name"`
 	Pruning      PruningLocal      `yaml:"pruning"`
-	Debug        JobDebugSettings  `yaml:"debug,optional"`
 	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
 }
@@ -190,13 +185,10 @@ func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error
 		return fmt.Errorf("value must not be empty")
 	default:
 		i.Manual = false
-		i.Interval, err = time.ParseDuration(s)
+		i.Interval, err = parsePositiveDuration(s)
 		if err != nil {
 			return err
 		}
-		if i.Interval <= 0 {
-			return fmt.Errorf("value must be a positive duration, got %q", s)
-		}
 	}
 	return nil
 }
@@ -228,10 +220,11 @@ type SnapshottingEnum struct {
 }
 
 type SnapshottingPeriodic struct {
-	Type     string        `yaml:"type"`
+	Type            string            `yaml:"type"`
-	Prefix   string        `yaml:"prefix"`
+	Prefix          string            `yaml:"prefix"`
-	Interval time.Duration `yaml:"interval,positive"`
+	Interval        *PositiveDuration `yaml:"interval"`
-	Hooks    HookList      `yaml:"hooks,optional"`
+	Hooks           HookList          `yaml:"hooks,optional"`
+	TimestampFormat string            `yaml:"timestamp_format,optional,default=dense"`
 }
 
 type CronSpec struct {
@@ -260,10 +253,11 @@ func (s *CronSpec) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool)
 }
 
 type SnapshottingCron struct {
-	Type   string   `yaml:"type"`
+	Type            string   `yaml:"type"`
-	Prefix string   `yaml:"prefix"`
+	Prefix          string   `yaml:"prefix"`
-	Cron   CronSpec `yaml:"cron"`
+	Cron            CronSpec `yaml:"cron"`
-	Hooks  HookList `yaml:"hooks,optional"`
+	Hooks           HookList `yaml:"hooks,optional"`
+	TimestampFormat string   `yaml:"timestamp_format,optional,default=dense"`
 }
 
 type SnapshottingManual struct {
@@ -483,14 +477,6 @@ type GlobalStdinServer struct {
 	SockDir string `yaml:"sockdir,default=/var/run/zrepl/stdinserver"`
 }
 
-type JobDebugSettings struct {
-	Conn *struct {
-		ReadDump  string `yaml:"read_dump"`
-		WriteDump string `yaml:"write_dump"`
-	} `yaml:"conn,optional"`
-	RPCLog bool `yaml:"rpc_log,optional,default=false"`
-}
-
 type HookList []HookEnum
 
 type HookEnum struct {
@@ -715,41 +701,3 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
 	}
 	return c, nil
 }
-
-var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
-
-func parsePositiveDuration(e string) (d time.Duration, err error) {
-	comps := durationStringRegex.FindStringSubmatch(e)
-	if len(comps) != 3 {
-		err = fmt.Errorf("does not match regex: %s %#v", e, comps)
-		return
-	}
-
-	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
-	if err != nil {
-		return 0, err
-	}
-	if durationFactor <= 0 {
-		return 0, errors.New("duration must be positive integer")
-	}
-
-	var durationUnit time.Duration
-	switch comps[2] {
-	case "s":
-		durationUnit = time.Second
-	case "m":
-		durationUnit = time.Minute
-	case "h":
-		durationUnit = time.Hour
-	case "d":
-		durationUnit = 24 * time.Hour
-	case "w":
-		durationUnit = 24 * 7 * time.Hour
-	default:
-		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
-		return
-	}
-
-	d = time.Duration(durationFactor) * durationUnit
-	return
-}

config/config_duration.go

@@ -0,0 +1,106 @@
+package config
+
+import (
+	"errors"
+	"fmt"
+	"regexp"
+	"strconv"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/zrepl/yaml-config"
+)
+
+type Duration struct{ d time.Duration }
+
+func (d Duration) Duration() time.Duration { return d.d }
+
+var _ yaml.Unmarshaler = &Duration{}
+
+func (d *Duration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	var s string
+	err := unmarshal(&s, false)
+	if err != nil {
+		return err
+	}
+	d.d, err = parseDuration(s)
+	if err != nil {
+		d.d = 0
+		return &yaml.TypeError{Errors: []string{fmt.Sprintf("cannot parse value %q: %s", s, err)}}
+	}
+	return nil
+}
+
+type PositiveDuration struct{ d Duration }
+
+var _ yaml.Unmarshaler = &PositiveDuration{}
+
+func (d PositiveDuration) Duration() time.Duration { return d.d.Duration() }
+
+func (d *PositiveDuration) UnmarshalYAML(unmarshal func(v interface{}, not_strict bool) error) error {
+	err := d.d.UnmarshalYAML(unmarshal)
+	if err != nil {
+		return err
+	}
+	if d.d.Duration() <= 0 {
+		return fmt.Errorf("duration must be positive, got %s", d.d.Duration())
+	}
+	return nil
+}
+
+func parsePositiveDuration(e string) (time.Duration, error) {
+	d, err := parseDuration(e)
+	if err != nil {
+		return d, err
+	}
+	if d <= 0 {
+		return 0, errors.New("duration must be positive integer")
+	}
+	return d, err
+}
+
+var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*([\+-]?\d+)\s*(|s|m|h|d|w)\s*$`)
+
+func parseDuration(e string) (d time.Duration, err error) {
+	comps := durationStringRegex.FindStringSubmatch(e)
+	if comps == nil {
+		err = fmt.Errorf("must match %s", durationStringRegex)
+		return
+	}
+	if len(comps) != 3 {
+		panic(pretty.Sprint(comps))
+	}
+
+	durationFactor, err := strconv.ParseInt(comps[1], 10, 64)
+	if err != nil {
+		return 0, err
+	}
+
+	var durationUnit time.Duration
+	switch comps[2] {
+	case "":
+		if durationFactor != 0 {
+			err = fmt.Errorf("missing time unit")
+			return
+		} else {
+			// It's the case where user specified '0'.
+			// We want to allow this, just like time.ParseDuration.
+		}
+	case "s":
+		durationUnit = time.Second
+	case "m":
+		durationUnit = time.Minute
+	case "h":
+		durationUnit = time.Hour
+	case "d":
+		durationUnit = 24 * time.Hour
+	case "w":
+		durationUnit = 24 * 7 * time.Hour
+	default:
+		err = fmt.Errorf("contains unknown time unit '%s'", comps[2])
+		return
+	}
+
+	d = time.Duration(durationFactor) * durationUnit
+	return
+}

config/config_snapshotting_test.go

@@ -35,8 +35,23 @@ jobs:
   snapshotting:
     type: periodic
     prefix: zrepl_
+    timestamp_format: dense
     interval: 10m
 `
+	cron := `
+  snapshotting:
+    type: cron
+    prefix: zrepl_
+    timestamp_format: human
+    cron: "10 * * * *"
+`
+
+	periodicDaily := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 1d
+`
 
 	hooks := `
   snapshotting:
@@ -74,10 +89,27 @@ jobs:
 		c = testValidConfig(t, fillSnapshotting(periodic))
 		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
 		assert.Equal(t, "periodic", snp.Type)
-		assert.Equal(t, 10*time.Minute, snp.Interval)
+		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
 		assert.Equal(t, "zrepl_", snp.Prefix)
 	})
 
+	t.Run("periodicDaily", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodicDaily))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 24*time.Hour, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "dense", snp.TimestampFormat)
+	})
+
+	t.Run("cron", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(cron))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
+		assert.Equal(t, "cron", snp.Type)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "human", snp.TimestampFormat)
+	})
+
 	t.Run("hooks", func(t *testing.T) {
 		c = testValidConfig(t, fillSnapshotting(hooks))
 		hs := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic).Hooks
@@ -88,3 +120,57 @@ jobs:
 	})
 
 }
+
+func TestSnapshottingTimestampDefaults(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+
+	periodic := `
+  snapshotting:
+    type: periodic
+    prefix: zrepl_
+    interval: 10m
+`
+	cron := `
+  snapshotting:
+    type: cron
+    prefix: zrepl_
+    cron: "10 * * * *"
+`
+
+	fillSnapshotting := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("periodic", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(periodic))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingPeriodic)
+		assert.Equal(t, "periodic", snp.Type)
+		assert.Equal(t, 10*time.Minute, snp.Interval.Duration())
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly
+	})
+
+	t.Run("cron", func(t *testing.T) {
+		c = testValidConfig(t, fillSnapshotting(cron))
+		snp := c.Jobs[0].Ret.(*PushJob).Snapshotting.Ret.(*SnapshottingCron)
+		assert.Equal(t, "cron", snp.Type)
+		assert.Equal(t, "zrepl_", snp.Prefix)
+		assert.Equal(t, "dense", snp.TimestampFormat) // default was set correctly
+	})
+}

daemon/filters/fsmapfilter.go

@@ -160,6 +160,14 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 	return
 }
 
+func (m DatasetMapFilter) UserSpecifiedDatasets() (datasets zfs.UserSpecifiedDatasetsSet) {
+	datasets = make(zfs.UserSpecifiedDatasetsSet)
+	for i := range m.entries {
+		datasets[m.entries[i].path.ToString()] = true
+	}
+	return
+}
+
 // Construct a new filter-only DatasetMapFilter from a mapping
 // The new filter allows exactly those paths that were not forbidden by the mapping.
 func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {

daemon/pruner/pruner.go

@@ -199,6 +199,16 @@ const (
 	Done
 )
 
+// Returns true in case the State is a terminal state(PlanErr, ExecErr, Done)
+func (s State) IsTerminal() bool {
+	switch s {
+	case PlanErr, ExecErr, Done:
+		return true
+	default:
+		return false
+	}
+}
+
 type updater func(func(*Pruner))
 
 func (p *Pruner) Prune() {

daemon/snapper/cron.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/hooks"
+	"github.com/zrepl/zrepl/util/suspendresumesafetimer"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -20,8 +21,9 @@ func cronFromConfig(fsf zfs.DatasetFilter, in config.SnapshottingCron) (*Cron, e
 		return nil, errors.Wrap(err, "hook config error")
 	}
 	planArgs := planArgs{
-		prefix: in.Prefix,
+		prefix:          in.Prefix,
-		hooks:  hooksList,
+		timestampFormat: in.TimestampFormat,
+		hooks:           hooksList,
 	}
 	return &Cron{config: in, fsf: fsf, planArgs: planArgs}, nil
 }
@@ -42,67 +44,45 @@ type Cron struct {
 
 func (s *Cron) Run(ctx context.Context, snapshotsTaken chan<- struct{}) {
 
-	t := time.NewTimer(0)
-	defer func() {
-		if !t.Stop() {
-			select {
-			case <-t.C:
-			default:
-			}
-		}
-	}()
 	for {
 		now := time.Now()
 		s.mtx.Lock()
 		s.wakeupTime = s.config.Cron.Schedule.Next(now)
 		s.mtx.Unlock()
 
-		// Re-arm the timer.
+		ctxDone := suspendresumesafetimer.SleepUntil(ctx, s.wakeupTime)
-		// Need to Stop before Reset, see docs.
+		if ctxDone != nil {
-		if !t.Stop() {
-			// Use non-blocking read from timer channel
-			// because, except for the first loop iteration,
-			// the channel is already drained
-			select {
-			case <-t.C:
-			default:
-			}
-		}
-		t.Reset(s.wakeupTime.Sub(now))
-
-		select {
-		case <-ctx.Done():
 			return
-		case <-t.C:
-			getLogger(ctx).Debug("cron timer fired")
-			s.mtx.Lock()
-			if s.running {
-				getLogger(ctx).Warn("snapshotting triggered according to cron rules but previous snapshotting is not done; not taking a snapshot this time")
-				s.wakeupWhileRunningCount++
-				s.mtx.Unlock()
-				continue
-			}
-			s.lastError = nil
-			s.lastPlan = nil
-			s.wakeupWhileRunningCount = 0
-			s.running = true
-			s.mtx.Unlock()
-			go func() {
-				err := s.do(ctx)
-				s.mtx.Lock()
-				s.lastError = err
-				s.running = false
-				s.mtx.Unlock()
-
-				select {
-				case snapshotsTaken <- struct{}{}:
-				default:
-					if snapshotsTaken != nil {
-						getLogger(ctx).Warn("callback channel is full, discarding snapshot update event")
-					}
-				}
-			}()
 		}
+
+		getLogger(ctx).Debug("cron timer fired")
+		s.mtx.Lock()
+		if s.running {
+			getLogger(ctx).Warn("snapshotting triggered according to cron rules but previous snapshotting is not done; not taking a snapshot this time")
+			s.wakeupWhileRunningCount++
+			s.mtx.Unlock()
+			continue
+		}
+		s.lastError = nil
+		s.lastPlan = nil
+		s.wakeupWhileRunningCount = 0
+		s.running = true
+		s.mtx.Unlock()
+		go func() {
+			err := s.do(ctx)
+			s.mtx.Lock()
+			s.lastError = err
+			s.running = false
+			s.mtx.Unlock()
+
+			select {
+			case snapshotsTaken <- struct{}{}:
+			default:
+				if snapshotsTaken != nil {
+					getLogger(ctx).Warn("callback channel is full, discarding snapshot update event")
+				}
+			}
+		}()
 	}
 
 }

daemon/snapper/impl.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -14,8 +15,9 @@ import (
 )
 
 type planArgs struct {
-	prefix string
+	prefix          string
-	hooks  *hooks.List
+	timestampFormat string
+	hooks           *hooks.List
 }
 
 type plan struct {
@@ -58,6 +60,21 @@ type snapProgress struct {
 	runResults hooks.PlanReport
 }
 
+func (plan *plan) formatNow(format string) string {
+	now := time.Now().UTC()
+	switch strings.ToLower(format) {
+	case "dense":
+		format = "20060102_150405_000"
+	case "human":
+		format = "2006-01-02_15:04:05"
+	case "iso-8601":
+		format = "2006-01-02T15:04:05.000Z"
+	case "unix-seconds":
+		return strconv.FormatInt(now.Unix(), 10)
+	}
+	return now.Format(format)
+}
+
 func (plan *plan) execute(ctx context.Context, dryRun bool) (ok bool) {
 
 	hookMatchCount := make(map[hooks.Hook]int, len(*plan.args.hooks))
@@ -68,7 +85,7 @@ func (plan *plan) execute(ctx context.Context, dryRun bool) (ok bool) {
 	anyFsHadErr := false
 	// TODO channel programs -> allow a little jitter?
 	for fs, progress := range plan.snaps {
-		suffix := time.Now().In(time.UTC).Format("20060102_150405_000")
+		suffix := plan.formatNow(plan.args.timestampFormat)
 		snapname := fmt.Sprintf("%s%s", plan.args.prefix, suffix)
 
 		ctx := logging.WithInjectedField(ctx, "fs", fs.ToString())

daemon/snapper/periodic.go

@@ -15,6 +15,7 @@ import (
 	"github.com/zrepl/zrepl/daemon/hooks"
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/util/envconst"
+	"github.com/zrepl/zrepl/util/suspendresumesafetimer"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -22,7 +23,7 @@ func periodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.Snap
 	if in.Prefix == "" {
 		return nil, errors.New("prefix must not be empty")
 	}
-	if in.Interval <= 0 {
+	if in.Interval.Duration() <= 0 {
 		return nil, errors.New("interval must be positive")
 	}
 
@@ -32,11 +33,12 @@ func periodicFromConfig(g *config.Global, fsf zfs.DatasetFilter, in *config.Snap
 	}
 
 	args := periodicArgs{
-		interval: in.Interval,
+		interval: in.Interval.Duration(),
 		fsf:      fsf,
 		planArgs: planArgs{
-			prefix: in.Prefix,
+			prefix:          in.Prefix,
-			hooks:  hookList,
+			timestampFormat: in.TimestampFormat,
+			hooks:           hookList,
 		},
 		// ctx and log is set in Run()
 	}
@@ -171,16 +173,13 @@ func periodicStateSyncUp(a periodicArgs, u updater) state {
 	u(func(s *Periodic) {
 		s.sleepUntil = syncPoint
 	})
-	t := time.NewTimer(time.Until(syncPoint))
+	ctxDone := suspendresumesafetimer.SleepUntil(a.ctx, syncPoint)
-	defer t.Stop()
+	if ctxDone != nil {
-	select {
-	case <-t.C:
-		return u(func(s *Periodic) {
-			s.state = Planning
-		}).sf()
-	case <-a.ctx.Done():
 		return onMainCtxDone(a.ctx, u)
 	}
+	return u(func(s *Periodic) {
+		s.state = Planning
+	}).sf()
 }
 
 func periodicStatePlan(a periodicArgs, u updater) state {
@@ -241,17 +240,13 @@ func periodicStateWait(a periodicArgs, u updater) state {
 		logFunc("enter wait-state after error")
 	})
 
-	t := time.NewTimer(time.Until(sleepUntil))
+	ctxDone := suspendresumesafetimer.SleepUntil(a.ctx, sleepUntil)
-	defer t.Stop()
+	if ctxDone != nil {
-
-	select {
-	case <-t.C:
-		return u(func(snapper *Periodic) {
-			snapper.state = Planning
-		}).sf()
-	case <-a.ctx.Done():
 		return onMainCtxDone(a.ctx, u)
 	}
+	return u(func(snapper *Periodic) {
+		snapper.state = Planning
+	}).sf()
 }
 
 func listFSes(ctx context.Context, mf zfs.DatasetFilter) (fss []*zfs.DatasetPath, err error) {

dist/openrc/zrepl

@@ -0,0 +1,23 @@
+#!/sbin/openrc-run
+
+command='/usr/local/bin/zrepl'
+command_args='daemon'
+command_background='true'
+pidfile="/run/${RC_SVCNAME}.pid"
+output_log="/var/log/${RC_SVCNAME}.log"
+error_log="/var/log/${RC_SVCNAME}.log"
+
+zrepl_runtime_dir='/var/run/zrepl'
+
+start() {
+	mkdir -p "$zrepl_runtime_dir/stdinserver"
+	chmod -R 0700 "$zrepl_runtime_dir"
+	default_start
+}
+
+stop() {
+	rm -rf "$zrepl_runtime_dir"
+	default_stop
+}
+
+# vi: noet sw=8 sts=0

dist/systemd/zrepl.service

@@ -9,6 +9,9 @@ ExecStart=/usr/local/bin/zrepl --config /etc/zrepl/zrepl.yml daemon
 RuntimeDirectory=zrepl zrepl/stdinserver
 RuntimeDirectoryMode=0700
 
+# Make Go produce coredumps
+Environment=GOTRACEBACK='crash'
+
 ProtectSystem=strict
 #PrivateDevices=yes # TODO ZFS needs access to /dev/zfs, could we limit this?
 ProtectKernelTunables=yes
@@ -27,7 +30,8 @@ ProtectHome=read-only
 # SystemCallFilter
 #   ~@privileged doesn't work with Ubuntu 18.04 ssh
 SystemCallFilter=~ @mount @cpu-emulation @keyring @module @obsolete @raw-io @debug @clock @resources
-
+# Go1.19 added automatic RLIMIT_NOFILE changes, so, we need to allow that
+SystemCallFilter= setrlimit
 
 [Install]
 WantedBy=multi-user.target

docs/Makefile

@@ -10,11 +10,11 @@ BUILDDIR      = _build
 
 # Put it first so that "make" without argument is like "make help".
 help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" -c sphinxconf $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
 .PHONY: help Makefile
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" -c sphinxconf $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/_static/banner.css

@@ -0,0 +1,43 @@
+/* https://github.com/sphinx-contrib/sphinxcontrib-versioning/blob/0b56210959c6b21bbb730072e81876491b4e1371/sphinxcontrib/versioning/_static/banner.css */
+
+.scv-banner {
+    padding: 3px;
+    border-radius: 2px;
+    font-size: 80%;
+    text-align: center;
+    color: white;
+    background: #d40 linear-gradient(-45deg,
+        rgba(255, 255, 255, 0.2) 0%,
+        rgba(255, 255, 255, 0.2) 25%,
+        transparent 25%,
+        transparent 50%,
+        rgba(255, 255, 255, 0.2) 50%,
+        rgba(255, 255, 255, 0.2) 75%,
+        transparent 75%,
+        transparent
+    );
+    background-size: 28px 28px;
+}
+.scv-banner > a {
+    color: white;
+}
+
+
+.scv-sphinx_rtd_theme {
+    background-color: #2980B9;
+}
+
+
+.scv-bizstyle {
+    background-color: #336699;
+}
+
+
+.scv-classic {
+    text-align: center !important;
+}
+
+
+.scv-traditional {
+    text-align: center !important;
+}

docs/_templates/page.html

@@ -0,0 +1,17 @@
+{% extends "!page.html" %}
+{% block body %}
+{% if current_version and latest_version and current_version != latest_version %}
+<p class="scv-banner scv-sphinx_rtd_theme">
+  <strong>
+    {% if current_version.is_released %}
+    You're reading an old version of this documentation.
+    If you want up-to-date information, please have a look at <a href="{{ vpathto(latest_version.name) }}">{{latest_version.name}}</a>.
+    {% else %}
+    You're reading the documentation for a development version.
+    For the latest released version, please have a look at <a href="{{ vpathto(latest_version.name) }}">{{latest_version.name}}</a>.
+    {% endif %}
+  </strong>
+</p>
+{% endif %}
+{{ super() }}
+{% endblock %}%

docs/_templates/versions.html

@@ -0,0 +1,27 @@
+{%- if current_version %}
+<div class="rst-versions" data-toggle="rst-versions" role="note" aria-label="versions">
+  <span class="rst-current-version" data-toggle="rst-current-version">
+    <span class="fa fa-book"> Other Versions</span>
+    v: {{ current_version.name }}
+    <span class="fa fa-caret-down"></span>
+  </span>
+  <div class="rst-other-versions">
+    {%- if versions.tags %}
+    <dl>
+      <dt>Tags</dt>
+      {%- for item in versions.tags %}
+      <dd><a href="{{ item.url }}">{{ item.name }}</a></dd>
+      {%- endfor %}
+    </dl>
+    {%- endif %}
+    {%- if versions.branches %}
+    <dl>
+      <dt>Branches</dt>
+      {%- for item in versions.branches %}
+      <dd><a href="{{ item.url }}">{{ item.name }}</a></dd>
+      {%- endfor %}
+    </dl>
+    {%- endif %}
+  </div>
+</div>
+{%- endif %}

docs/changelog.rst

@@ -16,13 +16,63 @@ Changelog
 The changelog summarizes bugfixes that are deemed relevant for users and package maintainers.
 Developers should consult the git commit log or GitHub issue tracker.
 
-0.6 (Unreleased)
+Next Release
-----------------
+------------
 
-* `Feature Wishlist on GitHub <https://github.com/zrepl/zrepl/discussions/547>`_
+The plan for the next release is to revisit how zrepl does snapshot management.
+High-level goals:
+
+- Make it easy to decouple snapshot management (snapshotting, pruning) from replication.
+- Ability to include/exclude snapshots from replication.
+  This is useful for aforementioned decoupling, e.g., separate snapshot prefixes for local & remote replication.
+  Also, it makes explicit that by default, zrepl replicates all snapshots, and that
+  replication has no concept of "zrepl-created snapshots", which is a common misconception.
+- Use of ``zfs snapshot`` comma syntax or channel programs to take snapshots of multiple
+  datasets atomically.
+- Provide an alternative to the ``grid`` pruning policy.
+  Most likely something based on hourly/daily/weekly/monthly "trains" plus a count.
+- Ability to prune at the granularity of the **group** of snapshots created at a given
+  time, as opposed to the individual snapshots within a dataset.
+  Maybe this will be addressed by the alternative to the ``grid`` pruning policy,
+  as it will likely be more predictable.
+
+Those changes will likely come with some breakage in the config.
+However, I want to avoid breaking **use cases** that are satisfied by the current design.
+There will be beta/RC releases to give users a chance to evaluate.
+
+0.6.1
+-----
+
+* |feature| add metric to detect filesystems rules that don't match any local dataset (thanks, `@gmekicaxcient <https://github.com/gmekicaxcient>`_).
+* |bugfix| ``zrepl status``: hide progress bar once all filesystems reach terminal state (thanks, `@0x3333 <https://github.com/0x3333>`_).
+* |bugfix| handling of tenative cursor presence if protection strategy doesn't use it (:issue:`714`).
+* |docs| address setup with two or more external disks (thanks, `@se-jaeger <https://github.com/se-jaeger>`_).
+* |docs| document ``replication`` and ``conflict_resolution`` options (thanks, `@InsanePrawn <https://github.com/InsanePrawn>`_).
+* |docs| docs: talks: add note on keep_bookmarks option (thanks, `@skirmess <https://github.com/skirmess>`_).
+* |maint| dist: add openrc service file (thanks, `@gramosg <https://github.com/gramosg>`_).
+* |maint| grafana: update dashboard to Grafana 9.3.6.
+* |maint| run platform tests as part of CI.
+* |maint| build: upgrade to Go 1.21 and update golangci-lint; minimum Go version for builds is now 1.20
+
+.. NOTE::
+  |  zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
+  |  You can support maintenance and feature development through one of the following services:
+  |  |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
+  |  Note that PayPal processing fees are relatively high for small donations.
+  |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
+
+0.6
+---
 
 * |feature| :ref:`Schedule-based snapshotting<job-snapshotting--cron>` using ``cron`` syntax instead of an interval.
-* |feature| Add ``ZREPL_DESTROY_MAX_BATCH_SIZE`` env var (default 0=unlimited).
+* |feature| Configurable initial replication policy.
+  When a filesystem is first replicated to a receiver, this control whether just the newest
+  snapshot will be replicated vs. all existing snapshots. Learn more :ref:`in the docs <conflict_resolution-initial_replication>`.
+* |feature| Configurable timestamp format for snapshot names via :ref:`timestamp_format<job-snapshotting-timestamp_format>`
+  (Thanks, `@ydylla <https://github.com/ydylla>`_).
+* |feature| Add ``ZREPL_DESTROY_MAX_BATCH_SIZE`` env var (default 0=unlimited)
+  (Thanks, `@3nprob <https://github.com/3nprob>`_).
+* |feature| Add ``zrepl configcheck --skip-cert-check`` flag (Thanks, `@cole-h <https://github.com/cole-h>`_).
 * |bugfix| Fix resuming from interrupted replications that use ``send.raw`` on unencrypted datasets.
 
   * The send options introduced in zrepl 0.4 allow users to specify additional zfs send flags for zrepl to use.
@@ -34,7 +84,7 @@ Developers should consult the git commit log or GitHub issue tracker.
   * However, this means that the ``zrepl status`` UI no longer indicates whether a replication step uses encrypted sends or not.
     The setting is still effective though.
 
-* |break| |feature| convert Prometheus metric ``zrepl_version_daemon`` to ``zrepl_start_time`` metric
+* |break| convert Prometheus metric ``zrepl_version_daemon`` to ``zrepl_start_time`` metric
 
   * The metric still reports the zrepl version in a label.
     But the metric *value* is now the Unix timestamp at the time the daemon was started.
@@ -43,6 +93,13 @@ Developers should consult the git commit log or GitHub issue tracker.
 * |bugfix| transient zrepl status error: ``Post "http://unix/status": EOF``
 * |bugfix| don't treat receive-side bookmarks as a replication conflict.
   This facilitates chaining of replication jobs. See :issue:`490`.
+* |bugfix| workaround for Go/gRPC problem on Illumos where zrepl would
+  crash when using the ``local`` transport type (:issue:`598`).
+* |bugfix| fix active child tasks panic that cold occur during replication plannig (:issue:`193abbe`)
+* |bugfix| ``zrepl status`` off-by-one error in display of completed step count (:commit:`ce6701f`)
+* |bugfix| Allow using day & week units for ``snapshotting.interval`` (:commit:`ffb1d89`)
+* |docs| ``docs/overview`` improvements (Thanks, `@jtagcat <https://github.com/jtagcat>`_).
+* |maint| Update to Go 1.19.
 
 0.5
 ---
@@ -75,14 +132,6 @@ Note to all users: please read up on the following OpenZFS bugs, as you might be
 
 Finally, I'd like to point you to the `GitHub discussion <https://github.com/zrepl/zrepl/discussions/547>`_ about which bugfixes and features should be prioritized in zrepl 0.6 and beyond!
 
-.. NOTE::
-  |  zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
-  |  You can support maintenance and feature development through one of the following services:
-  |  |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
-  |  Note that PayPal processing fees are relatively high for small donations.
-  |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
-
-
 0.4.0
 -----
 

docs/conf.py

@@ -1 +0,0 @@
-sphinxconf/conf.py

docs/conf.py

@@ -0,0 +1,187 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# zrepl documentation build configuration file, created by
+# sphinx-quickstart on Wed Nov  8 22:28:10 2017.
+#
+# This file is execfile()d with the current directory set to its
+# containing dir.
+#
+# Note that not all possible configuration values are present in this
+# autogenerated file.
+#
+# All configuration values have a default; values that are commented out
+# serve to show the default.
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- General configuration ------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = ['sphinx.ext.todo',
+    'sphinx.ext.githubpages',
+    'sphinx.ext.extlinks',
+    "sphinx_multiversion",
+]
+
+# suppress_warnings = ['image.nonlocal_uri']
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['./_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# General information about the project.
+project = 'zrepl'
+copyright = '2017-2023, Christian Schwarz'
+author = 'Christian Schwarz'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+#
+# The short X.Y version.
+#version = set by sphinxcontrib-versioning
+# The full version, including alpha/beta/rc tags.
+#release = version
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = 'en'
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This patterns also effect to html_static_path and html_extra_path
+exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+# If true, `todo` and `todoList` produce output, else they produce nothing.
+todo_include_todos = True
+
+
+# -- Options for HTML output ----------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'sphinx_rtd_theme'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+html_css_files = [
+    'banner.css',
+]
+
+html_logo = '_static/zrepl.svg'
+
+html_context = {
+  # https://github.com/rtfd/sphinx_rtd_theme/issues/205
+  # Add 'Edit on Github' link instead of 'View page source'
+  "display_github": True,
+  "github_user": "zrepl",
+  "github_repo": "zrepl",
+  "github_version": "master",
+  "conf_py_path": "/docs/",
+  "source_suffix": source_suffix,
+}
+
+# -- Options for HTMLHelp output ------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'zrepldoc'
+
+
+# -- Options for LaTeX output ---------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'zrepl.tex', 'zrepl Documentation',
+     'Christian Schwarz', 'manual'),
+]
+
+
+# -- Options for manual page output ---------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'zrepl', 'zrepl Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output -------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'zrepl', 'zrepl Documentation',
+     author, 'zrepl', 'One line description of project.',
+     'Miscellaneous'),
+]
+
+
+# -- Options for the extlinks extension -----------------------------------
+# http://www.sphinx-doc.org/en/stable/ext/extlinks.html
+extlinks = {
+    'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #%s'),
+    'repomasterlink':('https://github.com/zrepl/zrepl/blob/master/%s', '%s'),
+    'sampleconf':('https://github.com/zrepl/zrepl/blob/master/config/samples%s', 'config/samples%s'),
+    'commit':('https://github.com/zrepl/zrepl/commit/%s', 'commit %s'),
+        }
+

docs/configuration/conflict_resolution.rst

@@ -1,5 +1,6 @@
 .. include:: ../global.rst.inc
 
+.. _conflict_resolution-options:
 
 Conflict Resolution Options
 ===========================
@@ -15,7 +16,7 @@ Conflict Resolution Options
 
      ...
 
-.. _conflict_resolution-initial_replication-option-send_all_snapshots:
+.. _conflict_resolution-initial_replication:
 
 
 ``initial_replication`` option

docs/configuration/jobs.rst

@@ -30,6 +30,10 @@ Job Type ``push``
       - |snapshotting-spec|
     * - ``pruning``
       - |pruning-spec|
+    * - ``replication``
+      - |replication-options|
+    * - ``conflict_resolution``
+      - |conflict-resolution-options|
 
 Example config: :sampleconf:`/push.yml`
 
@@ -81,6 +85,10 @@ Job Type ``pull``
         | ``manual`` disables periodic pulling, replication then only happens on :ref:`wakeup <cli-signal-wakeup>`.
     * - ``pruning``
       - |pruning-spec|
+    * - ``replication``
+      - |replication-options|
+    * - ``conflict_resolution``
+      - |conflict-resolution-options|
 
 Example config: :sampleconf:`/pull.yml`
 

docs/configuration/misc.rst

@@ -44,27 +44,3 @@ Interval & duration fields in job definitions, pruning configurations, etc. must
 
     var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
     // s = second, m = minute, h = hour, d = day, w = week (7 days)
-
-Super-Verbose Job Debugging
----------------------------
-
-You have probably landed here because you opened an issue on GitHub and some developer told you to do this...
-So just read the annotated comments ;)
-
-::
-
-    job:
-    - name: ...
-      ...
-     # JOB DEBUGGING OPTIONS
-      # should be equal for all job types, but each job implements the debugging itself
-      debug:
-        conn: # debug the io.ReadWriteCloser connection
-          read_dump: /tmp/connlog_read   # dump results of Read() invocations to this file
-          write_dump: /tmp/connlog_write # dump results of Write() invocations to this file
-        rpc: # debug the RPC protocol implementation
-          log: true # log output from rpc layer to the job log
-
-.. ATTENTION::
-
-    Connection dumps will almost certainly contain your or other's private data. Do not share it in a bug report.

docs/configuration/replication.rst

@@ -1,5 +1,6 @@
 .. include:: ../global.rst.inc
 
+.. _replication-options:
 
 Replication Options
 ===================

docs/configuration/sendrecvoptions.rst

@@ -197,11 +197,8 @@ Mount behaviour
 * ``canmount``
 * ``overlay``
 
-Note: inheriting or overriding the ``mountpoint`` property on ZVOLs fails in ``zfs recv``.
+Note: Before `OpenZFS 2.0.5 <https://github.com/openzfs/zfs/issues/11416>`_, inheriting or overriding the ``mountpoint`` property on ZVOLs fails in ``zfs recv``.
-This is an `issue in OpenZFS <https://github.com/openzfs/zfs/issues/11416>`_ .
+If you are on such an older version, consider creating separate zrepl jobs for your ZVOL and filesystem datasets.
-As a workaround, consider creating separate zrepl jobs for your ZVOL and filesystem datasets.
-Please comment at zrepl :issue:`430` if you encounter this issue and/or would like zrepl to automatically work around it.
-
 
 Systemd
 -------

docs/configuration/snapshotting.rst

@@ -62,6 +62,9 @@ The ``periodic`` and ``cron`` snapshotting types share some common options and b
        type: periodic
        prefix: zrepl_
        interval: 10m
+       # Timestamp format that is used as snapshot suffix.
+       # Can be any of "dense" (default), "human", "iso-8601", "unix-seconds" or a custom Go time format (see https://go.dev/src/time/format.go)
+       timestamp_format: dense
        hooks: ...
     pruning: ...
 
@@ -91,6 +94,9 @@ The snapshotter uses the ``prefix`` to identify which snapshots it created.
        # (second, optional) minute hour day-of-month month day-of-week
        # This example takes snapshots daily at 3:00.
        cron: "0 3 * * *"
+       # Timestamp format that is used as snapshot suffix.
+       # Can be any of "dense" (default), "human", "iso-8601", "unix-seconds" or a custom Go time format (see https://go.dev/src/time/format.go)
+       timestamp_format: dense
      pruning: ...
 
 In ``cron`` mode, the snapshotter takes snaphots at fixed points in time.
@@ -98,6 +104,21 @@ See https://en.wikipedia.org/wiki/Cron for details on the syntax.
 zrepl uses the ``the github.com/robfig/cron/v3`` Go package for parsing.
 An optional field for "seconds" is supported to take snapshots at sub-minute frequencies.
 
+.. _job-snapshotting-timestamp_format:
+
+Timestamp Format
+~~~~~~~~~~~~~~~~
+
+The ``cron`` and ``periodic`` snapshotter support configuring a custom timestamp format that is used as suffix for the snapshot name.
+It can be used by setting ``timestamp_format`` to any of the following values:
+
+* ``dense`` (default) looks like ``20060102_150405_000``
+* ``human`` looks like ``2006-01-02_15:04:05``
+* ``iso-8601`` looks like ``2006-01-02T15:04:05.000Z``
+* ``unix-seconds`` looks like ``1136214245``
+* Any custom Go time format accepted by `time.Time#Format <https://go.dev/src/time/format.go>`_.
+
+
 ``manual`` Snapshotting
 -----------------------
 

docs/global.rst.inc

@@ -26,6 +26,8 @@
 .. |connect-transport| replace:: :ref:`connect specification<transport>`
 .. |send-options| replace:: :ref:`send options<job-send-options>`, e.g. for encrypted sends
 .. |recv-options| replace:: :ref:`recv options<job-recv-options>`
+.. |replication-options| replace:: :ref:`replication options<replication-options>`
+.. |conflict-resolution-options| replace:: :ref:`conflict resolution options<conflict_resolution-options>`
 .. |snapshotting-spec| replace:: :ref:`snapshotting specification <job-snapshotting-spec>`
 .. |pruning-spec| replace:: :ref:`pruning specification <prune>`
 .. |filter-spec| replace:: :ref:`filter specification<pattern-filter>`

docs/pr.rst

@@ -15,3 +15,5 @@ Talks & Presentations
   `Event <https://wiki.freebsd.org/DevSummit/201709>`__
   )
 
+  * Note: The remarks on ``keep_bookmarks`` are irrelevant as of zrepl 0.1 which introduced the zrepl-managed replication cursor bookmark.
+    Read the `Overview <overview-how-replication-works>`_ section to learn more.

docs/publish.sh

@@ -3,7 +3,8 @@ set -euo pipefail
 
 NON_INTERACTIVE=false
 DO_CLONE=false
-while getopts "ca" arg; do
+PUSH=false
+while getopts "caP" arg; do
     case "$arg" in
         "a")
             NON_INTERACTIVE=true
@@ -11,6 +12,9 @@ while getopts "ca" arg; do
         "c")
             DO_CLONE=true
             ;;
+        "P")
+            PUSH=true
+            ;;
         *)
             echo "invalid option '-$arg'"
             exit 1
@@ -26,8 +30,8 @@ checkout_repo_msg() {
     echo "clone ${GHPAGESREPO} to ${PUBLICDIR}:"
 }
 
-if ! type sphinx-versioning >/dev/null; then
+if ! type sphinx-multiversion >/dev/null; then
-    echo "install sphinx-versioning and come back"
+    echo "install sphinx-multiversion and come back"
     exit 1
 fi
 
@@ -49,11 +53,11 @@ else
     read -r
 fi
 
-pushd "$PUBLICDIR" 
+pushd "$PUBLICDIR"
 
 echo "verify we're in the GitHub pages repo..."
 git remote get-url origin | grep -E "^${GHPAGESREPO}\$"
-if [ "$?" -ne "0" ] ;then 
+if [ "$?" -ne "0" ] ;then
     checkout_repo_msg
     echo "finished checkout, please run again"
     exit 1
@@ -73,22 +77,15 @@ popd
 
 echo "building site"
 
-flags="$(python3 gen-sphinx-versioning-flags.py)"
+python3 run-sphinx-multiversion.py . ./public_git
-set -e
+
-sphinx-versioning build \
-   $flags \
-   docs ./public_git \
-   -- -c sphinxconf # older conf.py throw errors because they used
-                    # version = subprocess.show_output(["git", "describe"])
-                    # which fails when building with sphinxcontrib-versioning
-set +e
 
 CURRENT_COMMIT=$(git rev-parse HEAD)
 git status --porcelain
 if [[ "$(git status --porcelain)" != "" ]]; then
-    CURRENT_COMMIT="${CURRENT_COMMIT}(dirty)" 
+    CURRENT_COMMIT="${CURRENT_COMMIT}(dirty)"
 fi
-COMMIT_MSG="sphinx-versioning render from publish.sh - $(date -u) - ${CURRENT_COMMIT}"
+COMMIT_MSG="render from publish.sh - $(date -u) - ${CURRENT_COMMIT}"
 
 pushd "$PUBLICDIR"
 
@@ -100,6 +97,11 @@ if [ "$(git status --porcelain)" != "" ]; then
 else
     echo "nothing to commit"
 fi
-echo "pushing to GitHub pages repo"
+
-git push origin master
+if $PUSH; then
+    echo "pushing to GitHub pages repo"
+    git push origin master
+else
+    echo "not pushing to GitHub pages repo, set -P flag to push"
+fi
 

docs/quickstart/backup_to_external_disk.rst

@@ -33,4 +33,17 @@ You will likely want to customize some aspects mentioned in the top comment in t
 
 .. literalinclude:: ../../config/samples/quickstart_backup_to_external_disk.yml
 
+
+Offline Backups with two (or more) External Disks
+-------------------------------------------------
+
+It can be desirable to have multiple disk-based backups of the same machine.
+To accomplish this,
+
+* create one zpool per external HDD, each with a unique name, and
+* define a pair of ``push`` and ``sink`` job **for each** of these zpools, each with a unique ``name``, ``listener_name``, and ``root_fs``.
+
+The unique names ensure that the jobs don't step on each others' toes when managing :ref:`zrepl's ZFS abstractions <zrepl-zfs-abstractions>` . 
+
+
 :ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.

docs/requirements.txt

@@ -1,30 +1,24 @@
-alabaster==0.7.12
+alabaster==0.7.13
-attrs==19.1.0
+Babel==2.12.1
-Babel==2.7.0
+certifi==2023.7.22
-certifi==2019.6.16
+charset-normalizer==3.2.0
-chardet==3.0.4
+docutils==0.18.1
-Click==7.0
+idna==3.4
-colorclass==2.2.0
+imagesize==1.4.1
-docutils==0.15.2
+Jinja2==3.1.2
-idna==2.8
+MarkupSafe==2.1.3
-imagesize==1.1.0
+packaging==23.1
-Jinja2==2.10.1
+Pygments==2.16.1
-MarkupSafe==1.1.1
+requests==2.31.0
-packaging==19.1
+snowballstemmer==2.2.0
-Pygments==2.4.2
+Sphinx==7.2.5
-pyparsing==2.4.2
+sphinx-multiversion @ git+https://github.com/zrepl/sphinx-multiversion/@52c915d7ad898d9641ec48c8bbccb7d4f079db93
-pytz==2019.2
+sphinx-rtd-theme==1.3.0
-requests==2.22.0
+sphinxcontrib-applehelp==1.0.7
-six==1.12.0
+sphinxcontrib-devhelp==1.0.5
-snowballstemmer==1.9.1
+sphinxcontrib-htmlhelp==2.0.4
-Sphinx==1.8.5
+sphinxcontrib-jquery==4.1
-sphinx-rtd-theme==0.4.3
-sphinxcontrib-applehelp==1.0.1
-sphinxcontrib-devhelp==1.0.1
-sphinxcontrib-htmlhelp==1.0.2
 sphinxcontrib-jsmath==1.0.1
-sphinxcontrib-qthelp==1.0.2
+sphinxcontrib-qthelp==1.0.6
-sphinxcontrib-serializinghtml==1.1.3
+sphinxcontrib-serializinghtml==1.1.9
-git+https://github.com/rwblair/sphinxcontrib-versioning.git@7e3885a389a809e17ea55261316b7b0e98dbf98f#egg=sphinxcontrib-versioning
+urllib3==2.0.4
-sphinxcontrib-websupport==1.1.2
-urllib3==1.25.3

docs/run-sphinx-multiversion.py

@@ -1,7 +1,15 @@
 #!/usr/bin/env python3
 
+from pathlib import Path
 import subprocess
 import re
+import argparse
+import distutils
+
+argparser = argparse.ArgumentParser()
+argparser.add_argument("docsroot")
+argparser.add_argument("outdir")
+args = argparser.parse_args()
 
 output = subprocess.run(["git", "tag", "-l"], capture_output=True, check=True, text=True)
 tagRE = re.compile(r"^v(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(-rc(?P<rc>\d+))?$")
@@ -27,8 +35,8 @@ for line in output.stdout.split("\n"):
 
     t = Tag()
     t.orig = line
-    t.major = int(m["major"])   
+    t.major = int(m["major"])
-    t.minor = int(m["minor"])   
+    t.minor = int(m["minor"])
     t.patch = int(m["patch"])
     t.rc = int(m["rc"] if m["rc"] is not None else 0)
 
@@ -49,26 +57,16 @@ for (mm, l) in by_major_minor.items():
     latest_by_major_minor.append(l[-1])
 latest_by_major_minor.sort(key=lambda tag: (tag.major, tag.minor))
 
-# print(by_major_minor)
+cmdline = [
-# print(latest_by_major_minor)
+    "sphinx-multiversion",
-
+    "-D", "smv_tag_whitelist=^({})$".format("|".join([re.escape(tag.orig) for tag in latest_by_major_minor])),
-cmdline = []
+    "-D", "smv_branch_whitelist=^(master|stable)$",
-
+    "-D", "smv_remote_whitelist=^.*$",
-for latest_patch in latest_by_major_minor:
+    "-D", "smv_latest_version=stable",
-    cmdline.append("--whitelist-tags")
+    "-D", r"smv_released_pattern=^refs/(tags|heads|remotes/[^/]+)/(?!master).*$", # treat everything except master as released, that way, the banner message makes sense
-    cmdline.append(f"^{re.escape(latest_patch.orig)}$")
+    # "--dump-metadata", # for debugging
-
+    args.docsroot,
-# we want flexibility to update docs for the latest stable release
+    args.outdir,
-# => we have a branch for that, called `stable` which we move manually
+]
-# TODO: in the future, have f"stable-{latest_by_major_minor[-1]}"
+print(cmdline)
-default_version = "stable"
+subprocess.run(cmdline, check=True)
-cmdline.extend(["--whitelist-branches", default_version])
-
-cmdline.extend(["--root-ref", f"{default_version}"])
-cmdline.extend(["--banner-main-ref", f"{default_version}"])
-cmdline.extend(["--show-banner"])
-cmdline.extend(["--sort", "semver"])
-
-cmdline.extend(["--whitelist-branches", "master"])
-
-print(" ".join(cmdline))

docs/sphinxconf/conf.py

@@ -1,182 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# zrepl documentation build configuration file, created by
-# sphinx-quickstart on Wed Nov  8 22:28:10 2017.
-#
-# This file is execfile()d with the current directory set to its
-# containing dir.
-#
-# Note that not all possible configuration values are present in this
-# autogenerated file.
-#
-# All configuration values have a default; values that are commented out
-# serve to show the default.
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-# import os
-# import sys
-# sys.path.insert(0, os.path.abspath('.'))
-
-
-# -- General configuration ------------------------------------------------
-
-# If your documentation needs a minimal Sphinx version, state it here.
-#
-# needs_sphinx = '1.0'
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = ['sphinx.ext.todo',
-    'sphinx.ext.githubpages',
-    'sphinx.ext.extlinks']
-
-suppress_warnings = ['image.nonlocal_uri']
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['../_templates']
-
-# The suffix(es) of source filenames.
-# You can specify multiple suffix as a list of string:
-#
-# source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
-
-# The master toctree document.
-master_doc = 'index'
-
-# General information about the project.
-project = 'zrepl'
-copyright = '2017-2019, Christian Schwarz'
-author = 'Christian Schwarz'
-
-# The version info for the project you're documenting, acts as replacement for
-# |version| and |release|, also used in various other places throughout the
-# built documents.
-#
-# The short X.Y version.
-#version = set by sphinxcontrib-versioning
-# The full version, including alpha/beta/rc tags.
-#release = version
-
-# The language for content autogenerated by Sphinx. Refer to documentation
-# for a list of supported languages.
-#
-# This is also used if you do content translation via gettext catalogs.
-# Usually you set "language" from the command line for these cases.
-language = None
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This patterns also effect to html_static_path and html_extra_path
-exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
-
-# The name of the Pygments (syntax highlighting) style to use.
-pygments_style = 'sphinx'
-
-# If true, `todo` and `todoList` produce output, else they produce nothing.
-todo_include_todos = True
-
-
-# -- Options for HTML output ----------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = 'sphinx_rtd_theme'
-
-# Theme options are theme-specific and customize the look and feel of a theme
-# further.  For a list of options available for each theme, see the
-# documentation.
-#
-# html_theme_options = {}
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['../_static']
-
-
-html_logo = '../_static/zrepl.svg'
-
-html_context = {
-  # https://github.com/rtfd/sphinx_rtd_theme/issues/205
-  # Add 'Edit on Github' link instead of 'View page source'
-  "display_github": True, 
-  "github_user": "zrepl",
-  "github_repo": "zrepl",
-  "github_version": "master",
-  "conf_py_path": "/docs/",
-  "source_suffix": source_suffix,
-}
-
-# -- Options for HTMLHelp output ------------------------------------------
-
-# Output file base name for HTML help builder.
-htmlhelp_basename = 'zrepldoc'
-
-
-# -- Options for LaTeX output ---------------------------------------------
-
-latex_elements = {
-    # The paper size ('letterpaper' or 'a4paper').
-    #
-    # 'papersize': 'letterpaper',
-
-    # The font size ('10pt', '11pt' or '12pt').
-    #
-    # 'pointsize': '10pt',
-
-    # Additional stuff for the LaTeX preamble.
-    #
-    # 'preamble': '',
-
-    # Latex figure (float) alignment
-    #
-    # 'figure_align': 'htbp',
-}
-
-# Grouping the document tree into LaTeX files. List of tuples
-# (source start file, target name, title,
-#  author, documentclass [howto, manual, or own class]).
-latex_documents = [
-    (master_doc, 'zrepl.tex', 'zrepl Documentation',
-     'Christian Schwarz', 'manual'),
-]
-
-
-# -- Options for manual page output ---------------------------------------
-
-# One entry per manual page. List of tuples
-# (source start file, name, description, authors, manual section).
-man_pages = [
-    (master_doc, 'zrepl', 'zrepl Documentation',
-     [author], 1)
-]
-
-
-# -- Options for Texinfo output -------------------------------------------
-
-# Grouping the document tree into Texinfo files. List of tuples
-# (source start file, target name, title, author,
-#  dir menu entry, description, category)
-texinfo_documents = [
-    (master_doc, 'zrepl', 'zrepl Documentation',
-     author, 'zrepl', 'One line description of project.',
-     'Miscellaneous'),
-]
-
-
-# -- Options for the extlinks extension -----------------------------------
-# http://www.sphinx-doc.org/en/stable/ext/extlinks.html
-extlinks = {
-    'issue':('https://github.com/zrepl/zrepl/issues/%s', 'issue #'),
-    'repomasterlink':('https://github.com/zrepl/zrepl/blob/master/%s', ''),
-    'sampleconf':('https://github.com/zrepl/zrepl/blob/master/config/samples%s', 'config/samples'),
-    'commit':('https://github.com/zrepl/zrepl/commit/%s', 'commit '),
-        }
-

docs/supporters.rst

@@ -32,6 +32,7 @@ We would like to thank the following people and organizations for supporting zre
 
    <div class="fa fa-code" style="width: 1em;"></div>
 
+* |supporter-std| `Max Christian Pohle <https://coderonline.de>`_
 * |supporter-gold| Prominic.NET, Inc.
 * |supporter-std| Torsten Blum
 * |supporter-gold| Cyberiada GmbH

endpoint/endpoint.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"path"
+	"strings"
 
 	"github.com/kr/pretty"
 	"github.com/pkg/errors"
@@ -233,6 +234,21 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
 	//
 	// Note further that a resuming send, due to the idempotent nature of func CreateReplicationCursor and HoldStep,
 	// will never lose its step holds because we just (idempotently re-)created them above, before attempting the cleanup.
+	destroyTypes := AbstractionTypeSet{
+		AbstractionStepHold:                           true,
+		AbstractionTentativeReplicationCursorBookmark: true,
+	}
+	// The replication planner can also pick an endpoint zfs abstraction as FromVersion.
+	// Keep it, so that the replication will succeed.
+	//
+	// NB: there is no abstraction for snapshots, so, we only need to check bookmarks.
+	if sendArgs.FromVersion != nil && sendArgs.FromVersion.IsBookmark() {
+		dp, err := zfs.NewDatasetPath(sendArgs.FS)
+		if err != nil {
+			panic(err) // sendArgs is validated, this shouldn't happen
+		}
+		liveAbs = append(liveAbs, destroyTypes.ExtractBookmark(dp, sendArgs.FromVersion))
+	}
 	func() {
 		ctx, endSpan := trace.WithSpan(ctx, "cleanup-stale-abstractions")
 		defer endSpan()
@@ -245,35 +261,45 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, io.Rea
 			return keep
 		}
 		check := func(obsoleteAbs []Abstraction) {
-			// last line of defense: check that we don't destroy the incremental `from` and `to`
+			// Ensure that we don't delete `From` or `To`.
-			// if we did that, we might be about to blow away the last common filesystem version between sender and receiver
+			// Regardless of whether they are in AbstractionTypeSet or not.
-			mustLiveVersions := []zfs.FilesystemVersion{sendArgs.ToVersion}
+			// And produce a nice error message in case we do, to aid debugging the resulting panic.
-			if sendArgs.FromVersion != nil {
+			//
-				mustLiveVersions = append(mustLiveVersions, *sendArgs.FromVersion)
+			// This is especially important for `From`. We could break incremental replication
+			// if we deleted the last common filesystem version between sender and receiver.
+			type Problem struct {
+				sendArgsWhat string
+				fullpath     string
+				obsoleteAbs  Abstraction
 			}
-			for _, staleVersion := range obsoleteAbs {
+			problems := make([]Problem, 0)
-				for _, mustLiveVersion := range mustLiveVersions {
+			checkFullpaths := make(map[string]string, 2)
-					isSendArg := zfs.FilesystemVersionEqualIdentity(mustLiveVersion, staleVersion.GetFilesystemVersion())
+			checkFullpaths["ToVersion"] = sendArgs.ToVersion.FullPath(sendArgs.FS)
-					stepHoldBasedGuaranteeStrategy := false
+			if sendArgs.FromVersion != nil {
-					k := replicationGuaranteeStrategy.Kind()
+				checkFullpaths["FromVersion"] = sendArgs.FromVersion.FullPath(sendArgs.FS)
-					switch k {
+			}
-					case ReplicationGuaranteeKindResumability:
+			for _, a := range obsoleteAbs {
-						stepHoldBasedGuaranteeStrategy = true
+				for what, fullpath := range checkFullpaths {
-					case ReplicationGuaranteeKindIncremental:
+					if a.GetFullPath() == fullpath && a.GetType().IsSnapshotOrBookmark() {
-					case ReplicationGuaranteeKindNone:
+						problems = append(problems, Problem{
-					default:
+							sendArgsWhat: what,
-						panic(fmt.Sprintf("this is supposed to be an exhaustive match, got %v", k))
+							fullpath:     fullpath,
-					}
+							obsoleteAbs:  a,
-					isSnapshot := mustLiveVersion.IsSnapshot()
+						})
-					if isSendArg && (!isSnapshot || stepHoldBasedGuaranteeStrategy) {
-						panic(fmt.Sprintf("impl error: %q would be destroyed because it is considered stale but it is part of of sendArgs=%s", mustLiveVersion.String(), pretty.Sprint(sendArgs)))
 					}
 				}
 			}
-		}
+			if len(problems) == 0 {
-		destroyTypes := AbstractionTypeSet{
+				return
-			AbstractionStepHold:                           true,
+			}
-			AbstractionTentativeReplicationCursorBookmark: true,
+			var msg strings.Builder
+			fmt.Fprintf(&msg, "cleaning up send stale would destroy send args:\n")
+			fmt.Fprintf(&msg, "  SendArgs: %s\n", pretty.Sprint(sendArgs))
+			for _, check := range problems {
+				fmt.Fprintf(&msg, "would delete %s %s because it was deemed an obsolete abstraction: %s\n",
+					check.sendArgsWhat, check.fullpath, check.obsoleteAbs)
+			}
+			panic(msg.String())
 		}
 		abstractionsCacheSingleton.TryBatchDestroy(ctx, s.jobId, sendArgs.FS, destroyTypes, keep, check)
 	}()
@@ -428,6 +454,7 @@ func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, _ io.ReadCloser
 
 type FSFilter interface { // FIXME unused
 	Filter(path *zfs.DatasetPath) (pass bool, err error)
+	UserSpecifiedDatasets() zfs.UserSpecifiedDatasetsSet
 }
 
 // FIXME: can we get away without error types here?
@@ -587,6 +614,12 @@ func (f subroot) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 	return p.HasPrefix(f.localRoot) && !p.Equal(f.localRoot), nil
 }
 
+func (f subroot) UserSpecifiedDatasets() zfs.UserSpecifiedDatasetsSet {
+	return zfs.UserSpecifiedDatasetsSet{
+		f.localRoot.ToString(): true,
+	}
+}
+
 func (f subroot) MapToLocal(fs string) (*zfs.DatasetPath, error) {
 	p, err := zfs.NewDatasetPath(fs)
 	if err != nil {

endpoint/endpoint_guarantees.go

@@ -89,6 +89,8 @@ func ReplicationGuaranteeFromKind(k ReplicationGuaranteeKind) ReplicationGuarant
 
 type ReplicationGuaranteeNone struct{}
 
+func (g ReplicationGuaranteeNone) String() string { return "none" }
+
 func (g ReplicationGuaranteeNone) Kind() ReplicationGuaranteeKind {
 	return ReplicationGuaranteeKindNone
 }
@@ -107,6 +109,8 @@ func (g ReplicationGuaranteeNone) SenderPostRecvConfirmed(ctx context.Context, j
 
 type ReplicationGuaranteeIncremental struct{}
 
+func (g ReplicationGuaranteeIncremental) String() string { return "incremental" }
+
 func (g ReplicationGuaranteeIncremental) Kind() ReplicationGuaranteeKind {
 	return ReplicationGuaranteeKindIncremental
 }
@@ -144,6 +148,8 @@ func (g ReplicationGuaranteeIncremental) SenderPostRecvConfirmed(ctx context.Con
 
 type ReplicationGuaranteeResumability struct{}
 
+func (g ReplicationGuaranteeResumability) String() string { return "resumability" }
+
 func (g ReplicationGuaranteeResumability) Kind() ReplicationGuaranteeKind {
 	return ReplicationGuaranteeKindResumability
 }

endpoint/endpoint_zfs_abstraction.go

@@ -31,7 +31,7 @@ const (
 	AbstractionReplicationCursorBookmarkV2        AbstractionType = "replication-cursor-bookmark-v2"
 )
 
-var AbstractionTypesAll = map[AbstractionType]bool{
+var AbstractionTypesAll = AbstractionTypeSet{
 	AbstractionStepHold:                           true,
 	AbstractionLastReceivedHold:                   true,
 	AbstractionTentativeReplicationCursorBookmark: true,
@@ -181,6 +181,38 @@ func (s AbstractionTypeSet) Validate() error {
 	return nil
 }
 
+// Use the `BookmarkExtractor()` method of each abstraction type in this set
+// to try extract an abstraction from the given FilesystemVersion.
+//
+// Abstraction types in this set that don't have a bookmark extractor are skipped.
+//
+// Panics if more than one abstraction type matches.
+func (s AbstractionTypeSet) ExtractBookmark(dp *zfs.DatasetPath, v *zfs.FilesystemVersion) Abstraction {
+	matched := make(AbstractionTypeSet, 1)
+	var matchedAbs Abstraction
+	for absType := range s {
+		extractor := absType.BookmarkExtractor()
+		if extractor == nil {
+			continue
+		}
+		abstraction := extractor(dp, *v)
+		if abstraction != nil {
+			matched[absType] = true
+			matchedAbs = abstraction
+		}
+	}
+	if len(matched) == 0 {
+		return nil
+	}
+	if len(matched) == 1 {
+		if matchedAbs == nil {
+			panic("loop above should always set matchedAbs if there is a match")
+		}
+		return matchedAbs
+	}
+	panic(fmt.Sprintf("abstraction types extractors should not overlap: %s", matched))
+}
+
 type BookmarkExtractor func(fs *zfs.DatasetPath, v zfs.FilesystemVersion) Abstraction
 
 // returns nil if the abstraction type is not bookmark-based
@@ -238,6 +270,23 @@ func (t AbstractionType) BookmarkNamer() func(fs string, guid uint64, jobId JobI
 	}
 }
 
+func (t AbstractionType) IsSnapshotOrBookmark() bool {
+	switch t {
+	case AbstractionTentativeReplicationCursorBookmark:
+		return true
+	case AbstractionReplicationCursorBookmarkV1:
+		return true
+	case AbstractionReplicationCursorBookmarkV2:
+		return true
+	case AbstractionStepHold:
+		return false
+	case AbstractionLastReceivedHold:
+		return false
+	default:
+		panic(fmt.Sprintf("unimpl: %q", t))
+	}
+}
+
 type ListZFSHoldsAndBookmarksQuery struct {
 	FS ListZFSHoldsAndBookmarksQueryFilesystemFilter
 	// What abstraction types should match (any contained in the set)

logger/datastructures.go

@@ -150,9 +150,7 @@ func NewOutlets() *Outlets {
 func (os *Outlets) DeepCopy() (copy *Outlets) {
 	copy = NewOutlets()
 	for level := range os.outs {
-		for i := range os.outs[level] {
+		copy.outs[level] = append(copy.outs[level], os.outs[level]...)
-			copy.outs[level] = append(copy.outs[level], os.outs[level][i])
-		}
 	}
 	return copy
 }

packaging/deb/Dockerfile

@@ -7,7 +7,8 @@ RUN apt-get update && apt-get install -y \
     dh-exec \
 	binutils-aarch64-linux-gnu \
 	binutils-arm-linux-gnueabihf  \
-	binutils-i686-linux-gnu
+	binutils-i686-linux-gnu \
+	binutils-x86-64-linux-gnu
 
 RUN mkdir -p /build/src && chmod -R 0777 /build
 

packaging/rpm/zrepl.spec

@@ -16,7 +16,7 @@ zrepl is a one-stop, integrated solution for ZFS replication.}
 %define __strip /usr/bin/true
 
 Name:             zrepl
-Release:          1
+Release:          SUBSTITUTED_BY_MAKEFILE
 Summary:          One-stop, integrated solution for ZFS replication
 License:          MIT
 URL:              https://zrepl.github.io/

platformtest/logmockzfs/logzfsenv

@@ -4,6 +4,21 @@ export ZREPL_MOCK_ZFS_COMMAND_LOG="$1"
 shift
 export ZREPL_MOCK_ZFS_PATH="$1"
 shift
-export PATH="$(dirname "${BASH_SOURCE[0]}" )":"$PATH"
+dirname="$(dirname "${BASH_SOURCE[0]}")"
+# If we invoke this script from the top zrepl source tree, like so:
+#    ./platformtest/logmockzfs/logzfsenv ...
+# then dirname is relative, i.e., ./platformtest/logmockzfs.
+# If we put that relative dir in PATH, then Go >= 1.19 will refuse to
+# exec the `./platformtest/logmockzfs/zfs` wrapper script if it finds
+# it via PATH lookup. For Example:
+#   cmd := exec.Command("zfs")
+#   err := cmd.Run()
+# will fail with an error that errors.Is(err, exec.ErrDot), message:
+#   cannot run executable found relative to current directory
+# The solution is to use an abspath.
+# Learn more at https://pkg.go.dev/os/exec#hdr-Executables_in_the_current_directory
+# and https://go-review.googlesource.com/c/go/+/381374
+absdirname="$(readlink -e "$dirname")"
+export PATH="$absdirname":"$PATH"
 args=("$@")
 exec "${args[@]}"

platformtest/tests/generated_cases.go

@@ -20,6 +20,7 @@ var Cases = []Case{BatchDestroy,
 	ReplicationIncrementalCleansUpStaleAbstractionsWithCacheOnSecondReplication,
 	ReplicationIncrementalCleansUpStaleAbstractionsWithoutCacheOnSecondReplication,
 	ReplicationIncrementalDestroysStepHoldsIffIncrementalStepHoldsAreDisabledButStepHoldsExist,
+	ReplicationIncrementalHandlesFromVersionEqTentativeCursorCorrectly,
 	ReplicationIncrementalIsPossibleIfCommonSnapshotIsDestroyed,
 	ReplicationInitialAll,
 	ReplicationInitialFail,

platformtest/tests/replication.go

@@ -248,7 +248,10 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
 		require.NoError(ctx, err)
 		snap2Hold, err := endpoint.HoldStep(ctx, sfs, snap2, jobId) // no shadow
 		require.NoError(ctx, err)
-		return []endpoint.Abstraction{snap2Cursor, snap1Hold, snap2Hold}
+		// create artificial tentative cursor
+		snap3TentativeCursor, err := endpoint.CreateTentativeReplicationCursor(ctx, sfs, snap3, jobId)
+		require.NoError(ctx, err)
+		return []endpoint.Abstraction{snap2Cursor, snap1Hold, snap2Hold, snap3TentativeCursor}
 	}
 	createArtificalStaleAbstractions(sjid)
 	ojidSendAbstractions := createArtificalStaleAbstractions(ojid)
@@ -333,21 +336,29 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
 		require.NoError(ctx, err)
 		snap2OjidCursorName, err := endpoint.ReplicationCursorBookmarkName(sfs, snap2.Guid, ojid)
 		require.NoError(ctx, err)
+		snap3SjidTentativeCursorName, err := endpoint.TentativeReplicationCursorBookmarkName(sfs, snap3.Guid, sjid)
+		require.NoError(ctx, err)
+		snap3OjidTentativeCursorName, err := endpoint.TentativeReplicationCursorBookmarkName(sfs, snap3.Guid, ojid)
+		require.NoError(ctx, err)
 		var bmNames []string
 		for _, bm := range sBms {
 			bmNames = append(bmNames, bm.Name)
 		}
 
 		if invalidateCacheBeforeSecondReplication {
-			require.Len(ctx, sBms, 3)
+			require.Len(ctx, sBms, 4)
 			require.Contains(ctx, bmNames, snap5SjidCursorName)
 			require.Contains(ctx, bmNames, snap2OjidCursorName)
+			require.Contains(ctx, bmNames, snap3OjidTentativeCursorName)
 			require.Contains(ctx, bmNames, "2")
 		} else {
-			require.Len(ctx, sBms, 4)
+			require.Len(ctx, sBms, 6)
+			ctx.Logf("%s", pretty.Sprint(sBms))
 			require.Contains(ctx, bmNames, snap5SjidCursorName)
 			require.Contains(ctx, bmNames, snap2SjidCursorName)
 			require.Contains(ctx, bmNames, snap2OjidCursorName)
+			require.Contains(ctx, bmNames, snap3SjidTentativeCursorName)
+			require.Contains(ctx, bmNames, snap3OjidTentativeCursorName)
 			require.Contains(ctx, bmNames, "2")
 		}
 	}
@@ -370,6 +381,84 @@ func implReplicationIncrementalCleansUpStaleAbstractions(ctx *platformtest.Conte
 
 }
 
+func ReplicationIncrementalHandlesFromVersionEqTentativeCursorCorrectly(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		CREATEROOT
+		+  "sender"
+		+  "sender@1"
+		+  "receiver"
+		R  zfs create -p "${ROOTDS}/receiver/${ROOTDS}"
+	`)
+
+	sjid := endpoint.MustMakeJobID("sender-job")
+	rjid := endpoint.MustMakeJobID("receiver-job")
+
+	sfs := ctx.RootDataset + "/sender"
+	rfsRoot := ctx.RootDataset + "/receiver"
+
+	rep := replicationInvocation{
+		sjid:    sjid,
+		rjid:    rjid,
+		sfs:     sfs,
+		rfsRoot: rfsRoot,
+		// It doesn't really matter what guarantee we use here, as the second replication will configure another.
+		// But, in the real world, the only way for a stale tentative cursor to appear is if the guarantee is set to
+		// incremental replication and we crash before converting the tentative cursor into a regular cursor.
+		guarantee: pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeIncrementalReplication),
+	}
+
+	// Do initial replication to set up the test.
+	rep1 := rep.Do(ctx)
+	ctx.Logf("\n%s", pretty.Sprint(rep1))
+	sfsDs := mustDatasetPath(sfs)
+	snap1_sender := mustGetFilesystemVersion(ctx, sfs+"@1")
+	snap1_replicationCursor_name, err := endpoint.ReplicationCursorBookmarkName(sfs, snap1_sender.Guid, sjid)
+	require.NoError(ctx, err)
+	snap1_replicationCursor := mustGetFilesystemVersion(ctx, sfs+"#"+snap1_replicationCursor_name)
+
+	// The second replication will be done with a guarantee kind that doesn't create tentative cursors by itself.
+	// So, it would generally be right to clean up any tentative cursors on sfs since they're stale abstractions.
+	// However, if the cursor is used as the `from` version in any send step, we must not destroy it, as that
+	// would break incremental replication.
+	// NB: we only need to test the first step as all subsequent steps will be snapshot->snapshot.
+	rep.guarantee = pdu.ReplicationConfigProtectionWithKind(pdu.ReplicationGuaranteeKind_GuaranteeNothing)
+	// create the artificial cursor
+	snap1_tentativeCursor, err := endpoint.CreateTentativeReplicationCursor(ctx, sfs, snap1_sender, sjid)
+	require.NoError(ctx, err)
+	endpoint.AbstractionsCacheInvalidate(sfs)
+	// remove other bookmarks of snap1, and snap1 itself, to force the replication planner to use the tentative cursor
+	err = zfs.ZFSDestroyFilesystemVersion(ctx, sfsDs, &snap1_sender)
+	require.NoError(ctx, err)
+	err = zfs.ZFSDestroyFilesystemVersion(ctx, sfsDs, &snap1_replicationCursor)
+	require.NoError(ctx, err)
+	versions, err := zfs.ZFSListFilesystemVersions(ctx, sfsDs, zfs.ListFilesystemVersionsOptions{})
+	require.NoError(ctx, err)
+	require.Len(ctx, versions, 1)
+	require.Equal(ctx, versions[0].Guid, snap1_tentativeCursor.GetFilesystemVersion().Guid)
+	// create another snapshot so that replication does one incremental step `tentative_cursor` -> `@2`
+	mustSnapshot(ctx, sfs+"@2")
+	mustGetFilesystemVersion(ctx, sfs+"@2")
+	// do the replication
+	rep2 := rep.Do(ctx)
+	ctx.Logf("\n%s", pretty.Sprint(rep2))
+
+	// Ensure that the tentative cursor was used.
+	require.Len(ctx, rep2.Attempts, 1)
+	require.Equal(ctx, rep2.Attempts[0].State, report.AttemptDone)
+	require.Len(ctx, rep2.Attempts[0].Filesystems, 1)
+	require.Nil(ctx, rep2.Attempts[0].Filesystems[0].Error())
+	require.Len(ctx, rep2.Attempts[0].Filesystems[0].Steps, 1)
+	require.EqualValues(ctx, rep2.Attempts[0].Filesystems[0].CurrentStep, 1)
+	require.Len(ctx, rep2.Attempts[0].Filesystems[0].Steps, 1)
+	require.Equal(ctx, rep2.Attempts[0].Filesystems[0].Steps[0].Info.From, snap1_tentativeCursor.GetFilesystemVersion().RelName())
+
+	// Ensure that the tentative cursor was destroyed as part of SendPost.
+	_, err = zfs.ZFSGetFilesystemVersion(ctx, snap1_replicationCursor.FullPath(sfs))
+	_, ok := err.(*zfs.DatasetDoesNotExist)
+	require.True(ctx, ok)
+}
+
 type PartialSender struct {
 	*endpoint.Sender
 	failAfterByteCount int64

replication/report/replication_report.go

@@ -192,3 +192,14 @@ func (r *Report) GetFailedFilesystemsCountInLatestAttempt() int {
 		return 0
 	}
 }
+
+// Returns true in case the AttemptState is a terminal
+// state(AttemptPlanningError, AttemptFanOutError, AttemptDone)
+func (a AttemptState) IsTerminal() bool {
+	switch a {
+	case AttemptPlanningError, AttemptFanOutError, AttemptDone:
+		return true
+	default:
+		return false
+	}
+}

rpc/dataconn/timeoutconn/timeoutconn.go

@@ -107,34 +107,31 @@ func (c *Conn) RenewWriteDeadline() error {
 	return c.SetWriteDeadline(time.Now().Add(c.idleTimeout))
 }
 
-func (c *Conn) Read(p []byte) (n int, err error) {
+func (c *Conn) Read(p []byte) (n int, _ error) {
 	n = 0
-	err = nil
 restart:
 	if err := c.renewReadDeadline(); err != nil {
 		return n, err
 	}
 	var nCurRead int
-	nCurRead, err = c.Wire.Read(p[n:])
+	nCurRead, err := c.Wire.Read(p[n:])
 	n += nCurRead
 	if netErr, ok := err.(net.Error); ok && netErr.Timeout() && nCurRead > 0 {
-		err = nil
 		goto restart
 	}
 	return n, err
 }
 
-func (c *Conn) Write(p []byte) (n int, err error) {
+func (c *Conn) Write(p []byte) (n int, _ error) {
 	n = 0
 restart:
 	if err := c.RenewWriteDeadline(); err != nil {
 		return n, err
 	}
 	var nCurWrite int
-	nCurWrite, err = c.Wire.Write(p[n:])
+	nCurWrite, err := c.Wire.Write(p[n:])
 	n += nCurWrite
 	if netErr, ok := err.(net.Error); ok && netErr.Timeout() && nCurWrite > 0 {
-		err = nil
 		goto restart
 	}
 	return n, err
@@ -144,17 +141,16 @@ restart:
 // but is guaranteed to use the writev system call if the wrapped Wire
 // support it.
 // Note the Conn does not support writev through io.Copy(aConn, aNetBuffers).
-func (c *Conn) WritevFull(bufs net.Buffers) (n int64, err error) {
+func (c *Conn) WritevFull(bufs net.Buffers) (n int64, _ error) {
 	n = 0
 restart:
 	if err := c.RenewWriteDeadline(); err != nil {
 		return n, err
 	}
 	var nCurWrite int64
-	nCurWrite, err = io.Copy(c.Wire, &bufs)
+	nCurWrite, err := io.Copy(c.Wire, &bufs)
 	n += nCurWrite
 	if netErr, ok := err.(net.Error); ok && netErr.Timeout() && nCurWrite > 0 {
-		err = nil
 		goto restart
 	}
 	return n, err

util/suspendresumesafetimer/suspendresumesafetimer.go

@@ -0,0 +1,96 @@
+package suspendresumesafetimer
+
+import (
+	"context"
+	"time"
+
+	"github.com/zrepl/zrepl/util/envconst"
+)
+
+// The returned error is guaranteed to be the ctx.Err()
+func SleepUntil(ctx context.Context, sleepUntil time.Time) error {
+
+	// We use .Round(0) to strip the monotonic clock reading from the time.Time
+	// returned by time.Now(). That will make the before/after check in the ticker
+	// for-loop compare wall-clock times instead of monotonic time.
+	// Comparing wall clock time is necessary because monotonic time does not progress
+	// while the system is suspended.
+	//
+	// Background
+	//
+	// A time.Time carries a wallclock timestamp and optionally a monotonic clock timestamp.
+	// time.Now() returns a time.Time that carries both.
+	// time.Time.Add() applies the same delta to both timestamps in the time.Time.
+	// x.Sub(y) will return the *monotonic* delta if both x and y carry a monotonic timestamp.
+	// time.Until(x) == x.Sub(now) where `now` will have a monotonic timestamp.
+	// So, time.Until(x) with an `x` that has monotonic timestamp will return monotonic delta.
+	//
+	// Why Do We Care?
+	//
+	// On systems that suspend/resume, wall clock time progresses during suspend but
+	// monotonic time does not.
+	//
+	// So, suppose the following sequence of events:
+	//   x <== time.Now()
+	//   System suspends for 1 hour
+	//   delta <== time.Now().Sub(x)
+	// `delta` will be near 0 because time.Until() subtracts the monotonic
+	// timestamps, and monotonic time didn't progress during suspend.
+	//
+	// Now strip the timestamp using .Round(0)
+	//   x <== time.Now().Round(0)
+	//   System suspends for 1 hour
+	//   delta <== time.Now().Sub(x)
+	// `delta` will be 1 hour because time.Sub() subtracted wallclock timestamps
+	// because x didn't have a monotonic timestamp because we stripped it using .Round(0).
+	//
+	//
+	sleepUntil = sleepUntil.Round(0)
+
+	// Set up a timer so that, if the system doesn't suspend/resume,
+	// we get a precise wake-up time from the native Go timer.
+	monotonicClockTimer := time.NewTimer(time.Until(sleepUntil))
+	defer func() {
+		if !monotonicClockTimer.Stop() {
+			// non-blocking read since we can come here when
+			// we've already drained the channel through
+			//		case <-monotonicClockTimer.C
+			// in the `for` loop below.
+			select {
+			case <-monotonicClockTimer.C:
+			default:
+			}
+		}
+	}()
+
+	// Set up a ticker so that we're guaranteed to wake up periodically.
+	// We'll then get the current wall-clock time and check ourselves
+	// whether we're past the requested expiration time.
+	// Pick a 10 second check interval by default since it's rare that
+	// suspend/resume is done more frequently.
+	ticker := time.NewTicker(envconst.Duration("ZREPL_WALLCLOCKTIMER_MAX_DELAY", 10*time.Second))
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-monotonicClockTimer.C:
+			return nil
+		case <-ticker.C:
+			now := time.Now()
+			if now.Before(sleepUntil) {
+				// Continue waiting.
+
+				// Reset the monotonic timer to reset drift.
+				if !monotonicClockTimer.Stop() {
+					<-monotonicClockTimer.C
+				}
+				monotonicClockTimer.Reset(time.Until(sleepUntil))
+
+				continue
+			}
+			return nil
+		case <-ctx.Done():
+			return ctx.Err()
+		}
+	}
+}

zfs/mapping.go

@@ -3,12 +3,20 @@ package zfs
 import (
 	"context"
 	"fmt"
+
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
 type DatasetFilter interface {
 	Filter(p *DatasetPath) (pass bool, err error)
+	// The caller owns the returned set.
+	// Implementations should return a copy.
+	UserSpecifiedDatasets() UserSpecifiedDatasetsSet
 }
 
+// A set of dataset names that the user specified in the configuration file.
+type UserSpecifiedDatasetsSet map[string]bool
+
 // Returns a DatasetFilter that does not filter (passes all paths)
 func NoFilter() DatasetFilter {
 	return noFilter{}
@@ -18,7 +26,8 @@ type noFilter struct{}
 
 var _ DatasetFilter = noFilter{}
 
-func (noFilter) Filter(p *DatasetPath) (pass bool, err error) { return true, nil }
+func (noFilter) Filter(p *DatasetPath) (pass bool, err error)    { return true, nil }
+func (noFilter) UserSpecifiedDatasets() UserSpecifiedDatasetsSet { return nil }
 
 func ZFSListMapping(ctx context.Context, filter DatasetFilter) (datasets []*DatasetPath, err error) {
 	res, err := ZFSListMappingProperties(ctx, filter, nil)
@@ -61,6 +70,7 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert
 
 	go ZFSListChan(ctx, rchan, properties, nil, "-r", "-t", "filesystem,volume")
 
+	unmatchedUserSpecifiedDatasets := filter.UserSpecifiedDatasets()
 	datasets = make([]ZFSListMappingPropertiesResult, 0)
 	for r := range rchan {
 
@@ -74,6 +84,8 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert
 			return
 		}
 
+		delete(unmatchedUserSpecifiedDatasets, path.ToString())
+
 		pass, filterErr := filter.Filter(path)
 		if filterErr != nil {
 			return nil, fmt.Errorf("error calling filter: %s", filterErr)
@@ -87,5 +99,9 @@ func ZFSListMappingProperties(ctx context.Context, filter DatasetFilter, propert
 
 	}
 
+	jobid := zfscmd.GetJobIDOrDefault(ctx, "__nojobid")
+	metric := prom.ZFSListUnmatchedUserSpecifiedDatasetCount.WithLabelValues(jobid)
+	metric.Add(float64(len(unmatchedUserSpecifiedDatasets)))
+
 	return
 }

zfs/prometheus.go

@@ -3,10 +3,11 @@ package zfs
 import "github.com/prometheus/client_golang/prometheus"
 
 var prom struct {
-	ZFSListFilesystemVersionDuration *prometheus.HistogramVec
+	ZFSListFilesystemVersionDuration          *prometheus.HistogramVec
-	ZFSSnapshotDuration              *prometheus.HistogramVec
+	ZFSSnapshotDuration                       *prometheus.HistogramVec
-	ZFSBookmarkDuration              *prometheus.HistogramVec
+	ZFSBookmarkDuration                       *prometheus.HistogramVec
-	ZFSDestroyDuration               *prometheus.HistogramVec
+	ZFSDestroyDuration                        *prometheus.HistogramVec
+	ZFSListUnmatchedUserSpecifiedDatasetCount *prometheus.GaugeVec
 }
 
 func init() {
@@ -34,6 +35,15 @@ func init() {
 		Name:      "destroy_duration",
 		Help:      "Duration it took to destroy a dataset",
 	}, []string{"dataset_type", "filesystem"})
+	prom.ZFSListUnmatchedUserSpecifiedDatasetCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "zrepl",
+		Subsystem: "zfs",
+		Name:      "list_unmatched_user_specified_dataset_count",
+		Help: "When evaluating a DatsetFilter against zfs list output, this counter " +
+			"is incremented for every DatasetFilter rule that did not match any " +
+			"filesystem name in the zfs list output. Monitor for increases to detect filesystem " +
+			"filter rules that have no effect because they don't match any local filesystem.",
+	}, []string{"jobid"})
 }
 
 func PrometheusRegister(registry prometheus.Registerer) error {
@@ -49,5 +59,8 @@ func PrometheusRegister(registry prometheus.Registerer) error {
 	if err := registry.Register(prom.ZFSDestroyDuration); err != nil {
 		return err
 	}
+	if err := registry.Register(prom.ZFSListUnmatchedUserSpecifiedDatasetCount); err != nil {
+		return err
+	}
 	return nil
 }

zfs/zfs.go

@@ -343,7 +343,6 @@ const (
 
 type SendStream struct {
 	cmd          *zfscmd.Cmd
-	kill         context.CancelFunc
 	stdoutReader io.ReadCloser // not *os.File for mocking during platformtest
 	stderrBuf    *circlog.CircularLog
 
@@ -416,8 +415,29 @@ func (s *SendStream) killAndWait() error {
 	debug("sendStream: killAndWait enter")
 	defer debug("sendStream: killAndWait leave")
 
+	// ensure this function is only called once
+	if s.state != sendStreamOpen {
+		panic(s.state)
+	}
+
 	// send SIGKILL
-	s.kill()
+	// In an earlier version, we used the starting context.Context's cancel function
+	// for this, but in Go > 1.19, doing so will cause .Wait() to return the
+	// context cancel error instead of the *exec.ExitError.
+	err := s.cmd.Process().Kill()
+	if err != nil {
+		if err == os.ErrProcessDone {
+			// This can happen if
+			// (1) the process has already been .Wait()ed, or
+			// (2) some other goroutine cancels the context, likely further up
+			//     the context tree.
+			// Case (1) can't happen to us because we only call
+			// this function in sendStreamOpen state.
+			// In Case (2), it's still our job to .Wait(), so, fallthrough.
+		} else {
+			return err
+		}
+	}
 
 	// Close our read-end of the pipe.
 	//
@@ -963,10 +983,10 @@ func ZFSSend(ctx context.Context, sendArgs ZFSSendArgsValidated) (*SendStream, e
 
 	stream := &SendStream{
 		cmd:          cmd,
-		kill:         cancel,
 		stdoutReader: stdoutReader,
 		stderrBuf:    stderrBuf,
 	}
+	_ = cancel // the SendStream.killAndWait() will kill the process
 
 	return stream, nil
 }

zfs/zfscmd/zfscmd_context.go

@@ -19,6 +19,10 @@ func WithJobID(ctx context.Context, jobID string) context.Context {
 	return context.WithValue(ctx, contextKeyJobID, jobID)
 }
 
+func GetJobIDOrDefault(ctx context.Context, def string) string {
+	return getJobIDOrDefault(ctx, def)
+}
+
 func getJobIDOrDefault(ctx context.Context, def string) string {
 	ret, ok := ctx.Value(contextKeyJobID).(string)
 	if !ok {