experiment: zync: an rsync-like tool for ZFS datasets (re-uses zrepl abstractions)

Local sync and sync over SSH seems to work.
The sshdirect transport might be interesting / re-usable for zrepl as well, although we'd still need some kind of locking in that case.

.circleci/config.yml

@@ -1,74 +1,79 @@
-version: 2.0
+version: 2.1
-workflows:
-  version: 2
-  build:
-    jobs:
-      - build-1.11
-      - build-1.12
-      - build-1.13
-      - build-1.14
-      - build-latest
-      - test-build-in-docker
-jobs:
 
-  # build-latest serves as the template
+commands:
-  # we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
+  setup-home-local-bin:
-  build-latest: &build-latest
-    description: Builds zrepl
-    parameters:
-      image:
-        description: "the docker image that the job should use"
-        type: string
-    docker:
-      - image: circleci/golang:latest
-    environment:
-      # required by lazy.sh
-      TERM: xterm
-    working_directory: /go/src/github.com/zrepl/zrepl
     steps:
       - run:
-          name: Setup environment variables
+          shell: /bin/bash -euo pipefail
           command: |
-            # used by pip (for docs)
+            mkdir -p "$HOME/.local/bin"
-            echo 'export PATH="$HOME/.local/bin:$PATH"' >> $BASH_ENV
+            line='export PATH="$HOME/.local/bin:$PATH"'
-            # we use modules
+            if grep "$line" $BASH_ENV >/dev/null; then
-            echo 'export GO111MODULE=on' >> $BASH_ENV
+              echo "$line" >> $BASH_ENV
+            fi
 
+  invoke-lazy-sh:
+    parameters:
+      subcommand:
+        type: string
+    steps:
+      - run:
+          environment:
+            TERM: xterm
+          command: ./lazy.sh <<parameters.subcommand>>
+
+  apt-update-and-install-common-deps:
+    steps:
+      - run: sudo apt update && sudo apt install gawk make
+
+  restore-cache-gomod:
+    steps:
       - restore_cache:
-          keys:
+          key: go-mod-v4-{{ checksum "go.sum" }}
-            - source
+  save-cache-gomod:
-            - protobuf
+    steps:
-
-      - checkout
-
       - save_cache:
-          key: source
+          key: go-mod-v4-{{ checksum "go.sum" }}
           paths:
-            - ".git"
+            - "/go/pkg/mod"
-      
+
-      # install deps
+  install-godep:
-      - run: wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
+    steps:
-      - run: echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
+      - apt-update-and-install-common-deps
-      - run: sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
+      - invoke-lazy-sh:
+          subcommand: godep
+
+  install-docdep:
+    steps:
+      - apt-update-and-install-common-deps
+      - run: sudo apt install python3 python3-pip libgirepository1.0-dev
+      - invoke-lazy-sh:
+          subcommand: docdep
+
+  download-and-install-minio-client:
+    steps:
+      - setup-home-local-bin
+      - restore_cache:
+          key: minio-client-v2
+      - run:
+          shell: /bin/bash -eo pipefail
+          command: |
+            if which mc; then exit 0; fi
+            sudo curl -sSL https://dl.min.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2020-08-20T00-23-01Z \
+              -o "$HOME/.local/bin/mc"
+            sudo chmod +x "$HOME/.local/bin/mc"
       - save_cache:
-          key: protobuf
+          key: minio-client-v2
           paths:
-            - "/usr/include/google/protobuf"
+            - "$HOME/.local/bin/mc"
-            
-      - run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev gawk
-      - run: ./lazy.sh devsetup
-
-      - run: make zrepl-bin
-      - run: make vet
-      - run: make lint
-      - run: make release
-      - run: make test-go
-      # cannot run test-platform because circle-ci runs in linux containers
-
-      - store_artifacts:
-          path: ./artifacts/release
-          when: always
 
+  upload-minio:
+    parameters:
+      src:
+        type: string
+      dst:
+        type: string
+    steps:
       - run:
           shell: /bin/bash -eo pipefail
           when: always
@@ -78,80 +83,296 @@ jobs:
               exit 0
             fi
             set -u # from now on
-            
+
-            # Download and install minio
-            curl -sSL https://dl.minio.io/client/mc/release/linux-amd64/mc -o ${GOPATH}/bin/mc
-            chmod +x ${GOPATH}/bin/mc
             mc config host add --api s3v4 zrepl-minio https://minio.cschwarz.com ${MINIO_ACCESS_KEY} ${MINIO_SECRET_KEY}
-            
+
+            # keep in sync with set-github-minio-status
+            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
+
             # Upload artifacts
-            echo "$CIRCLE_BUILD_URL" > ./artifacts/release/cirlceci_build_url
+            mkdir -p ./artifacts
-            mc cp -r artifacts/release "zrepl-minio/zrepl-ci-artifacts/${CIRCLE_SHA1}/${CIRCLE_JOB}/"
+            mc cp -r <<parameters.src>> "zrepl-minio/$jobprefix/<<parameters.dst>>"
 
-            # Push Artifact Link to GitHub
+  set-github-minio-status:
-            REPO="zrepl/zrepl"
+    parameters:
-            COMMIT="${CIRCLE_SHA1}"
+      context:
-            JOB_NAME="${CIRCLE_JOB}"
+        type: string
-            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
+      description:
-              -H "Content-Type: application/json" \
+        type: string
-              -H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
+      minio-dst:
-              -X POST \
+        type: string
-              -d '{"context":"zrepl/publish-ci-artifacts", "state": "success", "description":"CI Build Artifacts for '"$JOB_NAME"'", "target_url":"https://minio.cschwarz.com/minio/zrepl-ci-artifacts/'"$COMMIT"'/"}'
+    steps:
-
-      # kick off binary packaging workflow
       - run:
           shell: /bin/bash -eo pipefail
           command: |
             if [ -n "$CIRCLE_PR_NUMBER" ]; then # CIRCLE_PR_NUMBER is guaranteed to be only present in forked PRs (external)
-              echo "Forked PR detected. Sry, can't trust you with credentials."
+              echo "Forked PR detected. Sry, can't trust you with credentials to external artifact store, use CircleCI's instead."
               exit 0
             fi
             set -u # from now on
-            GITHUB_ACCESS_TOKEN="$ZREPL_DEBIAN_BINARYPACKAGIN_TRIGGER_BUILD_GITHUB_TOKEN" .circleci/trigger_debian_binary_packaging_workflow.bash "$CIRCLE_SHA1" "${CIRCLE_JOB##build-}"
 
-  build-1.11:
+            # keep in sync with with upload-minio command
-    <<: *build-latest
+            jobprefix=zrepl-ci-artifacts/${CIRCLE_SHA1}-pipeline-<<pipeline.number>>/${CIRCLE_JOB}
-    docker:
+            # Push Artifact Link to GitHub
-    - image: circleci/golang:1.11
+            REPO="zrepl/zrepl"
+            COMMIT="${CIRCLE_SHA1}"
+            JOB_NAME="${CIRCLE_JOB}"
+            CONTEXT="<<parameters.context>>"
+            DESCRIPTION="<<parameters.description>>"
+            TARGETURL=https://minio.cschwarz.com/minio/"$jobprefix"/"<<parameters.minio-dst>>"
+            curl "https://api.github.com/repos/$REPO/statuses/$COMMIT" \
+              -H "Content-Type: application/json" \
+              -H "Authorization: token $GITHUB_COMMIT_STATUS_TOKEN" \
+              -X POST \
+              -d '{"context":"'"$CONTEXT"'", "state": "success", "description":"'"$DESCRIPTION"'", "target_url":"'"$TARGETURL"'"}'
 
-  build-1.12:
-    <<: *build-latest
-    docker:
-    - image: circleci/golang:1.12
 
-  build-1.13:
+  trigger-pipeline:
-    <<: *build-latest
+    parameters:
-    docker:
+      body_no_shell_subst:
-    - image: circleci/golang:1.13
+        type: string
-
-  build-1.14:
-    <<: *build-latest
-    docker:
-    - image: circleci/golang:1.14
-
-  # this job tries to mimic the build-in-docker instructions
-  # given in docs/installation.rst
-  #
-  # However, CircleCI doesn't support volume mounts, so we have to copy
-  # the source into the build-container by modifying the Dockerfile
-  test-build-in-docker:
-    description: Check that build-in-docker works
-    docker:
-      - image: circleci/golang:latest
-    environment:
-    working_directory: /go/src/github.com/zrepl/zrepl
     steps:
-    - checkout
+      - run: |
-    - setup_remote_docker
+          curl -X POST https://circleci.com/api/v2/project/github/zrepl/zrepl/pipeline \
-    - run:
+            -H 'Content-Type: application/json' \
-        name: (hacky) circleci doesn't allow volume mounts, so copy src to container
+            -H 'Accept: application/json' \
-        command: echo "ADD . /src" >> build.Dockerfile
+            -H "Circle-Token: $ZREPL_BOT_CIRCLE_TOKEN" \
-    - run:
+            --data '<<parameters.body_no_shell_subst>>'
-        name: (hacky) commit modified Dockerfile to avoid failing git clean check in Makefile
+
-        command: git -c user.name='circleci' -c user.email='circleci@localhost' commit -m 'CIRCLECI modified Dockerfile with zrepl src' --author 'autoauthor <circleci@localhost>' -- build.Dockerfile
+parameters:
-    - run:
+  do_ci:
-        name: build the build image (build deps)
+    type: boolean
-        command: docker build -t zrepl_build -f build.Dockerfile .
+    default: true
-    - run:
+
-        name: try compiling
+  do_release:
-        command: docker run -it zrepl_build make release
+    type: boolean
+    default: false
+
+  release_docker_baseimage_tag:
+    type: string
+    default: "1.15"
+
+workflows:
+  version: 2
+
+  ci:
+    when: << pipeline.parameters.do_ci >>
+    jobs:
+      - quickcheck-docs
+      - quickcheck-go: &quickcheck-go-smoketest
+          name: quickcheck-go-amd64-linux-1.15
+          goversion: &latest-go-release "1.15"
+          goos: linux
+          goarch: amd64
+      - test-go-on-latest-go-release:
+          goversion: *latest-go-release
+      - quickcheck-go:
+          requires:
+            - quickcheck-go-amd64-linux-1.15 #quickcheck-go-smoketest.name
+          matrix: &quickcheck-go-matrix
+            alias: quickcheck-go-matrix
+            parameters:
+              goversion: [*latest-go-release, "1.11"]
+              goos: ["linux", "freebsd"]
+              goarch: ["amd64", "arm64"]
+            exclude:
+              # don't re-do quickcheck-go-smoketest
+              - goversion: *latest-go-release
+                goos: linux
+                goarch: amd64
+              # not supported by Go 1.11
+              - goversion: "1.11"
+                goos: freebsd
+                goarch: arm64
+
+  release:
+    when: << pipeline.parameters.do_release >>
+    jobs:
+      - release-build
+      - release-deb:
+          requires:
+            - release-build
+      - release-rpm:
+          requires:
+            - release-build
+      - release-upload:
+          requires:
+            - release-build
+            - release-deb
+            - release-rpm
+
+  periodic:
+    triggers:
+      - schedule:
+          cron: "00 17 * * *"
+          filters:
+            branches:
+              only:
+                - master
+                - stable
+                - problame/circleci-build
+    jobs:
+      - periodic-full-pipeline-run
+
+  zrepl.github.io:
+    jobs:
+      - publish-zrepl-github-io:
+          filters:
+            branches:
+              only:
+                - stable
+
+jobs:
+  quickcheck-docs:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - checkout
+      - install-docdep
+      - run: make docs
+
+      - store_artifacts:
+          path: artifacts
+      - download-and-install-minio-client
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/${CIRCLE_JOB}
+          description: artifacts of CI job ${CIRCLE_JOB}
+          minio-dst: ""
+
+  quickcheck-go:
+    parameters:
+      goversion:
+        type: string
+      goos:
+        type: string
+      goarch:
+        type: string
+    docker:
+      - image: circleci/golang:<<parameters.goversion>>
+    environment:
+      GOOS: <<parameters.goos>>
+      GOARCH: <<parameters.goarch>>
+
+    steps:
+      - checkout
+
+      - restore-cache-gomod
+      - run: go mod download
+      - run: cd build && go mod download
+      - save-cache-gomod
+
+      - install-godep
+      - run: make formatcheck
+      - run: make generate-platform-test-list
+      - run: make zrepl-bin test-platform-bin
+      - run: make vet
+      - run: make lint
+
+      - download-and-install-minio-client
+      - run: rm -f artifacts/generate-platform-test-list
+      - store_artifacts:
+          path: artifacts
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/${CIRCLE_JOB}
+          description: artifacts of CI job ${CIRCLE_JOB}
+          minio-dst: ""
+
+  test-go-on-latest-go-release:
+    parameters:
+      goversion:
+        type: string
+    docker:
+      - image: circleci/golang:<<parameters.goversion>>
+    steps:
+      - checkout
+      - restore-cache-gomod
+      - run: make test-go
+      # don't save-cache-gomod here, test-go doesn't pull all the dependencies
+
+  release-build:
+    machine: true
+    steps:
+      - checkout
+      - run: make release-docker RELEASE_DOCKER_BASEIMAGE_TAG=<<pipeline.parameters.release_docker_baseimage_tag>>
+      - persist_to_workspace:
+          root: .
+          paths: [.]
+  release-deb:
+    machine: true
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make debs-docker
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "artifacts/*.deb"
+
+  release-rpm:
+    machine: true
+    steps:
+      - attach_workspace:
+          at: .
+      - run: make rpms-docker
+      - persist_to_workspace:
+          root: .
+          paths:
+            - "artifacts/*.rpm"
+
+  release-upload:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - attach_workspace:
+          at: .
+      - store_artifacts:
+          path: artifacts
+      - download-and-install-minio-client
+      - upload-minio:
+          src: artifacts
+          dst: ""
+      - set-github-minio-status:
+          context: artifacts/release
+          description: CI-generated release artifacts
+          minio-dst: ""
+
+  periodic-full-pipeline-run:
+    docker:
+      - image: cimg/base:2020.08
+    steps:
+      - trigger-pipeline:
+          body_no_shell_subst: '{"branch":"<<pipeline.git.branch>>", "parameters": { "do_ci": true, "do_release": true }}'
+
+  publish-zrepl-github-io:
+    docker:
+      - image: cimg/python:3.7
+    steps:
+      - checkout
+      - invoke-lazy-sh:
+          subcommand: docdep
+      - run:
+          command: |
+            git config --global user.email "me@cschwarz.com"
+            git config --global user.name "zrepl-github-io-ci"
+
+      # https://circleci.com/docs/2.0/add-ssh-key/#adding-multiple-keys-with-blank-hostnames
+      - run: ssh-add -D
+      # the default circleci ssh config only additional ssh keys for Host !github.com
+      - run:
+          command: |
+            cat > ~/.ssh/config \<<EOF
+            Host *
+              IdentityFile /home/circleci/.ssh/id_rsa_458e62c517f6c480e40452126ce47421
+            EOF
+      - add_ssh_keys:
+          fingerprints:
+            # deploy key for zrepl.github.io
+            - "45:8e:62:c5:17:f6:c4:80:e4:04:52:12:6c:e4:74:21"
+
+      - run: bash -x docs/publish.sh -c -a

.travis.yml

@@ -1,70 +0,0 @@
-dist: xenial
-services:
-- docker
-
-env: # for allow_failures: https://docs.travis-ci.com/user/customizing-the-build/
-matrix:
-  include:
-
-  - language: go
-    name: "Build in Docker (docs/installation.rst)"
-    script:
-    - sudo docker build -t zrepl_build -f build.Dockerfile .
-    - |
-      sudo docker run -it --rm \
-        -v "${PWD}:/go/src/github.com/zrepl/zrepl" \
-        --user "$(id -u):$(id -g)" \
-        zrepl_build make vendordeps release
-
-  - &zrepl_build_template
-    language: go
-    go_import_path: github.com/zrepl/zrepl
-    before_install:
-        - wget https://github.com/protocolbuffers/protobuf/releases/download/v3.6.1/protoc-3.6.1-linux-x86_64.zip
-        - echo "6003de742ea3fcf703cfec1cd4a3380fd143081a2eb0e559065563496af27807  protoc-3.6.1-linux-x86_64.zip" | sha256sum -c
-        - sudo unzip -d /usr protoc-3.6.1-linux-x86_64.zip
-        - ./lazy.sh godep
-        - make vendordeps
-    script:
-      - make
-      - make vet
-      - make test
-      - make lint
-      - make artifacts/zrepl-freebsd-amd64
-      - make artifacts/zrepl-linux-amd64
-      - make artifacts/zrepl-darwin-amd64
-    go:
-    - "1.11"
-
-  - <<: *zrepl_build_template
-    go:
-    - "1.12"
-
-  - <<: *zrepl_build_template
-    go:
-    - "master"
-
-  - &zrepl_docs_template 
-    language: python
-    python:
-    - "3.4"
-    install:
-    - sudo apt-get install libgirepository1.0-dev
-    - pip install -r docs/requirements.txt
-    script:
-    - make docs
-  - <<: *zrepl_docs_template
-    python:
-    - "3.5"
-  - <<: *zrepl_docs_template
-    python:
-    - "3.6"
-  - <<: *zrepl_docs_template
-    python:
-    - "3.7"
-  
-
-  allow_failures:
-  - <<: *zrepl_build_template
-    go:
-    - "master"

Makefile

@@ -28,6 +28,8 @@ GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
 GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
 GOLANGCI_LINT := golangci-lint
 GOCOVMERGE := gocovmerge
+RELEASE_DOCKER_BASEIMAGE_TAG ?= 1.15
+RELEASE_DOCKER_BASEIMAGE ?= golang:$(RELEASE_DOCKER_BASEIMAGE_TAG)
 
 ifneq ($(GOARM),)
 	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
@@ -57,6 +59,85 @@ ifeq (SIGN, 1)
 endif
 	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
 
+release-docker: $(ARTIFACTDIR)
+	sed 's/FROM.*!SUBSTITUTED_BY_MAKEFILE/FROM $(RELEASE_DOCKER_BASEIMAGE)/' build.Dockerfile > artifacts/release-docker.Dockerfile
+	docker build -t zrepl_release --pull -f artifacts/release-docker.Dockerfile .
+	docker run --rm -i -v $(CURDIR):/src -u $$(id -u):$$(id -g) \
+		zrepl_release \
+		make release GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+debs-docker:
+	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=deb
+rpms-docker:
+	$(MAKE) _debs_or_rpms_docker _DEB_OR_RPM=rpm
+_debs_or_rpms_docker: # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=amd64
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm64
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=arm GOARM=7
+	$(MAKE) $(_DEB_OR_RPM)-docker GOOS=linux GOARCH=386
+
+rpm: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+	$(eval _ZREPL_RPM_VERSION := $(subst -,.,$(_ZREPL_VERSION)))
+	$(eval _ZREPL_RPM_TOPDIR_ABS := $(CURDIR)/$(ARTIFACTDIR)/rpmbuild)
+	rm -rf "$(_ZREPL_RPM_TOPDIR_ABS)"
+	mkdir "$(_ZREPL_RPM_TOPDIR_ABS)"
+	mkdir -p "$(_ZREPL_RPM_TOPDIR_ABS)"/{SPECS,RPMS,BUILD,BUILDROOT}
+	sed "s/^Version:.*/Version:          $(_ZREPL_RPM_VERSION)/g" \
+	    packaging/rpm/zrepl.spec >  $(_ZREPL_RPM_TOPDIR_ABS)/SPECS/zrepl.spec
+
+	# see /usr/lib/rpm/platform
+ifeq ($(GOARCH),amd64)
+	$(eval _ZREPL_RPMBUILD_TARGET := x86_64)
+else ifeq ($(GOARCH), 386)
+	$(eval _ZREPL_RPMBUILD_TARGET := i386)
+else ifeq ($(GOARCH), arm64)
+	$(eval _ZREPL_RPMBUILD_TARGET := aarch64)
+else ifeq ($(GOARCH), arm)
+	$(eval _ZREPL_RPMBUILD_TARGET := armv7hl)
+else
+	$(eval _ZREPL_RPMBUILD_TARGET := $(GOARCH))
+endif
+	rpmbuild \
+		--build-in-place \
+		--define "_sourcedir $(CURDIR)" \
+		--define "_topdir $(_ZREPL_RPM_TOPDIR_ABS)" \
+		--define "_zrepl_binary_filename zrepl-$(ZREPL_TARGET_TUPLE)" \
+		--target $(_ZREPL_RPMBUILD_TARGET) \
+		-bb "$(_ZREPL_RPM_TOPDIR_ABS)"/SPECS/zrepl.spec
+	cp "$(_ZREPL_RPM_TOPDIR_ABS)"/RPMS/$(_ZREPL_RPMBUILD_TARGET)/zrepl-$(_ZREPL_RPM_VERSION)*.rpm $(ARTIFACTDIR)/
+
+rpm-docker:
+	docker build -t zrepl_rpm_pkg --pull -f packaging/rpm/Dockerfile .
+	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		zrepl_rpm_pkg \
+		make rpm GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
+deb: $(ARTIFACTDIR) # artifacts/_zrepl.zsh_completion artifacts/bash_completion docs zrepl-bin
+
+	cp packaging/deb/debian/changelog.template packaging/deb/debian/changelog
+	sed -i 's/DATE_DASH_R_OUTPUT/$(shell date -R)/' packaging/deb/debian/changelog
+	VERSION="$(subst -,.,$(_ZREPL_VERSION))"; \
+		export VERSION="$${VERSION#v}"; \
+		sed -i 's/VERSION/'"$$VERSION"'/' packaging/deb/debian/changelog
+
+ifeq ($(GOARCH), arm)
+	$(eval  DEB_HOST_ARCH := armhf)
+else ifeq ($(GOARCH), 386)
+	$(eval DEB_HOST_ARCH := i386)
+else
+	$(eval DEB_HOST_ARCH := $(GOARCH))
+endif
+
+	export ZREPL_DPKG_ZREPL_BINARY_FILENAME=zrepl-$(ZREPL_TARGET_TUPLE); \
+		dpkg-buildpackage -b --no-sign --host-arch $(DEB_HOST_ARCH)
+	cp ../*.deb artifacts/
+
+deb-docker:
+	docker build -t zrepl_debian_pkg --pull -f packaging/deb/Dockerfile .
+	docker run --rm -i -v $(CURDIR):/build/src -u $$(id -u):$$(id -g) \
+		zrepl_debian_pkg \
+		make deb GOOS=$(GOOS) GOARCH=$(GOARCH) GOARM=$(GOARM)
+
 # expects `release` target to have run before
 NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
 wrapup-and-checksum:
@@ -173,7 +254,7 @@ ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
 ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
 ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
 # ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
-ZREPL_PLATFORMTEST_ARGS := 
+ZREPL_PLATFORMTEST_ARGS :=
 _test-or-cover-platform-impl: $(ARTIFACTDIR)
 ifndef _TEST_PLATFORM_CMD
 	$(error _TEST_PLATFORM_CMD is undefined, caller 'cover-platform' or 'test-platform' should have defined it)
@@ -202,14 +283,22 @@ cover-full:
 
 ##################### DEV TARGETS #####################
 # not part of the build, must do that manually
-.PHONY: generate format
+.PHONY: generate formatcheck format
 
 generate: generate-platform-test-list
 	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
 	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
 
+GOIMPORTS := goimports -srcdir . -local 'github.com/zrepl/zrepl'
+FINDSRCFILES := find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go'
+
+formatcheck:
+	@# goimports doesn't have a knob to exit with non-zero status code if formatting is needed
+	@# see https://go-review.googlesource.com/c/tools/+/237378
+	@ affectedfiles=$$($(GOIMPORTS) -l $(shell $(FINDSRCFILES)) | tee /dev/stderr | wc -l); test "$$affectedfiles" = 0
+
 format:
-	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
+	@ $(GOIMPORTS) -w -d $(shell  $(FINDSRCFILES))
 
 ##################### NOARCH #####################
 .PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
@@ -233,11 +322,14 @@ $(ARTIFACTDIR)/_zrepl.zsh_completion:
 
 $(ARTIFACTDIR)/go_env.txt:
 	$(GO_ENV_VARS) $(GO) env > $@
+	$(GO) version >> $@
 
 docs: $(ARTIFACTDIR)/docs
+	# https://www.sphinx-doc.org/en/master/man/sphinx-build.html
 	make -C docs \
 		html \
 		BUILDDIR=../artifacts/docs \
+		SPHINXOPTS="-W --keep-going -n"
 
 docs-clean:
 	make -C docs \

README.md

@@ -1,7 +1,7 @@
 [![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
 [![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
 [![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
-[![Donate via Patreon](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow)](https://www.patreon.com/zrepl)
+[![Support me on Patreon](https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&suffix=%20patrons&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079)](https://patreon.com/zrepl)
 [![Donate via GitHub Sponsors](https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow)](https://github.com/sponsors/problame)
 [![Donate via Liberapay](https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
 [![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
@@ -30,20 +30,12 @@ zrepl is a one-stop ZFS backup & replication solution.
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
 
-## Package Maintainer Information
+## Building, Releasing, Downstream-Packaging
 
-* Follow the steps in `docs/installation.rst -> Compiling from Source` and read the Makefile / shell scripts used in this process.
+This section provides an overview of the zrepl build & release process.
-* Make sure your distro is compatible with the paths in `docs/installation.rst`.
+Check out `docs/installation/compile-from-source.rst` for build-from-source instructions.
-* Ship a default config that adheres to your distro's `hier` and logging system.
-* Ship a service manager file and _please_ try to upstream it to this repository.
-  * `dist/systemd` contains a Systemd unit template.
-* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
-* Use `make release ZREPL_VERSION='mydistro-1.2.3_1'`
-    * Your distro's name and any versioning supplemental to zrepl's (e.g. package revision) should be in this string
-* Use `sudo make test-platform` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
-* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
 
-## Developer Documentation
+### Overview
 
 zrepl is written in [Go](https://golang.org) and uses [Go modules](https://github.com/golang/go/wiki/Modules) to manage dependencies.
 The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
@@ -60,60 +52,61 @@ An HTML report can be generated using `make cover-html`.
 
 **Code generation** is triggered by `make generate`. Generated code is committed to the source tree.
 
-### Project Structure
+### Build & Release Process
 
-```
+**The `Makefile` is catering to the needs of developers & CI, not distro packagers**.
-├── artifacts               # build artifcats generate by make
+It provides phony targets for
-├── cli                     # wrapper around CLI package cobra
+* local development (building, running tests, etc)
-├── client                  # all subcommands that are not `daemon`
+* building a release in Docker (used by the CI & release management)
-├── config                  # config data types (=> package yaml-config)
+* building .deb and .rpm packages out of the release artifacts.
-│   └── samples
-├── daemon                  # the implementation of `zrepl daemon` subcommand
-│   ├── filters
-│   ├── hooks               # snapshot hooks
-│   ├── job                 # job implementations
-│   ├── logging             # logging outlets + formatters
-│   ├── nethelpers
-│   ├── prometheus
-│   ├── pruner              # pruner implementation
-│   ├── snapper             # snapshotter implementation
-├── dist                    # supplemental material for users & package maintainers
-├── docs                    # sphinx-based documentation
-│   ├── **/*.rst            # documentation in reStructuredText
-│   ├── sphinxconf
-│   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
-│   ├── requirements.txt    # pip3 requirements to build documentation
-│   ├── publish.sh          # shell script for automated rendering & deploy to zrepl.github.io repo
-│   └── public_git          # checkout of zrepl.github.io managed by above shell script
-├── endpoint                # implementation of replication endpoints (=> package replication)
-├── logger                  # our own logger package
-├── platformtest            # test suite for our zfs abstractions (error classification, etc)
-├── pruning                 # pruning rules (the logic, not the actual execution)
-│   └── retentiongrid
-├── replication
-│   ├── driver              # the driver of the replication logic (status reporting, error handling)
-│   ├── logic               # planning & executing replication steps via rpc
-|   |   └── pdu             # the generated gRPC & protobuf code used in replication (and endpoints)
-│   └── report              # the JSON-serializable report datastructures exposed to the client
-├── rpc                     # the hybrid gRPC + ./dataconn RPC client: connects to a remote replication.Endpoint
-│   ├── dataconn            # Bulk data-transfer RPC protocol
-│   ├── grpcclientidentity  # adaptor to inject package transport's 'client identity' concept into gRPC contexts
-│   ├── netadaptor          # adaptor to convert a package transport's Connecter and Listener into net.* primitives
-│   ├── transportmux        # TCP connecter and listener used to split control & data traffic
-│   └── versionhandshake    # replication protocol version handshake perfomed on newly established connections
-├── tlsconf                 # abstraction for Go TLS server + client config
-├── transport               # transport implementations
-│   ├── fromconfig
-│   ├── local
-│   ├── ssh
-│   ├── tcp
-│   └── tls
-├── util
-├── version                 # abstraction for versions (filled during build by Makefile)
-└── zfs                     # zfs(8) wrappers
-```
 
-### Coding Workflow
+**Build tooling & dependencies** are documented as code in `lazy.sh`.
+Go dependencies are then fetched by the go command and pip dependencies are pinned through a `requirements.txt`.
+
+**We use CircleCI for continuous integration**.
+There are two workflows:
+
+* `ci` runs for every commit / branch / tag pushed to GitHub.
+  It is supposed to run very fast (<5min and provides quick feedback to developers).
+  It runs formatting checks, lints and tests on the most important OSes / architectures.
+  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
+
+* `release` runs
+  * on manual triggers through the CircleCI API (in order to produce a release)
+  * periodically on `master`
+  Artifacts are published to minio.cschwarz.com (see GitHub Commit Status).
+
+**Releases** are issued via Git tags + GitHub Releases feature.
+The procedure to issue a release is as follows:
+* Issue the source release:
+  * Git tag the release on the `master` branch.
+  * Push the tag.
+  * Run `./docs/publish.sh` to re-build & push zrepl.github.io.
+* Issue the official binary release:
+  * Run the `release` pipeline (triggered via CircleCI API)
+  * Download the artifacts to the release manager's machine.
+  * Create a GitHub release, edit the changelog, upload all the release artifacts, including .rpm and .deb files.
+  * Issue the GitHub release.
+  * Add the .rpm and .deb files to the official zrepl repos, publish those.
+
+**Official binary releases are not re-built when Go receives an update. If the Go update is critical to zrepl (e.g. a Go security update that affects zrepl), we'd issue a new source release**.
+The rationale for this is that whereas distros provide a mechanism for this (`$zrepl_source_release-$distro_package_revision`), GitHub Releases doesn't which means we'd need to update the existing GitHub release's assets, which nobody would notice (no RSS feed updates, etc.).
+Downstream packagers can read the changelog to determine whether they want to push that minor release into their distro or simply skip it.
+
+### Additional Notes to Distro Package Maintainers
+
+* Use `sudo make test-platform-bin && sudo make test-platform` **on a test system** to validate that zrepl's abstractions on top of ZFS work with the system ZFS.
+* Ship a default config that adheres to your distro's `hier` and logging system.
+* Ship a service manager file and _please_ try to upstream it to this repository.
+  * `dist/systemd` contains a Systemd unit template.
+* Ship other material provided in `./dist`, e.g. in `/usr/share/zrepl/`.
+* Have a look at the `Makefile`'s `ZREPL_VERSION` variable and how it passed to Go's `ldFlags`.
+  This is how `zrepl version` knows what version number to show.
+  Your build system should set the `ldFlags` flags appropriately and add a prefix or suffix that indicates that the given zrepl binary is a distro build, not an official one.
+* Make sure you are informed about new zrepl versions, e.g. by subscribing to GitHub's release RSS feed.
+
+
+## Contributing Code
 
 * Open an issue when starting to hack on a new feature
 * Commits should reference the issue they are related to
@@ -123,9 +116,6 @@ An HTML report can be generated using `make cover-html`.
 
 Backward-incompatible changes must be documented in the git commit message and are listed in `docs/changelog.rst`.
 
-* Config-breaking changes must contain a line `BREAK CONFIG` in the commit message
-* Other breaking changes must contain a line `BREAK` in the commit message
-
 ### Glossary & Naming Inconsistencies
 
 In ZFS, *dataset* refers to the objects *filesystem*, *ZVOL* and *snapshot*. <br />
@@ -142,16 +132,3 @@ variables and types are often named *dataset* when they in fact refer to a *file
 There will not be a big refactoring (an attempt was made, but it's destroying too much history without much gain).
 
 However, new contributions & patches should fix naming without further notice in the commit message.
-
-### RPC debugging
-
-Optionally, there are various RPC-related environment variables, that if set to something != `""` will produce additional debug output on stderr:
-
-https://github.com/zrepl/zrepl/blob/master/rpc/rpc_debug.go#L11
-
-https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/dataconn_debug.go#L11
-
-https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/stream/stream_debug.go#L11
-
-https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go#L11
-

build.Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:latest
+FROM !SUBSTITUTED_BY_MAKEFILE
 
 RUN apt-get update && apt-get install -y \
     python3-pip \

cli/cli.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"

client/status.go

@@ -480,6 +480,11 @@ func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgress
 			t.newline()
 		}
 
+		if len(latest.Filesystems) == 0 {
+			t.write("NOTE: no filesystems were considered for replication!")
+			t.newline()
+		}
+
 		var maxFSLen int
 		for _, fs := range latest.Filesystems {
 			if len(fs.Info.Name) > maxFSLen {

config/config.go

@@ -324,6 +324,7 @@ type PruneKeepNotReplicated struct {
 type PruneKeepLastN struct {
 	Type  string `yaml:"type"`
 	Count int    `yaml:"count"`
+	Regex string `yaml:"regex,optional"`
 }
 
 type PruneKeepRegex struct { // FIXME rename to KeepRegex

config/retentiongrid.go

@@ -37,7 +37,7 @@ func (t *RetentionIntervalList) UnmarshalYAML(u func(interface{}, bool) error) (
 		return err
 	}
 
-	intervals, err := parseRetentionGridIntervalsString(in)
+	intervals, err := ParseRetentionIntervalSpec(in)
 	if err != nil {
 		return err
 	}
@@ -102,7 +102,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
 
 }
 
-func parseRetentionGridIntervalsString(s string) (intervals []RetentionInterval, err error) {
+func ParseRetentionIntervalSpec(s string) (intervals []RetentionInterval, err error) {
 
 	ges := strings.Split(s, "|")
 	intervals = make([]RetentionInterval, 0, 7*len(ges))

daemon/daemon.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/util/envconst"

daemon/hooks/hooks_test.go

@@ -10,6 +10,7 @@ import (
 	"text/template"
 
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"

daemon/job/active.go

@@ -9,6 +9,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/log"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"

daemon/job/active_test.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/transport"
 )

daemon/job/build_jobs_sendrecvoptions.go

@@ -2,6 +2,7 @@ package job
 
 import (
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/endpoint"

daemon/job/passive.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"

daemon/job/snapjob.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"

daemon/logging/build_logging.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/mattn/go-isatty"
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"github.com/zrepl/zrepl/logger"

daemon/logging/trace/trace.go

@@ -93,11 +93,15 @@ package trace
 import (
 	"context"
 	"fmt"
+	"os"
+	runtimedebug "runtime/debug"
 	"strings"
 	"time"
 
+	"github.com/kr/pretty"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/zrepl/zrepl/util/chainlock"
 )
 
@@ -120,15 +124,19 @@ func RegisterMetrics(r prometheus.Registerer) {
 }
 
 type traceNode struct {
-	id         string
+	// NOTE: members with prefix debug are only valid if the package variable debugEnabled is true.
-	annotation string
+
-	parentTask *traceNode
+	id                 string
+	annotation         string
+	parentTask         *traceNode
+	debugCreationStack string // debug: stack trace of when the traceNode was created
 
 	mtx chainlock.L
 
-	activeChildTasks int32 // only for task nodes, insignificant for span nodes
+	activeChildTasks      int32               // only for task nodes, insignificant for span nodes
-	parentSpan       *traceNode
+	debugActiveChildTasks map[*traceNode]bool // debug: set of active child tasks
-	activeChildSpan  *traceNode // nil if task or span doesn't have an active child span
+	parentSpan            *traceNode
+	activeChildSpan       *traceNode // nil if task or span doesn't have an active child span
 
 	startedAt time.Time
 	endedAt   time.Time
@@ -137,6 +145,14 @@ type traceNode struct {
 func (s *traceNode) StartedAt() time.Time { return s.startedAt }
 func (s *traceNode) EndedAt() time.Time   { return s.endedAt }
 
+// caller must hold mtx
+func (s *traceNode) debugString() string {
+	if !debugEnabled {
+		return "<debugEnabled=false>"
+	}
+	return pretty.Sprint(s)
+}
+
 // Returned from WithTask or WithSpan.
 // Must be called once the task or span ends.
 // See package-level docs for nesting rules.
@@ -196,8 +212,16 @@ func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc)
 		endedAt:   time.Time{},
 	}
 
+	if debugEnabled {
+		this.debugCreationStack = string(runtimedebug.Stack())
+		this.debugActiveChildTasks = map[*traceNode]bool{}
+	}
+
 	if parentTask != nil {
 		this.parentTask.activeChildTasks++
+		if debugEnabled {
+			this.parentTask.debugActiveChildTasks[this] = true
+		}
 		parentTask.mtx.Unlock()
 	}
 
@@ -218,7 +242,11 @@ func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc)
 			defer this.mtx.Lock().Unlock()
 
 			if this.activeChildTasks != 0 {
-				panic(errors.Wrapf(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks", this.activeChildTasks))
+				if debugEnabled {
+					// the debugString can be quite long and panic won't print it completely
+					fmt.Fprintf(os.Stderr, "going to panic due to activeChildTasks:\n%s\n", this.debugString())
+				}
+				panic(errors.WithMessagef(ErrTaskStillHasActiveChildTasks, "end task: %v active child tasks\n", this.activeChildTasks))
 			}
 
 			// support idempotent task ends
@@ -229,8 +257,15 @@ func WithTask(ctx context.Context, taskName string) (context.Context, DoneFunc)
 
 			if this.parentTask != nil {
 				this.parentTask.activeChildTasks--
+				if debugEnabled {
+					delete(this.parentTask.debugActiveChildTasks, this)
+				}
 				if this.parentTask.activeChildTasks < 0 {
-					panic("impl error: parent task with negative activeChildTasks count")
+					if debugEnabled {
+						// the debugString can be quite long and panic won't print it completely
+						fmt.Fprintf(os.Stderr, "going to panic due to activeChildTasks < 0:\n%s\n", this.parentTask.debugString())
+					}
+					panic(fmt.Sprintf("impl error: parent task with negative activeChildTasks count: %v", this.parentTask.activeChildTasks))
 				}
 			}
 			return false
@@ -279,6 +314,10 @@ func WithSpan(ctx context.Context, annotation string) (context.Context, DoneFunc
 		endedAt:   time.Time{},
 	}
 
+	if debugEnabled {
+		this.debugCreationStack = string(runtimedebug.Stack())
+	}
+
 	parentSpan.mtx.HoldWhile(func() {
 		if parentSpan.activeChildSpan != nil {
 			panic(ErrAlreadyActiveChildSpan)

daemon/logging/trace/trace_debug.go

@@ -3,9 +3,11 @@ package trace
 import (
 	"fmt"
 	"os"
+
+	"github.com/zrepl/zrepl/util/envconst"
 )
 
-const debugEnabled = false
+var debugEnabled = envconst.Bool("ZREPL_TRACE_DEBUG_ENABLED", false)
 
 func debug(format string, args ...interface{}) {
 	if !debugEnabled {

daemon/snapper/snapper.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"

debian

@@ -0,0 +1 @@
+packaging/deb/debian

docs/changelog.rst

@@ -27,6 +27,33 @@ We use the following annotations for classifying changes:
 * |bugfix| Change that fixes a bug, no regressions or incompatibilities expected.
 * |docs| Change to the documentation.
 
+0.3.1
+-----
+
+Mostly a bugfix release for :ref:`zrepl 0.3 <release-0.3>`.
+
+* |feature| pruning: add optional ``regex`` field to ``last_n`` rule
+* |docs| pruning: ``grid`` : improve documentation and add an example
+* |bugfix| pruning: ``grid``:  add all snapshots that do not match the regex to the rule's destroy list.
+  This brings the implementation in line with the docs.
+* |bugfix| ``easyrsa`` script in docs
+* |bugfix| platformtest: fix skipping encryption-only tests on systems that don't support encryption
+* |bugfix| replication: report AttemptDone if no filesystems are replicated
+* |feature| status + replication: warning if replication succeeeded without any filesystem being replicated
+* |docs| update multi-job & multi-host setup section
+* RPM Packaging
+* CI infrastructure rework
+* Continuous deployment of that new `stable` branch to zrepl.github.io.
+
+.. NOTE::
+   |  zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
+   |  You can support maintenance and feature development through one of the following services:
+   |  |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
+   |  Note that PayPal processing fees are relatively high for small donations.
+   |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
+
+.. _release-0.3:
+
 0.3
 ---
 
@@ -44,6 +71,13 @@ This is a big one! Headlining features:
 
    We highly recommend studying the updated :ref:`overview section of the configuration chapter <overview-how-replication-works>` to understand how replication works.
 
+.. TIP::
+
+   Go 1.15 changed the default TLS validation policy to **require Subject Alternative Names (SAN) in certificates**.
+   The openssl commands we provided in the quick-start guides up to and including the zrepl 0.3 docs seem not to work properly.
+   If you encounter certificate validation errors regarding SAN and wish to continue to use your old certificates, start the zrepl daemon with env var ``GODEBUG=x509ignoreCN=0``.
+   Alternatively, generate new certificates with SANs (see :ref:`both options int the TLS transport docs <transport-tcp+tlsclientauth-certgen>` ).
+
 Quick-start guides:
 
 * We have added :ref:`another quick-start guide for a typical workstation use case for zrepl <quickstart-backup-to-external-disk>`.
@@ -51,6 +85,7 @@ Quick-start guides:
 
 Additional changelog:
 
+* |break| Go 1.15 TLS changes mentioned above.
 * |break| |break_config| **more restrictive job names than in prior zrepl versions**
   Starting with this version, job names are going to be embedded into ZFS holds and bookmark names (see :ref:`this section for details <zrepl-zfs-abstractions>`).
   Therefore you might need to adjust your job names.
@@ -81,13 +116,6 @@ Additional changelog:
 * **[MAINTAINER NOTICE]** New platform tests in this version, please make sure you run them for your distro!
 * **[MAINTAINER NOTICE]** Please add the shell completions to the zrepl packages.
 
-.. NOTE::
-   |  zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
-   |  You can support maintenance and feature development through one of the following services:
-   |  |Donate via Patreon| |Donate via GitHub Sponsors| |Donate via Liberapay| |Donate via PayPal|
-   |  Note that PayPal processing fees are relatively high for small donations.
-   |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
-
 0.2.1
 -----
 

docs/configuration/overview.rst

@@ -59,8 +59,8 @@ Note that snapshot-creation denoted by "(snap)" is orthogonal to whether a job i
 | Pull mode             | ``pull``     | ``source``                       | * Central backup-server for many nodes                                             |
 |                       |              | (snap)                           | * Remote server to NAS behind NAT                                                  |
 +-----------------------+--------------+----------------------------------+------------------------------------------------------------------------------------+
-| Local replication     | | ``push`` + ``sink`` in one config             | * Backup FreeBSD boot pool                                                         |
+| Local replication     | | ``push`` + ``sink`` in one config             | * Backup to :ref:`locally attached disk <quickstart-backup-to-external-disk>`      |
-|                       | | with :ref:`local transport <transport-local>` |                                                                                    |
+|                       | | with :ref:`local transport <transport-local>` | * Backup FreeBSD boot pool                                                         |
 +-----------------------+--------------+----------------------------------+------------------------------------------------------------------------------------+
 | Snap & prune-only     | ``snap``     | N/A                              | * | Snapshots & pruning but no replication                                         |
 |                       | (snap)       |                                  |   | required                                                                       |
@@ -80,16 +80,19 @@ The active side (:ref:`push <job-push>` and :ref:`pull <job-pull>` job) executes
 .. TIP::
   The progress of the active side can be watched live using the ``zrepl status`` subcommand.
 
+.. _overview-passive-side--client-identity:
+
 How the Passive Side Works
 --------------------------
 
 The passive side (:ref:`sink <job-sink>` and :ref:`source <job-source>`) waits for connections from the corresponding active side,
 using the transport listener type specified in the ``serve`` field of the job configuration.
-Each transport listener provides a client's identity to the passive side job.
+When a client connects, the transport listener performS listener-specific access control (cert validation, IP ACLs, etc)
-It uses the client identity for access control:
+and determines the *client identity*.
+The passive side job then uses this client identity as follows:
 
 * The ``sink`` job maps requests from different client identities to their respective sub-filesystem tree ``root_fs/${client_identity}``.
-* The ``source`` job has a whitelist of client identities that are allowed pull access.
+* The ``source`` might, in the future, embed the client identity in :ref:`zrepl's ZFS abstraction names <zrepl-zfs-abstractions>` in order to support multi-host replication.
 
 .. TIP::
    The implementation of the ``sink`` job requires that the connecting client identities be a valid ZFS filesystem name components.
@@ -164,7 +167,7 @@ With the background knowledge from the previous paragraph, we now summarize the
 
 .. _replication-placeholder-property:
 
-**Placeholder filesystems** on the receiving side are regular ZFS filesystems with the placeholder property ``zrepl:placeholder=on``.
+**Placeholder filesystems** on the receiving side are regular ZFS filesystems with the ZFS property ``zrepl:placeholder=on``.
 Placeholders allow the receiving side to mirror the sender's ZFS dataset hierarchy without replicating every filesystem at every intermediary dataset path component.
 Consider the following example: ``S/H/J`` shall be replicated to ``R/sink/job/S/H/J``, but neither ``S/H`` nor ``S`` shall be replicated.
 ZFS requires the existence of ``R/sink/job/S`` and ``R/sink/job/S/H`` in order to receive into ``R/sink/job/S/H/J``.
@@ -185,7 +188,7 @@ Encoding the job name in the names ensures that multiple sending jobs can replic
 
 .. _tentative-replication-cursor-bookmarks:
 
-**Tentative replication cursor bookmarks** are short-lived boomkarks that protect the atomic moving-forward of the replication cursor and last-received-hold (see :issue:`this issue <340>`).
+**Tentative replication cursor bookmarks** are short-lived bookmarks that protect the atomic moving-forward of the replication cursor and last-received-hold (see :issue:`this issue <340>`).
 They are only necessary if step holds are not used as per the :ref:`replication.protection <replication-option-protection>` setting.
 The tentative replication cursor has the format ``#zrepl_CUSORTENTATIVE_G_<GUID>_J_<JOBNAME>``.
 The ``zrepl zfs-abstraction list`` command provides a listing of all bookmarks and holds managed by zrepl.
@@ -229,39 +232,53 @@ Limitations
 Multiple Jobs & More than 2 Machines
 ------------------------------------
 
+The quick-start guides focus on simple setups with a single sender and a single receiver.
+This section documents considerations for more complex setups.
+
 .. ATTENTION::
 
-  When using multiple jobs across single or multiple machines, the following rules are critical to avoid race conditions & data loss:
+   Before you continue, make sure you have a working understanding of :ref:`how zrepl works <overview-how-replication-works>`
+   and :ref:`what zrepl does to ensure <zrepl-zfs-abstractions>` that replication between sender and receiver is always
+   possible without conflicts.
+   This will help you understand why certain kinds of multi-machine setups do not (yet) work.
 
-  1. The sets of ZFS filesystems matched by the ``filesystems`` filter fields must be disjoint across all jobs configured on a machine.
+.. NOTE::
-  2. The ZFS filesystem subtrees of jobs with ``root_fs`` must be disjoint.
-  3. Across all zrepl instances on all machines in the replication domain, there must be a 1:1 correspondence between active and passive jobs.
 
-  Explanations & exceptions to above rules are detailed below.
+   If you can't find your desired configuration, have questions or would like to see improvements to multi-job setups, please `open an issue on GitHub <https://github.com/zrepl/zrepl/issues/new>`_.
 
-If you would like to see improvements to multi-job setups, please `open an issue on GitHub <https://github.com/zrepl/zrepl/issues/new>`_.
+Multiple Jobs on one Machine
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+As a general rule, multiple jobs configured on one machine **must operate on disjoint sets of filesystems**.
+Otherwise, concurrently running jobs might interfere when operating on the same filesystem.
 
-No Overlapping
+On your setup, ensure that
-^^^^^^^^^^^^^^
 
-Jobs run independently of each other.
+* all ``filesystems`` filter specifications are disjoint
-If two jobs match the same filesystem with their ``filesystems`` filter, they will operate on that filesystem independently and potentially in parallel.
+* no ``root_fs`` is a prefix or equal to another ``root_fs``
-For example, if job A prunes snapshots that job B is planning to replicate, the replication will fail because B assumed the snapshot to still be present.
+* no ``filesystems`` filter matches any ``root_fs``
-However, the next replication attempt will re-examine the situation from scratch and should work.
 
-N push jobs to 1 sink
+**Exceptions to the rule**:
-^^^^^^^^^^^^^^^^^^^^^
 
-The :ref:`sink job <job-sink>` namespaces by client identity.
+* A ``snap`` and ``push`` job on the same machine can match the same ``filesystems``.
-It is thus safe to push to one sink job with different client identities.
+  To avoid interference, only one of the jobs should be pruning snapshots on the sender, the other one should keep all snapshots.
-If the push jobs have the same client identity, the filesystems matched by the push jobs must be disjoint to avoid races.
+  Since the jobs won't coordinate, errors in the log are to be expected, but :ref:`zrepl's ZFS abstractions <zrepl-zfs-abstractions>` ensure that ``push`` and ``sink`` can always replicate incrementally.
+  This scenario is detailed in one of the :ref:`quick-start guides <quickstart-backup-to-external-disk>`.
 
-N pull jobs from 1 source
-^^^^^^^^^^^^^^^^^^^^^^^^^
 
-Multiple pull jobs pulling from the same source have potential for race conditions during pruning:
+More Than 2 Machines
-each pull job prunes the source side independently, causing replication-prune and prune-prune races.
+^^^^^^^^^^^^^^^^^^^^
 
-There is currently no way for a pull job to filter which snapshots it should attempt to replicate.
+This section might be relevant to users who wish to *fan-in* (N machines replicate to 1) or *fan-out* (replicate 1 machine to N machines).
-Thus, it is not possible to just manually assert that the prune rules of all pull jobs are disjoint to avoid replication-prune and prune-prune races.
+
+**Working setups**:
+
+* N ``push`` identities, 1 ``sink`` (as long as the different push jobs have a different :ref:`client identity <overview-passive-side--client-identity>`)
+
+  * ``sink`` constrains each client to a disjoint sub-tree of the sink-side dataset hierarchy ``${root_fs}/${client_identity}``.
+    Therefore, the different clients cannot interfere.
+
+
+**Setups that do not work**:
+
+* N ``pull`` identities, 1 ``source`` job. Tracking :issue:`380`.
 

docs/configuration/prune.rst

@@ -93,22 +93,23 @@ Policy ``grid``
       ...
 
 The retention grid can be thought of as a time-based sieve that thins out snapshots as they get older.
+
 The ``grid`` field specifies a list of adjacent time intervals.
+Each interval is a bucket with a maximum capacity of ``keep`` snapshots.
 The following procedure happens during pruning:
 
 #. The list of snapshots is filtered by the regular expression in ``regex``.
-   Only snapshots names that match the regex are considered for this rule, all others are not affected.
+   Only snapshots names that match the regex are considered for this rule, all others will be pruned unless another rule keeps them.
-#. The filtered list of snapshots is sorted by ``creation``.
+#. The snapshots that match ``regex`` are placed onto a time axis according to their ``creation`` date.
-#. The left edge of the first interval is aligned to the ``creation`` date of the youngest snapshot.
+   The youngest snapshot is on the left, the oldest on the right.
-#. A list of buckets is created, one for each interval.
+#. The first buckets are placed "under" that axis so that the ``grid`` spec's first bucket's left edge aligns with youngest snapshot.
-#. The snapshots are placed into the bucket that matches their ``creation`` date.
+#. All subsequent buckets are placed adjacent to their predecessor bucket.
-#. For each bucket
+#. Now each snapshot on the axis either falls into one bucket or it is older than our rightmost bucket.
+   Buckets are left-inclusive and right-exclusive which means that a snapshot on the edge of bucket will always 'fall into the right one'.
+#. Snapshots older than the rightmost bucket **not kept** by this gridspec.
+#. For each bucket, we only keep the ``keep`` oldest snapshots.
 
-   #. the contained snapshot list is sorted by creation.
+The syntax to describe the bucket list is as follows:
-   #. snapshots from the list, oldest first, are destroyed until the specified ``keep`` count is reached.
-   #. all remaining snapshots on the list are kept.
-
-The syntax to describe the list of time intervals ("buckets") is as follows:
 
 ::
 
@@ -131,30 +132,30 @@ The syntax to describe the list of time intervals ("buckets") is as follows:
            regex: .*
           `
 
-          0h         1h         2h         3h         4h         5h         6h         7h         8h
+          0h        1h        2h        3h        4h        5h        6h        7h        8h        9h
-          |          |          |          |          |          |          |          |          |
+          |         |         |         |         |         |         |         |         |         |
-          |-Bucket 1-|------Bucket 2-------|-------Bucket 3------|------------Bucket 4------------|
+          |-Bucket1-|-----Bucket 2------|------Bucket 3-----|-----------Bucket 4----------|
-          | keep=all |       keep=1        |        keep=1       |             keep=1             |
+          | keep=all|      keep=1       |       keep=1      |            keep=1           |
 
 
 
-          Let us consider the following set of snapshots @a-zA-C , taken at an interval of ~15min:
+          Let us consider the following set of snapshots @a-zA-C:
 
 
-          |  a  b  c  d  e  f  g  h  i  j  k  l  m  n  o  p  q  r  s  t  u  v  w  x  y  z  A  B  C |
+          |  a  b  c  d  e  f  g  h  i  j  k  l  m  n  o  p  q  r  s  t  u  v  w  x  y  z  A  B  C  D        |
-
 
           The `grid` algorithm maps them to their respective buckets:
 
           Bucket 1: a, b, c
           Bucket 2: d,e,f,g,h,i,j
-          Bucket 3: k,l,m,n,o,p,q,r
+          Bucket 3: k,l,m,n,o,p
-          Bucket 4: q,r,s,t,u,v,w,x,y,z,A,B,C
+          Bucket 4: q,r, q,r,s,t,u,v,w,x,y,z
+          None:     A,B,C,D
 
           It then applies the per-bucket pruning logic described above which resulting in the
           following list of remaining snapshots.
 
-          |  a  b  c  d                    k                       s                               |
+          |  a  b  c                    j                 p                             z                    |
 
           Note that it only makes sense to grow (not shorten) the interval duration for buckets
           further in the past since each bucket acts like a low-pass filter for incoming snapshots
@@ -174,9 +175,11 @@ Policy ``last_n``
          keep_receiver:
          - type: last_n
            count: 10
+           regex: ^zrepl_.*$ # optional
      ...
 
-``last_n`` keeps the last ``count`` snapshots (last = youngest = most recent creation date).
+``last_n`` filters the snapshot list by ``regex``, then keeps the last ``count`` snapshots in that list (last = youngest = most recent creation date)
+All snapshots that don't match ``regex`` or exceed ``count`` in the filtered list are destroyed unless matched by other rules.
 
 .. _prune-keep-regex:
 

docs/configuration/snapshotting.rst

@@ -131,7 +131,7 @@ The following environment variables are set:
 * ``ZREPL_SNAPNAME``: the zrepl-generated snapshot name (e.g. ``zrepl_20380119_031407_000``)
 * ``ZREPL_DRYRUN``: set to ``"true"`` if a dry run is in progress so scripts can print, but not run, their commands
 
-An empty template hook can be found in :sampleconf:`hooks/template.sh`.
+An empty template hook can be found in :sampleconf:`/hooks/template.sh`.
 
 .. _job-hook-type-postgres-checkpoint:
 

docs/configuration/transports.rst

@@ -91,7 +91,8 @@ The ``tls`` transport uses TCP + TLS with client authentication using client cer
 The client identity is the common name (CN) presented in the client certificate.
 
 It is recommended to set up a dedicated CA infrastructure for this transport, e.g. using OpenVPN's `EasyRSA <https://github.com/OpenVPN/easy-rsa>`_.
-For a simple 2-machine setup, see the :ref:`instructions below<transport-tcp+tlsclientauth-2machineopenssl>`.
+For a simple 2-machine setup, mutual TLS might also be sufficient.
+We provide :ref:`copy-pastable instructions to generate the certificates below <transport-tcp+tlsclientauth-certgen>`.
 
 The implementation uses `Go's TLS library <https://golang.org/pkg/crypto/tls/>`_.
 Since Go binaries are statically linked, you or your distribution need to recompile zrepl when vulnerabilities in that library are disclosed.
@@ -103,6 +104,16 @@ If intermediate CAs are used, the **full chain** must be present in either in th
 Regardless, the client's certificate must be first in the ``cert`` file, with each following certificate directly certifying the one preceding it (see `TLS's specification <https://tools.ietf.org/html/rfc5246#section-7.4.2>`_).
 This is the common default when using a CA management tool.
 
+.. NOTE::
+
+   As of Go 1.15 (zrepl 0.3.0 and newer), the Go TLS / x509 library **requrires Subject Alternative Names**
+   be present in certificates. You might need to re-generate your certificates using one of the :ref:`two alternatives
+   provided below<transport-tcp+tlsclientauth-certgen>`.
+
+   Note further that zrepl continues to use the CommonName field to assign client identities.
+   Hence, we recommend to keep the Subject Alternative Name and the CommonName in sync.
+
+
 Serve
 ~~~~~
 
@@ -147,45 +158,25 @@ The ``server_cn`` specifies the expected common name (CN) of the server's certif
 It overrides the hostname specified in ``address``.
 The connection fails if either do not match.
 
+.. _transport-tcp+tlsclientauth-certgen:
+
 .. _transport-tcp+tlsclientauth-2machineopenssl:
 
-Self-Signed Certificates
+Mutual-TLS between Two Machines
-~~~~~~~~~~~~~~~~~~~~~~~~
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Tools like `EasyRSA <https://github.com/OpenVPN/easy-rsa>`_ make it easy to manage CA infrastructure for multiple clients, e.g. a central zrepl backup server (in sink mode).
 However, for a two-machine setup, self-signed certificates distributed using an out-of-band mechanism will also work just fine:
 
 Suppose you have a push-mode setup, with `backups.example.com` running the :ref:`sink job <job-sink>`, and `prod.example.com` running the :ref:`push job <job-push>`.
 Run the following OpenSSL commands on each host, substituting HOSTNAME in both filenames and the interactive input prompt by OpenSSL:
 
 .. code-block:: bash
-   :emphasize-lines: 1-5,24
 
-   openssl req -x509 -sha256 -nodes \
+   (name=HOSTNAME; openssl req -x509 -sha256 -nodes \
-      -newkey rsa:4096 \
+    -newkey rsa:4096 \
-      -days 365 \
+    -days 365 \
-      -keyout HOSTNAME.key \
+    -keyout $name.key \
-      -out HOSTNAME.crt
+    -out $name.crt -addext "subjectAltName = DNS:$name" -subj "/CN=$name")
-
-   #Generating a 4096 bit RSA private key
-   #................++++
-   #.++++
-   #writing new private key to 'backups.key'
-   #-----
-   #You are about to be asked to enter information that will be incorporated
-   #into your certificate request.
-   #What you are about to enter is what is called a Distinguished Name or a DN.
-   #There are quite a few fields but you can leave some blank
-   #For some fields there will be a default value,
-   #If you enter '.', the field will be left blank.
-   #-----
-   #Country Name (2 letter code) [XX]:
-   #State or Province Name (full name) []:
-   #Locality Name (eg, city) [Default City]:
-   #Organization Name (eg, company) [Default Company Ltd]:
-   #Organizational Unit Name (eg, section) []:
-   #Common Name (eg, your name or your server's hostname) []:HOSTNAME
-   #Email Address []:
 
 Now copy each machine's ``HOSTNAME.crt`` to the other machine's ``/etc/zrepl/HOSTNAME.crt``, for example using `scp`.
 The serve & connect configuration will thus look like the following:
@@ -216,6 +207,36 @@ The serve & connect configuration will thus look like the following:
      ...
 
 
+Certificate Authority using EasyRSA
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+For more than two machines, it might make sense to set up a CA infrastructure.
+Tools like `EasyRSA <https://github.com/OpenVPN/easy-rsa>`_ make this very easy:
+
+::
+
+     #!/usr/bin/env bash
+     set -euo pipefail
+
+     HOSTS=(backupserver prod1 prod2 prod3)
+
+     curl -L https://github.com/OpenVPN/easy-rsa/releases/download/v3.0.7/EasyRSA-3.0.7.tgz > EasyRSA-3.0.7.tgz
+     echo "157d2e8c115c3ad070c1b2641a4c9191e06a32a8e50971847a718251eeb510a8  EasyRSA-3.0.7.tgz" | sha256sum -c
+     rm -rf EasyRSA-3.0.7
+     tar -xf EasyRSA-3.0.7.tgz
+     cd EasyRSA-3.0.7
+     ./easyrsa
+     ./easyrsa init-pki
+     ./easyrsa build-ca nopass
+
+     for host in "${HOSTS[@]}"; do
+         ./easyrsa build-serverClient-full $host nopass
+         echo cert for host $host available at pki/issued/$host.crt
+         echo key for host $host available at pki/private/$host.key
+     done
+     echo ca cert available at pki/ca.crt
+
+
 .. _transport-ssh+stdinserver:
 
 ``ssh+stdinserver`` Transport

docs/gen-sphinx-versioning-flags.py

@@ -58,13 +58,11 @@ for latest_patch in latest_by_major_minor:
     cmdline.append("--whitelist-tags")
     cmdline.append(f"^{re.escape(latest_patch.orig)}$")
 
-# we want to render the latest non-rc version as the default page
+# we want flexibility to update docs for the latest stable release
-# (latest_by_major_minor is already sorted)
+# => we have a branch for that, called `stable` which we move manually
-default_version = latest_by_major_minor[-1]
+# TODO: in the future, have f"stable-{latest_by_major_minor[-1]}"
-for tag in reversed(latest_by_major_minor):
+default_version = "stable"
-    if tag.rc == 0:
+cmdline.extend(["--whitelist-branches", default_version])
-        default_version = tag
-        break
 
 cmdline.extend(["--root-ref", f"{default_version}"])
 cmdline.extend(["--banner-main-ref", f"{default_version}"])
@@ -73,4 +71,4 @@ cmdline.extend(["--sort", "semver"])
 
 cmdline.extend(["--whitelist-branches", "master"])
 
-print(" ".join(cmdline))
+print(" ".join(cmdline))

docs/global.rst.inc

@@ -13,7 +13,7 @@
    :target: https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96
 .. |Donate via Liberapay| image:: https://img.shields.io/liberapay/patrons/zrepl.svg?logo=liberapay
    :target: https://liberapay.com/zrepl/donate
-.. |Donate via Patreon| image:: https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow
+.. |Donate via Patreon| image:: https://img.shields.io/badge/dynamic/json?color=yellow&label=Patreon&query=data.attributes.patron_count&suffix=%20patrons&url=https%3A%2F%2Fwww.patreon.com%2Fapi%2Fcampaigns%2F3095079
    :target: https://www.patreon.com/zrepl
 .. |Donate via GitHub Sponsors| image:: https://img.shields.io/static/v1?label=Sponsor&message=%E2%9D%A4&logo=GitHub&style=flat&color=yellow
    :target: https://github.com/sponsors/problame

docs/installation.rst

@@ -14,6 +14,7 @@ Installation
     installation/user-privileges
     installation/packages
     installation/apt-repos
+    installation/rpm-repos
     installation/compile-from-source
     installation/freebsd-jail-with-iocage
     installation/what-next

docs/installation/apt-repos.rst

@@ -7,7 +7,7 @@ Debian / Ubuntu APT repositories
 We maintain APT repositories for Debian, Ubuntu and derivatives.
 The fingerprint of the signing key is ``E101 418F D3D6 FBCB 9D65  A62D 7086 99FC 5F2E BF16``.
 It is available at `<https://zrepl.cschwarz.com/apt/apt-key.asc>`_ .
-Please open an issue `in the packaging repository <https://github.com/zrepl/debian-binary-packaging>`_ if you encounter any issues with the repository.
+Please open an issue in on GitHub if you encounter any issues with the repository.
 
 The following snippet configure the repository for your Debian or Ubuntu release:
 
@@ -24,6 +24,6 @@ The following snippet configure the repository for your Debian or Ubuntu release
 
 .. NOTE::
 
-   Until zrepl reaches 1.0, all APT repositories will be updated to the latest zrepl release immediately.
+   Until zrepl reaches 1.0, the repositories will be updated to the latest zrepl release immediately.
    This includes breaking changes between zrepl versions.
    Use ``apt-mark hold zrepl`` to prevent upgrades of zrepl.

docs/installation/compile-from-source.rst

@@ -9,37 +9,35 @@ Producing a release requires **Go 1.11** or newer and **Python 3** + **pip3** +
 A tutorial to install Go is available over at `golang.org <https://golang.org/doc/install>`_.
 Python and pip3 should probably be installed via your distro's package manager.
 
+::
+   cd to/your/zrepl/checkout
+   python3 -m venv3
+   source venv3/bin/activate
+   ./lazy.sh devsetup
+   make release
+   # build artifacts are available in ./artifacts/release
+
+The Python venv is used for the documentation build dependencies.
+If you just want to build the zrepl binary, leave it out and use `./lazy.sh godep` instead.
+
 Alternatively, you can use the Docker build process:
 it is used to produce the official zrepl `binary releases`_
 and serves as a reference for build dependencies and procedure:
 
 ::
 
-    git clone https://github.com/zrepl/zrepl.git && \
+    cd to/your/zrepl/checkout
-    cd zrepl && \
+    # make sure your user has access to the docker socket
-    sudo docker build -t zrepl_build -f build.Dockerfile . && \
+    make release-docker
-    sudo docker run -it --rm \
+    # if you want .deb or .rpm packages, invoke the follwoing
-        -v "${PWD}:/src" \
+    # targets _after_ you invoked release-docker
-        --user "$(id -u):$(id -g)" \
+    make deb-docker
-        zrepl_build make release
+    make rpm-docker
+    # build artifacts are available in ./artifacts/release
+    # packages are available in ./artifacts
 
-Alternatively, you can install build dependencies on your local system and then build in your ``$GOPATH``:
-
-::
-
-    mkdir -p "${GOPATH}/src/github.com/zrepl/zrepl"
-    git clone https://github.com/zrepl/zrepl.git "${GOPATH}/src/github.com/zrepl/zrepl"
-    cd "${GOPATH}/src/github.com/zrepl/zrepl"
-    python3 -m venv3
-    source venv3/bin/activate
-    ./lazy.sh devsetup
-    make release
-
-The Python venv is used for the documentation build dependencies.
-If you just want to build the zrepl binary, leave it out and use `./lazy.sh godep` instead.
-Either way, all build results are located in the ``artifacts/`` directory.
 
 .. NOTE::
 
-    It is your job to install the appropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
+    It is your job to install the built binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
     Otherwise, the examples in the :ref:`quick-start guides <quickstart-toc>` may need to be adjusted.

docs/installation/packages.rst

@@ -30,15 +30,12 @@ The following list may be incomplete, feel free to submit a PR with an update:
     * - Arch Linux
       - ``yay install zrepl``
       - Available on `AUR <https://aur.archlinux.org/packages/zrepl>`_
-    * - Fedora
+    * - Fedora, CentOS, RHEL, OpenSUSE
       - ``dnf install zrepl``
-      - Available on `COPR <https://copr.fedorainfracloud.org/coprs/poettlerric/zrepl/>`_
+      - :ref:`RPM repository config <installation-rpm-repos>`
-    * - CentOS/RHEL
-      - ``yum install zrepl``
-      - Available on `COPR <https://copr.fedorainfracloud.org/coprs/poettlerric/zrepl/>`_
     * - Debian + Ubuntu
       - ``apt install zrepl``
-      - APT repository config :ref:`see below <installation-apt-repos>`
+      - :ref:`APT repository config <installation-apt-repos>`
     * - OmniOS
       - ``pkg install zrepl``
       - Available since `r151030 <https://pkg.omniosce.org/r151030/extra/en/search.shtml?token=zrepl&action=Search>`_

docs/installation/rpm-repos.rst

@@ -0,0 +1,30 @@
+.. _installation-rpm-repos:
+
+RPM repositories
+~~~~~~~~~~~~~~~~
+
+We provide a single RPM repository for all RPM-based Linux distros.
+The zrepl binary in the repo is the same as the one published to GitHub.
+Since Go binaries are statically linked, the RPM should work about everywhere.
+
+The fingerprint of the signing key is ``F6F6 E8EA 6F2F 1462 2878 B5DE 50E3 4417 826E 2CE6``.
+It is available at `<https://zrepl.cschwarz.com/rpm/rpm-key.asc>`_ .
+Please open an issue on GitHub if you encounter any issues with the repository.
+
+Copy-paste the following snippet into your shell to set up the zrepl repository.
+Then ``dnf install zrepl`` and make sure to confirm that the signing key matches the one shown above.
+
+::
+
+    cat > /etc/yum.repos.d/zrepl.repo <<EOF
+    [zrepl]
+    name = zrepl
+    baseurl = https://zrepl.cschwarz.com/rpm/repo
+    gpgkey = https://zrepl.cschwarz.com/rpm/rpm-key.asc
+    EOF
+
+.. NOTE::
+
+   Until zrepl reaches 1.0, the repository will be updated to the latest zrepl release immediately.
+   This includes breaking changes between zrepl versions.
+   If that bothers you, use the `dnf versionlock plugin <https://dnf-plugins-core.readthedocs.io/en/latest/versionlock.html>`_ to pin the version of zrepl on your system.

docs/publish.sh

@@ -1,14 +1,30 @@
 #!/bin/bash
 set -eo pipefail
 
+
+NON_INTERACTIVE=false
+DO_CLONE=false
+while getopts "ca" arg; do
+    case "$arg" in
+        "a")
+            NON_INTERACTIVE=true
+            ;;
+        "c")
+            DO_CLONE=true
+            ;;
+        *)
+            echo invalid option
+            exit 1
+            ;;
+    esac
+done
+
 GHPAGESREPO="git@github.com:zrepl/zrepl.github.io.git"
 SCRIPTDIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
 PUBLICDIR="${SCRIPTDIR}/public_git"
 
 checkout_repo_msg() {
     echo "clone ${GHPAGESREPO} to ${PUBLICDIR}:"
-    echo "	git clone ${GHPAGESREPO} ${PUBLICDIR}"
-    git clone "${GHPAGESREPO}" "${PUBLICDIR}"
 }
 
 exit_msg() {
@@ -25,11 +41,19 @@ cd "$SCRIPTDIR"
 
 if [ ! -d "$PUBLICDIR" ]; then
     checkout_repo_msg
-    exit 1
+    if $DO_CLONE; then
+        git clone "${GHPAGESREPO}" "${PUBLICDIR}"
+    else
+        exit 1
+    fi
 fi
 
-echo -n "PRESS ENTER to confirm you commited and pushed docs changes and tags to the zrepl repo"
+if $NON_INTERACTIVE; then
-read
+    echo "non-interactive mode"
+else
+    echo -n "PRESS ENTER to confirm you commited and pushed docs changes and tags to the zrepl repo"
+    read -r
+fi
 
 pushd "$PUBLICDIR" 
 
@@ -52,9 +76,10 @@ popd
 
 echo "building site"
 
+flags="$(python3 gen-sphinx-versioning-flags.py)"
 set -e
 sphinx-versioning build \
-   $(python3 gen-sphinx-versioning-flags.py) \
+   $flags \
    docs ./public_git \
    -- -c sphinxconf # older conf.py throw errors because they used
                     # version = subprocess.show_output(["git", "describe"])
@@ -66,7 +91,7 @@ git status --porcelain
 if [[ "$(git status --porcelain)" != "" ]]; then
     CURRENT_COMMIT="${CURRENT_COMMIT}(dirty)" 
 fi
-COMMIT_MSG="sphinx-versioning render from publish.sh - `date -u` - ${CURRENT_COMMIT}"
+COMMIT_MSG="sphinx-versioning render from publish.sh - $(date -u) - ${CURRENT_COMMIT}"
 
 pushd "$PUBLICDIR"
 

docs/quickstart/continuous_server_backup.rst

@@ -51,21 +51,18 @@ To get things going quickly, we skip setting up a CA and generate two self-signe
 For convenience, we generate the key pairs on our local machine and distribute them using ssh:
 
 .. code-block:: bash
-   :emphasize-lines: 6,13
 
-   openssl req -x509 -sha256 -nodes \
+   (name=backups; openssl req -x509 -sha256 -nodes \
-      -newkey rsa:4096 \
+    -newkey rsa:4096 \
-      -days 365 \
+    -days 365 \
-      -keyout backups.key \
+    -keyout $name.key \
-      -out backups.crt
+    -out $name.crt -addext "subjectAltName = DNS:$name" -subj "/CN=$name")
-   # ... and use "backups" as Common Name (CN)
 
-   openssl req -x509 -sha256 -nodes \
+   (name=prod; openssl req -x509 -sha256 -nodes \
-      -newkey rsa:4096 \
+    -newkey rsa:4096 \
-      -days 365 \
+    -days 365 \
-      -keyout prod.key \
+    -keyout $name.key \
-      -out prod.crt
+    -out $name.crt -addext "subjectAltName = DNS:$name" -subj "/CN=$name")
-   # ... and use "prod" as Common Name (CN)
 
    ssh root@backups "mkdir /etc/zrepl"
    scp  backups.key backups.crt prod.crt root@backups:/etc/zrepl
@@ -94,4 +91,4 @@ We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`
 Go Back To Quickstart Guide
 ---------------------------
 
-:ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.
+:ref:`Click here <quickstart-apply-config>` to go back to the quickstart guide.

docs/supporters.rst

@@ -32,6 +32,8 @@ We would like to thank the following people and organizations for supporting zre
 
    <div class="fa fa-code" style="width: 1em;"></div>
 
+* |supporter-gold| Cyberiada GmbH
+* |supporter-std| `Gordon Schulz <https://github.com/azmodude>`_
 * |supporter-std| `@jwittlincohen <https://github.com/jwittlincohen>`_
 * |supporter-std| `Michael D. Schmitt <https://waterbendingscroll.dancingdragons.org>`_
 * |supporter-std| `Hans Schulz <https://github.com/schulzh>`_
@@ -46,7 +48,7 @@ We would like to thank the following people and organizations for supporting zre
 * |supporter-gold| `MNX.io <https://mnx.io>`_
 * |supporter-std| `Marshall Clyburn <https://github.com/mdclyburn>`_
 * |supporter-code| `Ross Williams <https://github.com/overhacked>`_
-* |supporter-std| Mike T.
+* |supporter-gold| Mike T.
 * |supporter-code| `Justin Scholz <https://github.com/JMoVS>`_
 * |supporter-code| `InsanePrawn <https://github.com/InsanePrawn>`_
 * |supporter-code| `Ben Woods <https://www.freshports.org/sysutils/zrepl/>`_

endpoint/endpoint.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/kr/pretty"
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/replication/logic/pdu"

endpoint/endpoint_abstractions_cache.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 
 	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"github.com/zrepl/zrepl/util/chainlock"
 )

endpoint/endpoint_guarantees.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"github.com/zrepl/zrepl/zfs"
 )

endpoint/endpoint_zfs_abstraction_last_received_hold.go

@@ -6,6 +6,7 @@ import (
 	"regexp"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/zfs"
 )
 

endpoint/endpoint_zfs_abstraction_replication_cursor.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/zfs"
 )
 

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4
 	github.com/golang/protobuf v1.3.2
 	github.com/google/uuid v1.1.1
+	github.com/hashicorp/yamux v0.0.0-20200609203250-aecfd211c9ce
 	github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8
 	github.com/kr/pretty v0.1.0
 	github.com/lib/pq v1.2.0
@@ -33,6 +34,7 @@ require (
 	github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d
 	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // go1.12 thinks it needs this
 	github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd
+	golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2
 	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
 	golang.org/x/sys v0.0.0-20191026070338-33540a1f6037
@@ -40,3 +42,5 @@ require (
 	gonum.org/v1/gonum v0.7.0 // indirect
 	google.golang.org/grpc v1.17.0
 )
+
+replace github.com/problame/go-netssh => /home/cs/zrepl/go-netssh

go.sum

@@ -120,6 +120,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/yamux v0.0.0-20200609203250-aecfd211c9ce h1:7UnVY3T/ZnHUrfviiAgIUjg2PXxsQfs5bphsG8F7Keo=
+github.com/hashicorp/yamux v0.0.0-20200609203250-aecfd211c9ce/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
@@ -300,6 +302,7 @@ github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd/go.mod h1:JmNwis
 github.com/zrepl/zrepl v0.2.0/go.mod h1:M3Zv2IGSO8iYpUjsZD6ayZ2LHy7zyMfzet9XatKOrZ8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=

lazy.sh

@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 set -e
 
 
@@ -14,23 +14,31 @@ step() {
     echo "${bold}$1${normal}"
 }
 
-if ! type go >/dev/null; then
-    step "go binary not installed or not in \$PATH" 1>&2
-    exit 1
-fi
-
-if [ -z "$GOPATH" ]; then
-    step "Make sure you have your GOPATH configured correctly" 1>&2
-    exit 1
-fi
-
-CHECKOUTPATH="${GOPATH}/src/github.com/zrepl/zrepl"
-
 godep() {
     step "install build dependencies (versions pinned in build/go.mod and build/tools.go)"
+
+    if ! type go >/dev/null; then
+        step "go binary not installed or not in \$PATH" 1>&2
+        exit 1
+    fi
+
+    if [ -z "$GOPATH" ]; then
+        step "Your GOPATH is not configured correctly" 1>&2
+        exit 1
+    fi
+
+    if ! (echo "$PATH" | grep "${GOPATH}/bin" > /dev/null); then
+        step "GOPATH/bin is not in your PATH (it should be towards the start of it)"
+        exit 1
+    fi
+
     pushd "$(dirname "${BASH_SOURCE[0]}")"/build
     set -x
     export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
+    source <(go env)
+    export GOOS="$GOHOSTOS"
+    export GOARCH="$GOHOSTARCH"
+    # TODO GOARM=$GOHOSTARM?
     go build -v -mod=readonly -o "$GOPATH/bin/stringer"      golang.org/x/tools/cmd/stringer
     go build -v -mod=readonly -o "$GOPATH/bin/protoc-gen-go" github.com/golang/protobuf/protoc-gen-go
     go build -v -mod=readonly -o "$GOPATH/bin/enumer"        github.com/alvaroloes/enumer
@@ -51,8 +59,9 @@ docdep() {
         exit 1
     fi
     step "Installing doc build dependencies"
-    local reqpath="${CHECKOUTPATH}/docs/requirements.txt"
+    # shellcheck disable=SC2155
-    if [ ! -z "$ZREPL_LAZY_DOCS_REQPATH" ]; then
+    local reqpath="$(dirname "${BASH_SOURCE[0]}")/docs/requirements.txt"
+    if [ -n "$ZREPL_LAZY_DOCS_REQPATH" ]; then
         reqpath="$ZREPL_LAZY_DOCS_REQPATH"
     fi
     pip3 install -r "$reqpath"
@@ -63,9 +72,15 @@ release() {
     make release
 }
 
+# shellcheck disable=SC2198
+if [ -z "$@" ]; then
+    step "No command specified, exiting"
+    exit 1
+fi
+
 for cmd in "$@"; do
     case "$cmd" in
-        godep|docdep|release_bins|docs)
+        godep|docdep|release|docs)
             eval $cmd
             continue
             ;;

packaging/deb/Dockerfile

@@ -0,0 +1,11 @@
+FROM debian:latest
+
+RUN apt-get update && apt-get install -y \
+	build-essential \
+    devscripts \
+    dh-exec
+
+RUN mkdir -p /build/src && chmod -R 0777 /build
+
+WORKDIR /build/src
+

packaging/deb/debian/changelog.template

@@ -0,0 +1,5 @@
+zrepl (VERSION) unstable;
+
+  Check https://zrepl.github.io/changelog.html.
+
+ -- zrepl build process <me@cschwarz.com>  DATE_DASH_R_OUTPUT

packaging/deb/debian/compat

@@ -0,0 +1 @@
+11

packaging/deb/debian/control

@@ -0,0 +1,8 @@
+Source: zrepl
+Maintainer: Christian Schwarz <me@cschwarz.com>
+Build-Depends: dh-exec
+
+Package: zrepl
+Architecture: arm64 amd64 armhf i386
+Description: ZFS replication
+  zrepl is a one-stop solution for ZFS filesystem replication

packaging/deb/debian/rules

@@ -0,0 +1,50 @@
+#!/usr/bin/make -f
+
+%:
+	dh $@
+
+override_dh_strip:
+
+override_dh_auto_build:
+
+override_dh_auto_clean:
+
+	rm -rf debian/renamedir
+
+override_dh_auto_install:
+
+	# install the zrepl.service for
+	# dh_install_systemd and dh_systemd_{start,enable}
+	install dist/systemd/zrepl.service debian/zrepl.service
+	sed -i 's#ExecStart=/usr/local/bin/zrepl #ExecStart=/usr/bin/zrepl #' debian/zrepl.service
+	sed -i 's#ExecStartPre=/usr/local/bin/zrepl #ExecStartPre=/usr/bin/zrepl #' debian/zrepl.service
+
+	mkdir -p debian/renamedir
+
+	# install binary
+	stat artifacts/$(ZREPL_DPKG_ZREPL_BINARY_FILENAME)
+	cp --preserve=all artifacts/$(ZREPL_DPKG_ZREPL_BINARY_FILENAME) debian/renamedir/zrepl
+	dh_install debian/renamedir/zrepl usr/bin
+
+	# install bash completion
+	dh_install artifacts/bash_completion etc/bash_completion.d/zrepl
+	dh_install artifacts/_zrepl.zsh_completion usr/share/zsh/vendor-completions
+
+	# install docs
+	dh_install artifacts/docs/html usr/share/doc/zrepl/docs/
+
+	# install examples
+	dh_install config/samples/* usr/share/doc/zrepl/examples
+
+	# install default config
+	mkdir -p debian/tmp/etc/zrepl
+	chmod 0700 debian/tmp/etc/zrepl
+	sed 's#USR_SHARE_ZREPL#/usr/share/doc/zrepl#' packaging/systemd-default-zrepl.yml > debian/renamedir/zrepl.yml
+	dh_install debian/renamedir/zrepl.yml etc/zrepl
+
+	# save git revision of this packaging repo
+	echo $$(git rev-parse HEAD) > debian/packaging-repo-git-revision
+	dh_install debian/packaging-repo-git-revision usr/share/doc/zrepl
+
+override_dh_auto_test:
+	# don't run tests at this point

packaging/rpm/Dockerfile

@@ -0,0 +1,7 @@
+FROM fedora:latest
+
+RUN dnf install -y git make bash rpm-build 'dnf-command(builddep)'
+ADD packaging/rpm/zrepl.spec /tmp/zrepl.spec
+RUN dnf builddep -y /tmp/zrepl.spec
+RUN mkdir -p /build/src && chmod -R 0777 /build
+WORKDIR /build/src

packaging/rpm/zrepl.spec

@@ -0,0 +1,78 @@
+# This Spec file packages pre-built artifacts in the artifacts directory
+# into an RPM. This means that rpmbuild won't actually build any new artifacts
+#
+# Read the Makefile first, then come back here.
+
+# Global meta data
+Version:          SUBSTITUTED_BY_MAKEFILE
+%global common_description %{expand:
+zrepl is a one-stop, integrated solution for ZFS replication.}
+
+# use gzip to be compatible with centos7
+%define _source_payload w9.gzdio
+%define _binary_payload w9.gzdio
+
+# don't strip pre-built binaries
+%define __strip /usr/bin/true
+
+Name:             zrepl
+Release:          1
+Summary:          One-stop, integrated solution for ZFS replication
+License:          MIT
+URL:              https://zrepl.github.io/
+# Source: we use rpmbuild --build-in-tree => no source
+BuildRequires:    systemd
+Requires(post):   systemd
+Requires(preun):  systemd
+Requires(postun): systemd
+
+%description
+%{common_description}
+
+%prep
+# we use rpmbuild --build-in-tree => no prep or setup
+
+%build
+# we don't actually need to build anything here, that has already been done by the Makefile
+
+# Correct the path in the systemd unit file
+sed s:/usr/local/bin/:%{_bindir}/:g dist/systemd/zrepl.service > artifacts/rpmbuild/zrepl.service
+
+# Generate the default configuration file
+sed 's#USR_SHARE_ZREPL#%{_datadir}/doc/zrepl#' packaging/systemd-default-zrepl.yml > artifacts/rpmbuild/zrepl.yml
+
+%install
+install -Dm 0755 artifacts/%{_zrepl_binary_filename}    %{buildroot}%{_bindir}/zrepl
+install -Dm 0644 artifacts/rpmbuild/zrepl.service       %{buildroot}%{_unitdir}/zrepl.service
+install -Dm 0644 artifacts/_zrepl.zsh_completion        %{buildroot}%{_datadir}/zsh/site-functions/_zrepl
+install -Dm 0644 artifacts/bash_completion              %{buildroot}%{_datadir}/bash-completion/completions/zrepl
+install -Dm 0644 artifacts/rpmbuild/zrepl.yml           %{buildroot}%{_sysconfdir}/zrepl/zrepl.yml
+install -d                                              %{buildroot}%{_datadir}/doc/zrepl
+cp -a   artifacts/docs/html                             %{buildroot}%{_datadir}/doc/zrepl/html
+cp -a   config/samples                                  %{buildroot}%{_datadir}/doc/zrepl/examples
+
+%post
+%systemd_post zrepl.service
+
+
+%preun
+%systemd_preun zrepl.service
+
+
+%postun
+%systemd_postun_with_restart zrepl.service
+
+
+%files
+%defattr(-,root,root)
+%license LICENSE
+%{_bindir}/zrepl
+%config %{_unitdir}/zrepl.service
+%dir %{_sysconfdir}/zrepl
+%config %{_sysconfdir}/zrepl/zrepl.yml
+%{_datadir}/zsh/site-functions/_zrepl
+%{_datadir}/bash-completion/completions/zrepl
+%{_datadir}/doc/zrepl
+
+%changelog
+# TODO: auto-fill changelog from git? -> need same solution for debian

packaging/systemd-default-zrepl.yml

@@ -0,0 +1,13 @@
+global:
+  logging:
+    # use syslog instead of stdout because it makes journald happy
+    - type: syslog
+      format: human
+      level: warn
+
+jobs:
+#  - name: foo
+#    type: bar
+
+# see USR_SHARE_ZREPL/examples
+# or https://zrepl.github.io/configuration/overview.html

platformtest/harness/harness.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/config"

platformtest/tests/generated_cases.go

@@ -29,7 +29,8 @@ var Cases = []Case{BatchDestroy,
 	ReplicationStepCompletedLostBehavior__GuaranteeResumability,
 	ResumableRecvAndTokenHandling,
 	ResumeTokenParsing,
-	SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden,
+	SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_false,
+	SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_true,
 	SendArgsValidationResumeTokenDifferentFilesystemForbidden,
 	SendArgsValidationResumeTokenEncryptionMismatchForbidden,
 	UndestroyableSnapshotParsing,

platformtest/tests/holds.go

@@ -4,6 +4,7 @@ import (
 	"path"
 
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/platformtest"
 	"github.com/zrepl/zrepl/zfs"
 )

platformtest/tests/idempotentBookmark.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/stretchr/testify/assert"
+
 	"github.com/zrepl/zrepl/platformtest"
 	"github.com/zrepl/zrepl/zfs"
 )

platformtest/tests/recvForceIntoEncryptedErr.go

@@ -4,11 +4,20 @@ import (
 	"fmt"
 
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/platformtest"
 	"github.com/zrepl/zrepl/zfs"
 )
 
 func ReceiveForceIntoEncryptedErr(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	require.NoError(ctx, err, "encryption feature test failed")
+	if !supported {
+		ctx.SkipNow()
+		return
+	}
+
 	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
 		DESTROYROOT
 		CREATEROOT
@@ -32,6 +41,7 @@ func ReceiveForceIntoEncryptedErr(ctx *platformtest.Context) {
 
 	sendStream, err := zfs.ZFSSend(ctx, sendArgs)
 	require.NoError(ctx, err)
+	defer sendStream.Close()
 
 	recvOpts := zfs.RecvOptions{
 		RollbackAndForceRecv: true,

platformtest/tests/recvRollback.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/platformtest"
 	"github.com/zrepl/zrepl/zfs"
 )

platformtest/tests/replication.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/kr/pretty"
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/platformtest"

platformtest/tests/sendArgsValidation.go

@@ -9,11 +9,22 @@ import (
 	"github.com/zrepl/zrepl/zfs"
 )
 
-func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformtest.Context) {
+func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_true(ctx *platformtest.Context) {
+	sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx, true)
+}
+
+func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden__EncryptionSupported_false(ctx *platformtest.Context) {
+	sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx, false)
+}
+
+func sendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden_impl(ctx *platformtest.Context, testForEncryptionSupported bool) {
 
 	supported, err := zfs.EncryptionCLISupported(ctx)
 	check(err)
-	expectNotSupportedErr := !supported
+	if supported != testForEncryptionSupported {
+		ctx.SkipNow()
+	}
+	noEncryptionCLISupport := !supported
 
 	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
 	DESTROYROOT
@@ -44,9 +55,11 @@ func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformt
 		// fallthrough
 	}
 
-	if expectNotSupportedErr {
+	if noEncryptionCLISupport {
 		require.Error(ctx, err)
-		require.Equal(ctx, zfs.ErrEncryptedSendNotSupported, err)
+		saverr, ok := err.(*zfs.ZFSSendArgsValidationError)
+		require.True(ctx, ok, "%T", err)
+		require.Equal(ctx, zfs.ZFSSendArgsEncryptedSendRequestedButFSUnencrypted, saverr.What)
 		return
 	}
 	require.Error(ctx, err)
@@ -138,13 +151,7 @@ func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.
 }
 
 func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest.Context) {
-
+	supported, err := zfs.ResumeSendSupported(ctx)
-	supported, err := zfs.EncryptionCLISupported(ctx)
-	check(err)
-	if !supported {
-		ctx.SkipNow()
-	}
-	supported, err = zfs.ResumeSendSupported(ctx)
 	check(err)
 	if !supported {
 		ctx.SkipNow()

pruning/keep_grid.go

@@ -3,7 +3,6 @@ package pruning
 import (
 	"fmt"
 	"regexp"
-	"sort"
 	"time"
 
 	"github.com/pkg/errors"
@@ -30,75 +29,88 @@ func NewKeepGrid(in *config.PruneGrid) (p *KeepGrid, err error) {
 		return nil, errors.Wrap(err, "Regex is invalid")
 	}
 
+	return newKeepGrid(re, in.Grid)
+}
+
+func MustNewKeepGrid(regex, gridspec string) *KeepGrid {
+
+	ris, err := config.ParseRetentionIntervalSpec(gridspec)
+	if err != nil {
+		panic(err)
+	}
+
+	re := regexp.MustCompile(regex)
+
+	grid, err := newKeepGrid(re, ris)
+	if err != nil {
+		panic(err)
+	}
+	return grid
+}
+
+func newKeepGrid(re *regexp.Regexp, configIntervals []config.RetentionInterval) (*KeepGrid, error) {
+	if re == nil {
+		panic("re must not be nil")
+	}
+
+	if len(configIntervals) == 0 {
+		return nil, errors.New("retention grid must specify at least one interval")
+	}
+
+	intervals := make([]retentiongrid.Interval, len(configIntervals))
+	for i := range configIntervals {
+		intervals[i] = &configIntervals[i]
+	}
+
 	// Assert intervals are of increasing length (not necessarily required, but indicates config mistake)
 	lastDuration := time.Duration(0)
-	for i := range in.Grid {
+	for i := range intervals {
 
-		if in.Grid[i].Length() < lastDuration {
+		if intervals[i].Length() < lastDuration {
 			// If all intervals before were keep=all, this is ok
 			allPrevKeepCountAll := true
 			for j := i - 1; allPrevKeepCountAll && j >= 0; j-- {
-				allPrevKeepCountAll = in.Grid[j].KeepCount() == config.RetentionGridKeepCountAll
+				allPrevKeepCountAll = intervals[j].KeepCount() == config.RetentionGridKeepCountAll
 			}
 			if allPrevKeepCountAll {
 				goto isMonotonicIncrease
 			}
-			err = errors.New("retention grid interval length must be monotonically increasing")
+			return nil, errors.New("retention grid interval length must be monotonically increasing")
-			return
 		}
 	isMonotonicIncrease:
-		lastDuration = in.Grid[i].Length()
+		lastDuration = intervals[i].Length()
-
-	}
-
-	retentionIntervals := make([]retentiongrid.Interval, len(in.Grid))
-	for i := range in.Grid {
-		retentionIntervals[i] = &in.Grid[i]
 	}
 
 	return &KeepGrid{
-		retentiongrid.NewGrid(retentionIntervals),
+		retentionGrid: retentiongrid.NewGrid(intervals),
-		re,
+		re:            re,
 	}, nil
 }
 
-type retentionGridAdaptor struct {
-	Snapshot
-}
-
-func (a retentionGridAdaptor) LessThan(b retentiongrid.Entry) bool {
-	return a.Date().Before(b.Date())
-}
-
 // Prune filters snapshots with the retention grid.
 func (p *KeepGrid) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
 
-	snaps = filterSnapList(snaps, func(snapshot Snapshot) bool {
+	matching, notMatching := partitionSnapList(snaps, func(snapshot Snapshot) bool {
 		return p.re.MatchString(snapshot.Name())
 	})
-	if len(snaps) == 0 {
-		return nil
-	}
 
-	// Build adaptors for retention grid
+	// snaps that don't match the regex are not kept by this rule
-	adaptors := make([]retentiongrid.Entry, 0)
+	destroyList = append(destroyList, notMatching...)
-	for i := range snaps {
-		adaptors = append(adaptors, retentionGridAdaptor{snaps[i]})
-	}
 
-	// determine 'now' edge
+	if len(matching) == 0 {
-	sort.SliceStable(adaptors, func(i, j int) bool {
+		return destroyList
-		return adaptors[i].LessThan(adaptors[j])
+	}
-	})
-	now := adaptors[len(adaptors)-1].Date()
 
 	// Evaluate retention grid
-	_, removea := p.retentionGrid.FitEntries(now, adaptors)
+	entrySlice := make([]retentiongrid.Entry, 0)
+	for i := range matching {
+		entrySlice = append(entrySlice, matching[i])
+	}
+	_, gridDestroyList := p.retentionGrid.FitEntries(entrySlice)
 
 	// Revert adaptors
-	destroyList = make([]Snapshot, len(removea))
+	for i := range gridDestroyList {
-	for i := range removea {
+		destroyList = append(destroyList, gridDestroyList[i].(Snapshot))
-		destroyList[i] = removea[i].(retentionGridAdaptor).Snapshot
 	}
 	return destroyList
 }

pruning/keep_helpers.go

@@ -10,6 +10,17 @@ func filterSnapList(snaps []Snapshot, predicate func(Snapshot) bool) []Snapshot
 	return r
 }
 
+func partitionSnapList(snaps []Snapshot, predicate func(Snapshot) bool) (sTrue, sFalse []Snapshot) {
+	for i := range snaps {
+		if predicate(snaps[i]) {
+			sTrue = append(sTrue, snaps[i])
+		} else {
+			sFalse = append(sFalse, snaps[i])
+		}
+	}
+	return
+}
+
 func shallowCopySnapList(snaps []Snapshot) []Snapshot {
 	c := make([]Snapshot, len(snaps))
 	copy(c, snaps)

pruning/keep_last_n.go

@@ -1,20 +1,35 @@
 package pruning
 
 import (
+	"regexp"
 	"sort"
+	"strings"
 
 	"github.com/pkg/errors"
 )
 
 type KeepLastN struct {
-	n int
+	n  int
+	re *regexp.Regexp
 }
 
-func NewKeepLastN(n int) (*KeepLastN, error) {
+func MustKeepLastN(n int, regex string) *KeepLastN {
+	k, err := NewKeepLastN(n, regex)
+	if err != nil {
+		panic(err)
+	}
+	return k
+}
+
+func NewKeepLastN(n int, regex string) (*KeepLastN, error) {
 	if n <= 0 {
 		return nil, errors.Errorf("must specify positive number as 'keep last count', got %d", n)
 	}
-	return &KeepLastN{n}, nil
+	re, err := regexp.Compile(regex)
+	if err != nil {
+		return nil, errors.Errorf("invalid regex %q: %s", regex, err)
+	}
+	return &KeepLastN{n, re}, nil
 }
 
 func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
@@ -23,11 +38,25 @@ func (k KeepLastN) KeepRule(snaps []Snapshot) (destroyList []Snapshot) {
 		return []Snapshot{}
 	}
 
-	res := shallowCopySnapList(snaps)
+	matching, notMatching := partitionSnapList(snaps, func(snapshot Snapshot) bool {
-
+		return k.re.MatchString(snapshot.Name())
-	sort.Slice(res, func(i, j int) bool {
-		return res[i].Date().After(res[j].Date())
 	})
+	// snaps that don't match the regex are not kept by this rule
+	destroyList = append(destroyList, notMatching...)
 
-	return res[k.n:]
+	if len(matching) == 0 {
+		return destroyList
+	}
+
+	sort.Slice(matching, func(i, j int) bool {
+		// by date (youngest first)
+		id, jd := matching[i].Date(), matching[j].Date()
+		if !id.Equal(jd) {
+			return id.After(jd)
+		}
+		// then lexicographically descending (e.g. b, a)
+		return strings.Compare(matching[i].Name(), matching[j].Name()) == 1
+	})
+	destroyList = append(destroyList, matching[k.n:]...)
+	return destroyList
 }

pruning/keep_last_n_test.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestKeepLastN(t *testing.T) {
@@ -28,7 +29,7 @@ func TestKeepLastN(t *testing.T) {
 		"keep2": {
 			inputs: inputs["s1"],
 			rules: []KeepRule{
-				KeepLastN{2},
+				MustKeepLastN(2, ""),
 			},
 			expDestroy: map[string]bool{
 				"1": true, "2": true, "3": true,
@@ -37,37 +38,66 @@ func TestKeepLastN(t *testing.T) {
 		"keep1OfTwoWithSameTime": { // Keep one of two with same time
 			inputs: inputs["s1"],
 			rules: []KeepRule{
-				KeepLastN{1},
+				MustKeepLastN(1, ""),
-			},
-			expDestroyAlternatives: []map[string]bool{
-				{"1": true, "2": true, "3": true, "4": true},
-				{"1": true, "2": true, "3": true, "5": true},
 			},
+			expDestroy: map[string]bool{"1": true, "2": true, "3": true, "4": true},
 		},
 		"keepMany": {
 			inputs: inputs["s1"],
 			rules: []KeepRule{
-				KeepLastN{100},
+				MustKeepLastN(100, ""),
 			},
 			expDestroy: map[string]bool{},
 		},
-		"empty": {
+		"empty_input": {
 			inputs: inputs["s2"],
 			rules: []KeepRule{
-				KeepLastN{100},
+				MustKeepLastN(100, ""),
 			},
 			expDestroy: map[string]bool{},
 		},
+		"empty_regex": {
+			inputs: inputs["s1"],
+			rules: []KeepRule{
+				MustKeepLastN(4, ""),
+			},
+			expDestroy: map[string]bool{
+				"1": true,
+			},
+		},
+		"multiple_regexes": {
+			inputs: []Snapshot{
+				stubSnap{"a1", false, o(10)},
+				stubSnap{"b1", false, o(11)},
+				stubSnap{"a2", false, o(20)},
+				stubSnap{"b2", false, o(21)},
+				stubSnap{"a3", false, o(30)},
+				stubSnap{"b3", false, o(31)},
+			},
+			rules: []KeepRule{
+				MustKeepLastN(2, "^a"),
+				MustKeepLastN(2, "^b"),
+			},
+			expDestroy: map[string]bool{
+				"a1": true,
+				"b1": true,
+			},
+		},
 	}
 
 	testTable(tcs, t)
 
 	t.Run("mustBePositive", func(t *testing.T) {
 		var err error
-		_, err = NewKeepLastN(0)
+		_, err = NewKeepLastN(0, "foo")
 		assert.Error(t, err)
-		_, err = NewKeepLastN(-5)
+		_, err = NewKeepLastN(-5, "foo")
 		assert.Error(t, err)
 	})
 
+	t.Run("emptyRegexAllowed", func(t *testing.T) {
+		_, err := NewKeepLastN(23, "")
+		require.NoError(t, err)
+	})
+
 }

pruning/pruning.go

@@ -60,7 +60,7 @@ func RuleFromConfig(in config.PruningEnum) (KeepRule, error) {
 	case *config.PruneKeepNotReplicated:
 		return NewKeepNotReplicated(), nil
 	case *config.PruneKeepLastN:
-		return NewKeepLastN(v.Count)
+		return NewKeepLastN(v.Count, v.Regex)
 	case *config.PruneKeepRegex:
 		return NewKeepRegex(v.Regex, v.Negate)
 	case *config.PruneGrid:

pruning/pruning_test.go

@@ -3,6 +3,9 @@ package pruning
 import (
 	"testing"
 	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 type stubSnap struct {
@@ -18,10 +21,9 @@ func (s stubSnap) Replicated() bool { return s.replicated }
 func (s stubSnap) Date() time.Time { return s.date }
 
 type testCase struct {
-	inputs                 []Snapshot
+	inputs     []Snapshot
-	rules                  []KeepRule
+	rules      []KeepRule
-	expDestroy, effDestroy map[string]bool
+	expDestroy map[string]bool
-	expDestroyAlternatives []map[string]bool
 }
 
 type snapshotList []Snapshot
@@ -44,41 +46,20 @@ func (l snapshotList) NameList() []string {
 }
 
 func testTable(tcs map[string]testCase, t *testing.T) {
-	mapEqual := func(a, b map[string]bool) bool {
-		if len(a) != len(b) {
-			return false
-		}
-		for k, v := range a {
-			if w, ok := b[k]; !ok || v != w {
-				return false
-			}
-		}
-		return true
-	}
-
 	for name := range tcs {
 		t.Run(name, func(t *testing.T) {
 			tc := tcs[name]
-			remove := PruneSnapshots(tc.inputs, tc.rules)
+			destroyList := PruneSnapshots(tc.inputs, tc.rules)
-			tc.effDestroy = make(map[string]bool)
+			destroySet := make(map[string]bool, len(destroyList))
-			for _, s := range remove {
+			for _, s := range destroyList {
-				tc.effDestroy[s.Name()] = true
+				destroySet[s.Name()] = true
 			}
-			if tc.expDestroyAlternatives == nil {
+			t.Logf("destroySet:\n%#v", destroySet)
-				if tc.expDestroy == nil {
+			t.Logf("expected:\n%#v", tc.expDestroy)
-					panic("must specify either expDestroyAlternatives or expDestroy")
+
-				}
+			require.Equal(t, len(tc.expDestroy), len(destroySet))
-				tc.expDestroyAlternatives = []map[string]bool{tc.expDestroy}
+			for name := range destroySet {
-			}
+				assert.True(t, tc.expDestroy[name], "%q", name)
-			var okAlt map[string]bool = nil
-			for _, alt := range tc.expDestroyAlternatives {
-				t.Logf("testing possible result: %v", alt)
-				if mapEqual(alt, tc.effDestroy) {
-					okAlt = alt
-				}
-			}
-			if okAlt == nil {
-				t.Errorf("no alternatives matched result: %v", tc.effDestroy)
 			}
 		})
 	}
@@ -94,6 +75,10 @@ func TestPruneSnapshots(t *testing.T) {
 		},
 	}
 
+	reltime := func(secs int64) time.Time {
+		return time.Unix(secs, 0)
+	}
+
 	tcs := map[string]testCase{
 		"simple": {
 			inputs: inputs["s1"],
@@ -140,6 +125,24 @@ func TestPruneSnapshots(t *testing.T) {
 			},
 			expDestroy: map[string]bool{},
 		},
+		"multiple_grids_with_disjoint_regexes": {
+			inputs: []Snapshot{
+				stubSnap{"p1_a", false, reltime(4)},
+				stubSnap{"p2_a", false, reltime(5)},
+				stubSnap{"p1_b", false, reltime(14)},
+				stubSnap{"p2_b", false, reltime(15)},
+				stubSnap{"p1_c", false, reltime(29)},
+				stubSnap{"p2_c", false, reltime(30)},
+			},
+			rules: []KeepRule{
+				MustNewKeepGrid("^p1_", `1x10s | 1x10s`),
+				MustNewKeepGrid("^p2_", `1x10s | 1x10s`),
+			},
+			expDestroy: map[string]bool{
+				"p1_a": true,
+				"p2_a": true,
+			},
+		},
 	}
 
 	testTable(tcs, t)

pruning/retentiongrid/retentiongrid.go

@@ -16,85 +16,127 @@ type Grid struct {
 	intervals []Interval
 }
 
-//A point inside the grid, i.e. a thing the grid can decide to remove
 type Entry interface {
 	Date() time.Time
-	LessThan(b Entry) bool
-}
-
-func dateInInterval(date, startDateInterval time.Time, i Interval) bool {
-	return date.After(startDateInterval) && date.Before(startDateInterval.Add(i.Length()))
 }
 
 func NewGrid(l []Interval) *Grid {
+	if len(l) == 0 {
+		panic("must specify at least one interval")
+	}
 	// TODO Maybe check for ascending interval lengths here, although the algorithm
 	// 		itself doesn't care about that.
 	return &Grid{l}
 }
 
-// Partition a list of RetentionGridEntries into the Grid,
+func (g Grid) FitEntries(entries []Entry) (keep, remove []Entry) {
-// relative to a given start date `now`.
-//
-// The `keepCount` oldest entries per `retentiongrid.Interval` are kept (`keep`),
-// the others are removed (`remove`).
-//
-// Entries that are younger than `now` are always kept.
-// Those that are older than the earliest beginning of an interval are removed.
-func (g Grid) FitEntries(now time.Time, entries []Entry) (keep, remove []Entry) {
 
-	type bucket struct {
+	if len(entries) == 0 {
-		entries []Entry
+		return
 	}
+
+	// determine 'now' based on youngest snapshot
+	// => sort youngest-to-oldest
+	sort.SliceStable(entries, func(i, j int) bool {
+		return entries[i].Date().After(entries[j].Date())
+	})
+	now := entries[0].Date()
+
+	return g.fitEntriesWithNow(now, entries)
+}
+
+type bucket struct {
+	keepCount     int
+	youngerThan   time.Time
+	olderThanOrEq time.Time
+	entries       []Entry
+}
+
+func makeBucketFromInterval(olderThanOrEq time.Time, i Interval) bucket {
+	var b bucket
+	kc := i.KeepCount()
+	if kc == 0 {
+		panic("keep count 0 is not allowed")
+	}
+	if (kc < 0) && kc != RetentionGridKeepCountAll {
+		panic("negative keep counts are not allowed")
+	}
+	b.keepCount = kc
+	b.olderThanOrEq = olderThanOrEq
+	b.youngerThan = b.olderThanOrEq.Add(-i.Length())
+	return b
+}
+
+func (b *bucket) Contains(e Entry) bool {
+	d := e.Date()
+	olderThan := d.Before(b.olderThanOrEq)
+	eq := d.Equal(b.olderThanOrEq)
+	youngerThan := d.After(b.youngerThan)
+	return (olderThan || eq) && youngerThan
+}
+
+func (b *bucket) AddIfContains(e Entry) (added bool) {
+	added = b.Contains(e)
+	if added {
+		b.entries = append(b.entries, e)
+	}
+	return
+}
+
+func (b *bucket) RemoveYoungerSnapsExceedingKeepCount() (removed []Entry) {
+
+	if b.keepCount == RetentionGridKeepCountAll {
+		return nil
+	}
+
+	removeCount := len(b.entries) - b.keepCount
+	if removeCount <= 0 {
+		return nil
+	}
+
+	// sort youngest-to-oldest
+	sort.SliceStable(b.entries, func(i, j int) bool {
+		return b.entries[i].Date().After(b.entries[j].Date())
+	})
+
+	return b.entries[:removeCount]
+}
+
+func (g Grid) fitEntriesWithNow(now time.Time, entries []Entry) (keep, remove []Entry) {
+
 	buckets := make([]bucket, len(g.intervals))
 
+	buckets[0] = makeBucketFromInterval(now, g.intervals[0])
+	for i := 1; i < len(g.intervals); i++ {
+		buckets[i] = makeBucketFromInterval(buckets[i-1].youngerThan, g.intervals[i])
+	}
+
 	keep = make([]Entry, 0)
 	remove = make([]Entry, 0)
 
-	oldestIntervalStart := now
+assignEntriesToBuckets:
-	for i := range g.intervals {
-		oldestIntervalStart = oldestIntervalStart.Add(-g.intervals[i].Length())
-	}
-
 	for ei := 0; ei < len(entries); ei++ {
 		e := entries[ei]
-
+		// unconditionally keep entries that are in the future
-		date := e.Date()
+		if now.Before(e.Date()) {
-
-		if date == now || date.After(now) {
 			keep = append(keep, e)
-			continue
+			continue assignEntriesToBuckets
-		} else if date.Before(oldestIntervalStart) {
-			remove = append(remove, e)
-			continue
 		}
-
+		// add to matching bucket, if any
-		iStartTime := now
+		for bi := range buckets {
-		for i := 0; i < len(g.intervals); i++ {
+			if buckets[bi].AddIfContains(e) {
-			iStartTime = iStartTime.Add(-g.intervals[i].Length())
+				continue assignEntriesToBuckets
-			if date == iStartTime || dateInInterval(date, iStartTime, g.intervals[i]) {
-				buckets[i].entries = append(buckets[i].entries, e)
 			}
 		}
+		// unconditionally remove entries older than the oldest bucket
+		remove = append(remove, e)
 	}
 
-	for bi, b := range buckets {
+	// now apply the `KeepCount` per bucket
-
+	for _, b := range buckets {
-		interval := g.intervals[bi]
+		destroy := b.RemoveYoungerSnapsExceedingKeepCount()
-
+		remove = append(remove, destroy...)
-		sort.SliceStable(b.entries, func(i, j int) bool {
+		keep = append(keep, b.entries...)
-			return b.entries[i].LessThan((b.entries[j]))
-		})
-
-		i := 0
-		for ; (interval.KeepCount() == RetentionGridKeepCountAll || i < interval.KeepCount()) && i < len(b.entries); i++ {
-			keep = append(keep, b.entries[i])
-		}
-		for ; i < len(b.entries); i++ {
-			remove = append(remove, b.entries[i])
-		}
-
 	}
-
 	return
-
 }

pruning/retentiongrid/retentiongrid_test.go

@@ -10,25 +10,18 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-type retentionIntervalStub struct {
+type testInterval struct {
 	length    time.Duration
 	keepCount int
 }
 
-func (i *retentionIntervalStub) Length() time.Duration {
+func (i *testInterval) Length() time.Duration { return i.length }
-	return i.length
+func (i *testInterval) KeepCount() int        { return i.keepCount }
-}
-
-func (i *retentionIntervalStub) KeepCount() int {
-	return i.keepCount
-}
 
 func gridFromString(gs string) (g *Grid) {
-	intervals := strings.Split(gs, "|")
+	sintervals := strings.Split(gs, "|")
-	g = &Grid{
+	intervals := make([]Interval, len(sintervals))
-		intervals: make([]Interval, len(intervals)),
+	for idx, i := range sintervals {
-	}
-	for idx, i := range intervals {
 		comps := strings.SplitN(i, ",", 2)
 		var durationStr, numSnapsStr string
 		durationStr = comps[0]
@@ -39,7 +32,7 @@ func gridFromString(gs string) (g *Grid) {
 		}
 
 		var err error
-		var interval retentionIntervalStub
+		var interval testInterval
 
 		if interval.keepCount, err = strconv.Atoi(numSnapsStr); err != nil {
 			panic(err)
@@ -48,44 +41,38 @@ func gridFromString(gs string) (g *Grid) {
 			panic(err)
 		}
 
-		g.intervals[idx] = &interval
+		intervals[idx] = &interval
 	}
-	return
+	return NewGrid(intervals)
 }
 
-type dummySnap struct {
+type testSnap struct {
 	Name       string
 	ShouldKeep bool
 	date       time.Time
 }
 
-func (ds dummySnap) Date() time.Time {
+func (ds testSnap) Date() time.Time { return ds.date }
-	return ds.date
-}
-
-func (ds dummySnap) LessThan(b Entry) bool {
-	return ds.date.Before(b.(dummySnap).date) // don't have a txg here
-}
 
 func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, remove []Entry) {
 
-	snapDescr := func(d dummySnap) string {
+	snapDescr := func(d testSnap) string {
 		return fmt.Sprintf("%s@%s", d.Name, d.date.Sub(now))
 	}
 
 	t.Logf("keep list:\n")
 	for k := range keep {
-		t.Logf("\t%s\n", snapDescr(keep[k].(dummySnap)))
+		t.Logf("\t%s\n", snapDescr(keep[k].(testSnap)))
 	}
 	t.Logf("remove list:\n")
 	for k := range remove {
-		t.Logf("\t%s\n", snapDescr(remove[k].(dummySnap)))
+		t.Logf("\t%s\n", snapDescr(remove[k].(testSnap)))
 	}
 
 	t.Logf("\n\n")
 
 	for _, s := range input {
-		d := s.(dummySnap)
+		d := s.(testSnap)
 		descr := snapDescr(d)
 		t.Logf("testing %s\n", descr)
 		if d.ShouldKeep {
@@ -97,21 +84,18 @@ func validateRetentionGridFitEntries(t *testing.T, now time.Time, input, keep, r
 
 	t.Logf("resulting list:\n")
 	for k := range keep {
-		t.Logf("\t%s\n", snapDescr(keep[k].(dummySnap)))
+		t.Logf("\t%s\n", snapDescr(keep[k].(testSnap)))
 	}
 }
 
-func TestRetentionGridFitEntriesEmptyInput(t *testing.T) {
+func TestEmptyInput(t *testing.T) {
 	g := gridFromString("10m|10m|10m|1h")
-	keep, remove := g.FitEntries(time.Now(), []Entry{})
+	keep, remove := g.FitEntries([]Entry{})
 	assert.Empty(t, keep)
 	assert.Empty(t, remove)
 }
 
-func TestRetentionGridFitEntriesIntervalBoundariesAndAlignment(t *testing.T) {
+func TestIntervalBoundariesAndAlignment(t *testing.T) {
-
-	// Intervals are (duration], i.e. 10min is in the first interval, not in the second
-
 	g := gridFromString("10m|10m|10m")
 
 	t.Logf("%#v\n", g)
@@ -119,20 +103,51 @@ func TestRetentionGridFitEntriesIntervalBoundariesAndAlignment(t *testing.T) {
 	now := time.Unix(0, 0)
 
 	snaps := []Entry{
-		dummySnap{"0", true, now.Add(1 * time.Minute)},    // before now
+		testSnap{"0", true, now.Add(1 * time.Minute)},    // before now => keep unconditionally
-		dummySnap{"1", true, now},                         // before now
+		testSnap{"1", true, now},                         // 1st interval left edge => inclusive
-		dummySnap{"2", true, now.Add(-10 * time.Minute)},  // 1st interval
+		testSnap{"2", true, now.Add(-10 * time.Minute)},  // 2nd interval left edge => inclusive
-		dummySnap{"3", true, now.Add(-20 * time.Minute)},  // 2nd interval
+		testSnap{"3", true, now.Add(-20 * time.Minute)},  // 3rd interval left edge => inclusuive
-		dummySnap{"4", true, now.Add(-30 * time.Minute)},  // 3rd interval
+		testSnap{"4", false, now.Add(-30 * time.Minute)}, // 3rd interval right edge => excludive
-		dummySnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval
+		testSnap{"5", false, now.Add(-40 * time.Minute)}, // after last interval => remove unconditionally
 	}
 
-	keep, remove := g.FitEntries(now, snaps)
+	keep, remove := g.fitEntriesWithNow(now, snaps)
 	validateRetentionGridFitEntries(t, now, snaps, keep, remove)
-
 }
 
-func TestRetentionGridFitEntries(t *testing.T) {
+func TestKeepsOldestSnapsInABucket(t *testing.T) {
+	g := gridFromString("1m,2")
+
+	relt := func(secs int64) time.Time { return time.Unix(secs, 0) }
+
+	snaps := []Entry{
+		testSnap{"1", true, relt(1)},
+		testSnap{"2", true, relt(2)},
+		testSnap{"3", false, relt(3)},
+		testSnap{"4", false, relt(4)},
+		testSnap{"5", false, relt(5)},
+	}
+
+	now := relt(6)
+	keep, remove := g.FitEntries(snaps)
+	validateRetentionGridFitEntries(t, now, snaps, keep, remove)
+}
+
+func TestRespectsKeepCountAll(t *testing.T) {
+	g := gridFromString("1m,-1|1m,1")
+	relt := func(secs int64) time.Time { return time.Unix(secs, 0) }
+	snaps := []Entry{
+		testSnap{"a", true, relt(0)},
+		testSnap{"b", true, relt(-1)},
+		testSnap{"c", true, relt(-2)},
+		testSnap{"d", false, relt(-60)},
+		testSnap{"e", true, relt(-61)},
+	}
+	keep, remove := g.FitEntries(snaps)
+	validateRetentionGridFitEntries(t, relt(61), snaps, keep, remove)
+}
+
+func TestComplex(t *testing.T) {
 
 	g := gridFromString("10m,-1|10m|10m,2|1h")
 
@@ -141,20 +156,29 @@ func TestRetentionGridFitEntries(t *testing.T) {
 	now := time.Unix(0, 0)
 
 	snaps := []Entry{
-		dummySnap{"1", true, now.Add(3 * time.Minute)},   // pre-now must always be kept
+		// pre-now must always be kept
-		dummySnap{"b1", true, now.Add(-6 * time.Minute)}, // 1st interval allows unlimited entries
+		testSnap{"1", true, now.Add(3 * time.Minute)},
-		dummySnap{"b3", true, now.Add(-8 * time.Minute)}, // 1st interval allows unlimited entries
+		// 1st interval allows unlimited entries
-		dummySnap{"b2", true, now.Add(-9 * time.Minute)}, // 1st interval allows unlimited entries
+		testSnap{"b1", true, now.Add(-6 * time.Minute)},
-		dummySnap{"a", false, now.Add(-11 * time.Minute)},
+		testSnap{"b3", true, now.Add(-8 * time.Minute)},
-		dummySnap{"c", true, now.Add(-19 * time.Minute)}, // 2nd interval allows 1 entry
+		testSnap{"b2", true, now.Add(-9 * time.Minute)},
-		dummySnap{"foo", false, now.Add(-25 * time.Minute)},
+		// 2nd interval allows 1 entry
-		dummySnap{"bar", true, now.Add(-26 * time.Minute)}, // 3rd interval allows 2 entries
+		testSnap{"a", false, now.Add(-11 * time.Minute)},
-		dummySnap{"border", true, now.Add(-30 * time.Minute)},
+		testSnap{"c", true, now.Add(-19 * time.Minute)},
-		dummySnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)},
+		// 3rd interval allows 2 entries
-		dummySnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)}, // before earliest interval must always be deleted
+		testSnap{"foo", true, now.Add(-25 * time.Minute)},
-		dummySnap{"f", false, now.Add(-2 * time.Hour)},
+		testSnap{"bar", true, now.Add(-26 * time.Minute)},
+		// this is at the left edge of the 4th interval
+		testSnap{"border", false, now.Add(-30 * time.Minute)},
+		// right in the 4th interval
+		testSnap{"d", true, now.Add(-1*time.Hour - 15*time.Minute)},
+		// on the right edge of 4th interval => not in it => delete
+		testSnap{"q", false, now.Add(-1*time.Hour - 30*time.Minute)},
+		// older then 4th interval => always delete
+		testSnap{"e", false, now.Add(-1*time.Hour - 31*time.Minute)},
+		testSnap{"f", false, now.Add(-2 * time.Hour)},
 	}
-	keep, remove := g.FitEntries(now, snaps)
+	keep, remove := g.fitEntriesWithNow(now, snaps)
 
 	validateRetentionGridFitEntries(t, now, snaps, keep, remove)
 

replication/driver/replication_driver.go

@@ -9,12 +9,14 @@ import (
 	"sync"
 	"time"
 
+	"github.com/kr/pretty"
 	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/daemon/logging/trace"
-	"github.com/zrepl/zrepl/zfs"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"github.com/zrepl/zrepl/zfs"
+
 	"github.com/zrepl/zrepl/replication/report"
 	"github.com/zrepl/zrepl/util/chainlock"
 	"github.com/zrepl/zrepl/util/envconst"
@@ -228,7 +230,10 @@ func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
 			errRep := cur.errorReport()
 
 			if rep.State == report.AttemptDone {
-				log.Debug("attempt completed successfully")
+				if len(rep.Filesystems) == 0 {
+					log.Warn("no filesystems were considered for replication")
+				}
+				log.Debug("attempt completed")
 				break
 			}
 
@@ -236,7 +241,7 @@ func Do(ctx context.Context, planner Planner) (ReportFunc, WaitFunc) {
 			log.WithField("most_recent_err", mostRecentErr).WithField("most_recent_err_class", mostRecentErrClass).Debug("most recent error used for re-connect decision")
 			if mostRecentErr == nil {
 				// inconsistent reporting, let's bail out
-				log.Warn("attempt does not report done but error report does not report errors, aborting run")
+				log.WithField("attempt_state", rep.State).Warn("attempt does not report done but error report does not report errors, aborting run")
 				break
 			}
 			log.WithError(mostRecentErr.Err).Error("most recent error in this attempt")
@@ -310,6 +315,9 @@ func (a *attempt) doGlobalPlanning(ctx context.Context, prev *attempt) map[*fs]*
 		return nil
 	}
 
+	// a.fss != nil indicates that there was no planning error (see doc comment)
+	a.fss = make([]*fs, 0)
+
 	for _, pfs := range pfss {
 		fs := &fs{
 			fs: pfs,
@@ -675,10 +683,12 @@ func (a *attempt) report() *report.AttemptReport {
 		r.Filesystems[i] = a.fss[i].report()
 	}
 
-	state := report.AttemptPlanning
+	var state report.AttemptState
-	if a.planErr != nil {
+	if a.planErr == nil && a.fss == nil {
+		state = report.AttemptPlanning
+	} else if a.planErr != nil && a.fss == nil {
 		state = report.AttemptPlanningError
-	} else if a.fss != nil {
+	} else if a.planErr == nil && a.fss != nil {
 		if a.finishedAt.IsZero() {
 			state = report.AttemptFanOutFSs
 		} else {
@@ -691,6 +701,8 @@ func (a *attempt) report() *report.AttemptReport {
 				state = report.AttemptFanOutError
 			}
 		}
+	} else {
+		panic(fmt.Sprintf("attempt.planErr and attempt.fss must not both be != nil:\n%s\n%s", pretty.Sprint(a.planErr), pretty.Sprint(a.fss)))
 	}
 	r.State = state
 

replication/driver/replication_driver_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/replication/report"

replication/driver/replication_stepqueue_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/montanaflynn/stats"
 	"github.com/stretchr/testify/assert"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 )
 

replication/logic/diff/diff.go

@@ -77,56 +77,48 @@ func IncrementalPath(receiver, sender []*FilesystemVersion) (incPath []*Filesyst
 	receiver = SortVersionListByCreateTXGThenBookmarkLTSnapshot(receiver)
 	sender = SortVersionListByCreateTXGThenBookmarkLTSnapshot(sender)
 
-	// Find most recent common ancestor by name, preferring snapshots over bookmarks
+	var mrcaCandidate struct {
+		found bool
+		guid  uint64
+		r, s  int
+	}
 
-	mrcaRcv := len(receiver) - 1
+findCandidate:
-	mrcaSnd := len(sender) - 1
+	for r := len(receiver) - 1; r >= 0; r-- {
-
+		for s := len(sender) - 1; s >= 0; s-- {
-	for mrcaRcv >= 0 && mrcaSnd >= 0 {
+			if sender[s].GetGuid() == receiver[r].GetGuid() {
-		if receiver[mrcaRcv].Guid == sender[mrcaSnd].Guid {
+				mrcaCandidate.guid = sender[s].GetGuid()
-			// Since we arrive from the end of the array, and because we defined bookmark < snapshot,
+				mrcaCandidate.s = s
-			// this condition will match snapshot first, which is what we want because it gives us
+				mrcaCandidate.r = r
-			// size estimation
+				mrcaCandidate.found = true
-			break
+				break findCandidate
-		}
+			}
-		receiverCreation, err := receiver[mrcaRcv].CreationAsTime()
-		if err != nil {
-			panic(err) // FIXME move this to a sorting phase before
-		}
-		senderCreation, err := sender[mrcaSnd].CreationAsTime()
-		if err != nil {
-			panic(err) // FIXME move this to the sorting phase before
-		}
-
-		if receiverCreation.Before(senderCreation) {
-			mrcaSnd--
-		} else {
-			mrcaRcv--
 		}
 	}
 
-	if mrcaRcv == -1 || mrcaSnd == -1 {
+	// handle failure cases
+	if !mrcaCandidate.found {
 		return nil, &ConflictNoCommonAncestor{
 			SortedSenderVersions:   sender,
 			SortedReceiverVersions: receiver,
 		}
-	}
+	} else if mrcaCandidate.r != len(receiver)-1 {
-
-	if mrcaRcv != len(receiver)-1 {
 		return nil, &ConflictDiverged{
 			SortedSenderVersions:   sender,
 			SortedReceiverVersions: receiver,
-			CommonAncestor:         sender[mrcaSnd],
+			CommonAncestor:         sender[mrcaCandidate.s],
-			SenderOnly:             sender[mrcaSnd+1:],
+			SenderOnly:             sender[mrcaCandidate.s+1:],
-			ReceiverOnly:           receiver[mrcaRcv+1:],
+			ReceiverOnly:           receiver[mrcaCandidate.r+1:],
 		}
 	}
 
+	// incPath is possible
+
 	// incPath must not contain bookmarks except initial one,
 	incPath = make([]*FilesystemVersion, 0, len(sender))
-	incPath = append(incPath, sender[mrcaSnd])
+	incPath = append(incPath, sender[mrcaCandidate.s])
 	// it's ok if incPath[0] is a bookmark, but not the subsequent ones in the incPath
-	for i := mrcaSnd + 1; i < len(sender); i++ {
+	for i := mrcaCandidate.s + 1; i < len(sender); i++ {
 		if sender[i].Type == FilesystemVersion_Snapshot && incPath[len(incPath)-1].Guid != sender[i].Guid {
 			incPath = append(incPath, sender[i])
 		}

replication/logic/replication_logic.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 
 	"github.com/zrepl/zrepl/logger"

replication/logic/replication_logic_policy.go

@@ -2,6 +2,7 @@ package logic
 
 import (
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/replication/logic/pdu"
 )

rpc/rpc_client.go

@@ -10,9 +10,10 @@ import (
 	"sync/atomic"
 	"time"
 
-	"github.com/zrepl/zrepl/daemon/logging/trace"
 	"google.golang.org/grpc"
 
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+
 	"github.com/google/uuid"
 
 	"github.com/zrepl/zrepl/replication/logic"

transport/tcp/serve_tcp_ipmap.go

@@ -8,8 +8,9 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
-	"github.com/zrepl/zrepl/transport"
 	"golang.org/x/sys/unix"
+
+	"github.com/zrepl/zrepl/transport"
 )
 
 type ipMapEntry struct {

transport/transport.go

@@ -8,6 +8,7 @@ import (
 	"syscall"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/rpc/dataconn/timeoutconn"

util/semaphore/semaphore.go

@@ -3,8 +3,9 @@ package semaphore
 import (
 	"context"
 
-	"github.com/zrepl/zrepl/daemon/logging/trace"
 	wsemaphore "golang.org/x/sync/semaphore"
+
+	"github.com/zrepl/zrepl/daemon/logging/trace"
 )
 
 type S struct {

util/semaphore/semaphore_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/daemon/logging/trace"
 )
 

version/version.go

@@ -13,6 +13,7 @@ var (
 
 type ZreplVersionInformation struct {
 	Version         string
+	RuntimeGo       string
 	RuntimeGOOS     string
 	RuntimeGOARCH   string
 	RUNTIMECompiler string
@@ -21,6 +22,7 @@ type ZreplVersionInformation struct {
 func NewZreplVersionInformation() *ZreplVersionInformation {
 	return &ZreplVersionInformation{
 		Version:         zreplVersion,
+		RuntimeGo:       runtime.Version(),
 		RuntimeGOOS:     runtime.GOOS,
 		RuntimeGOARCH:   runtime.GOARCH,
 		RUNTIMECompiler: runtime.Compiler,
@@ -28,8 +30,8 @@ func NewZreplVersionInformation() *ZreplVersionInformation {
 }
 
 func (i *ZreplVersionInformation) String() string {
-	return fmt.Sprintf("zrepl version=%s GOOS=%s GOARCH=%s Compiler=%s",
+	return fmt.Sprintf("zrepl version=%s go=%s GOOS=%s GOARCH=%s Compiler=%s",
-		i.Version, i.RuntimeGOOS, i.RuntimeGOARCH, i.RUNTIMECompiler)
+		i.Version, i.RuntimeGo, i.RuntimeGOOS, i.RuntimeGOARCH, i.RUNTIMECompiler)
 }
 
 var prometheusMetric = prometheus.NewUntypedFunc(

zfs/placeholder.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 

zfs/versions.go

@@ -4,8 +4,10 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/pkg/errors"
@@ -39,6 +41,7 @@ func (s VersionTypeSet) zfsListTFlagRepr() string {
 	for t := range s {
 		types = append(types, t.String())
 	}
+	sort.StringSlice(types).Sort()
 	return strings.Join(types, ",")
 }
 func (s VersionTypeSet) String() string { return s.zfsListTFlagRepr() }
@@ -218,14 +221,23 @@ func ZFSListFilesystemVersions(ctx context.Context, fs *DatasetPath, options Lis
 	promTimer := prometheus.NewTimer(prom.ZFSListFilesystemVersionDuration.WithLabelValues(fs.ToString()))
 	defer promTimer.ObserveDuration()
 
+	// Note: we don't create a separate trace.Task here because our loop that consumes
+	// the goroutine's output doesn't use ctx.
 	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
+	// make sure the goroutine doesn't outlive this function call
-	go ZFSListChan(ctx, listResults,
+	var wg sync.WaitGroup
-		[]string{"name", "guid", "createtxg", "creation", "userrefs"},
+	wg.Add(1)
-		fs,
+	defer wg.Wait()
-		"-r", "-d", "1",
+	defer cancel() // on exit, cancel list process before waiting for it
-		"-t", options.typesFlagArgs(),
+	go func() {
-		"-s", "createtxg", fs.ToString())
+		defer wg.Done()
+		ZFSListChan(ctx, listResults,
+			[]string{"name", "guid", "createtxg", "creation", "userrefs"},
+			fs,
+			"-r", "-d", "1",
+			"-t", options.typesFlagArgs(),
+			"-s", "createtxg", fs.ToString())
+	}()
 
 	res = make([]FilesystemVersion, 0)
 	for listResult := range listResults {

zfs/zfs.go

@@ -173,6 +173,8 @@ func ZFSList(ctx context.Context, properties []string, zfsArgs ...string) (res [
 		"-o", strings.Join(properties, ","))
 	args = append(args, zfsArgs...)
 
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, args...)
 	stdout, stderrBuf, err := cmd.StdoutPipeWithErrorBuf()
 	if err != nil {
@@ -182,6 +184,11 @@ func ZFSList(ctx context.Context, properties []string, zfsArgs ...string) (res [
 	if err = cmd.Start(); err != nil {
 		return
 	}
+	// in case we return early, we want to kill the zfs list process and wait for it to exit
+	defer func() {
+		_ = cmd.Wait()
+	}()
+	defer cancel()
 
 	s := bufio.NewScanner(stdout)
 	buf := make([]byte, 1024)
@@ -244,6 +251,8 @@ func ZFSListChan(ctx context.Context, out chan ZFSListResult, properties []strin
 		}
 	}
 
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
 	cmd := zfscmd.CommandContext(ctx, ZFS_BINARY, args...)
 	stdout, stderrBuf, err := cmd.StdoutPipeWithErrorBuf()
 	if err != nil {
@@ -259,6 +268,7 @@ func ZFSListChan(ctx context.Context, out chan ZFSListResult, properties []strin
 		// in which case we'll return an 'unexpected output' error and not the exit status
 		_ = cmd.Wait()
 	}()
+	defer cancel() // in case we return before our regular call to cmd.Wait(), kill the zfs list process
 
 	s := bufio.NewScanner(stdout)
 	buf := make([]byte, 1024) // max line length

zfs/zfscmd/zfscmd.go

@@ -79,6 +79,13 @@ func (c *Cmd) log() Logger {
 	return getLogger(c.ctx).WithField("cmd", c.String())
 }
 
+// Start the command.
+//
+// This creates a new trace.WithTask as a child task of the ctx passed to CommandContext.
+// If the process is successfully started (err == nil), it is the CALLER'S RESPONSIBILITY to ensure that
+// the spawned process does not outlive the ctx's trace.Task.
+//
+// If this method returns an error, the Cmd instance is invalid. Start must not be called repeatedly.
 func (c *Cmd) Start() (err error) {
 	c.startPre(true)
 	err = c.cmd.Start()
@@ -86,7 +93,9 @@ func (c *Cmd) Start() (err error) {
 	return err
 }
 
-// only call this after a successful call to .Start()
+// Get the underlying os.Process.
+//
+// Only call this method after a successful call to .Start().
 func (c *Cmd) Process() *os.Process {
 	if c.startedAt.IsZero() {
 		panic("calling Process() only allowed after successful call to Start()")
@@ -94,6 +103,10 @@ func (c *Cmd) Process() *os.Process {
 	return c.cmd.Process
 }
 
+// Blocking wait for the process to exit.
+// May be called concurrently and repeatly (exec.Cmd.Wait() semantics apply).
+//
+// Only call this method after a successful call to .Start().
 func (c *Cmd) Wait() (err error) {
 	c.waitPre()
 	err = c.cmd.Wait()
@@ -112,14 +125,16 @@ func (c *Cmd) startPre(newTask bool) {
 }
 
 func (c *Cmd) startPost(err error) {
-	now := time.Now()
 
-	c.mtx.Lock()
+	now := time.Now()
 	c.startedAt = now
-	c.mtx.Unlock()
 
 	startPostReport(c, err, now)
 	startPostLogging(c, err, now)
+
+	if err != nil {
+		c.waitReturnEndSpanCb()
+	}
 }
 
 func (c *Cmd) waitPre() {

zync/README.md

@@ -0,0 +1,7 @@
+A tool that re-used zrepl abstractions to be an rsync-like sync tool for ZFS.
+
+Test environment:
+
+```
+go build -o zync && rsync -a ./zync root@192.168.124.233:/usr/local/bin/ && sudo ./zync local:///rpool/zrepltlstest/src ssh://root:%2Fhome%2Fcs%2Fzrepl%2Fzrepl%2Fzync%2Ftestid@192.168.124.233/p1/zync_sink 
+```

zync/transport/sshdirect/dial.go

@@ -0,0 +1,406 @@
+package sshdirect
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net"
+	"os/exec"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/hashicorp/yamux"
+	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/util/circlog"
+)
+
+type Endpoint struct {
+	Host         string
+	User         string
+	Port         uint16
+	IdentityFile string
+	SSHCommand   string
+	Options      []string
+	RunCommand   []string
+}
+
+func (e Endpoint) CmdArgs() (cmd string, args []string, env []string) {
+
+	if e.SSHCommand != "" {
+		cmd = e.SSHCommand
+	} else {
+		cmd = "ssh"
+	}
+
+	args = make([]string, 0, 2*len(e.Options)+4)
+	args = append(args,
+		"-p", fmt.Sprintf("%d", e.Port),
+		"-T",
+		"-i", e.IdentityFile,
+		"-o", "BatchMode=yes",
+	)
+	for _, option := range e.Options {
+		args = append(args, "-o", option)
+	}
+	args = append(args, fmt.Sprintf("%s@%s", e.User, e.Host))
+
+	args = append(args, e.RunCommand...)
+
+	env = []string{}
+
+	return
+}
+
+type SSHConn struct {
+	cmd    *exec.Cmd
+	stdin  io.WriteCloser
+	stdout io.ReadCloser
+
+	shutdownMtx    sync.Mutex
+	shutdownResult *shutdownResult // TODO not used anywhere
+	cmdCancel      context.CancelFunc
+}
+
+const go_network string = "netssh"
+
+type clientAddr struct {
+	pid int
+}
+
+func (a clientAddr) Network() string {
+	return go_network
+}
+
+func (a clientAddr) String() string {
+	return fmt.Sprintf("pid=%d", a.pid)
+}
+
+func (conn *SSHConn) LocalAddr() net.Addr {
+	proc := conn.cmd.Process
+	if proc == nil {
+		return clientAddr{-1}
+	}
+	return clientAddr{proc.Pid}
+}
+
+func (conn *SSHConn) RemoteAddr() net.Addr {
+	return conn.LocalAddr()
+}
+
+// Read implements io.Reader.
+// It returns *IOError for any non-nil error that is != io.EOF.
+func (conn *SSHConn) Read(p []byte) (int, error) {
+	n, err := conn.stdout.Read(p)
+	if err != nil && err != io.EOF {
+		return n, &IOError{err}
+	}
+	return n, err
+}
+
+// Write implements io.Writer.
+// It returns *IOError for any error != nil.
+func (conn *SSHConn) Write(p []byte) (int, error) {
+	n, err := conn.stdin.Write(p)
+	if err != nil {
+		return n, &IOError{err}
+	}
+	return n, err
+}
+
+func (conn *SSHConn) CloseWrite() error {
+	return conn.stdin.Close()
+}
+
+type deadliner interface {
+	SetReadDeadline(time.Time) error
+	SetWriteDeadline(time.Time) error
+}
+
+func (conn *SSHConn) SetReadDeadline(t time.Time) error {
+	// type assertion is covered by test TestExecCmdPipesDeadlineBehavior
+	return conn.stdout.(deadliner).SetReadDeadline(t)
+}
+
+func (conn *SSHConn) SetWriteDeadline(t time.Time) error {
+	// type assertion is covered by test TestExecCmdPipesDeadlineBehavior
+	return conn.stdin.(deadliner).SetWriteDeadline(t)
+}
+
+func (conn *SSHConn) SetDeadline(t time.Time) error {
+	// try both
+	rerr := conn.SetReadDeadline(t)
+	werr := conn.SetWriteDeadline(t)
+	if rerr != nil {
+		return rerr
+	}
+	if werr != nil {
+		return werr
+	}
+	return nil
+}
+
+func (conn *SSHConn) Close() error {
+	conn.shutdownProcess()
+	return nil // FIXME: waitError will be non-zero because we signaled it, shutdownProcess needs to distinguish that
+}
+
+type shutdownResult struct {
+	waitErr error
+}
+
+func (conn *SSHConn) shutdownProcess() *shutdownResult {
+	conn.shutdownMtx.Lock()
+	defer conn.shutdownMtx.Unlock()
+
+	if conn.shutdownResult != nil {
+		return conn.shutdownResult
+	}
+
+	wait := make(chan error, 1)
+	go func() {
+		if err := conn.cmd.Process.Signal(syscall.SIGTERM); err != nil {
+			// TODO log error
+			return
+		}
+		wait <- conn.cmd.Wait()
+	}()
+
+	timeout := time.NewTimer(1 * time.Second) // FIXME const
+	defer timeout.Stop()
+
+	select {
+	case waitErr := <-wait:
+		conn.shutdownResult = &shutdownResult{waitErr}
+	case <-timeout.C:
+		conn.cmdCancel()
+		waitErr := <-wait // reuse existing Wait invocation, must not call twice
+		conn.shutdownResult = &shutdownResult{waitErr}
+	}
+	return conn.shutdownResult
+}
+
+// Cmd returns the underlying *exec.Cmd (the ssh client process)
+// Use read-only, should not be necessary for regular users.
+func (conn *SSHConn) Cmd() *exec.Cmd {
+	return conn.cmd
+}
+
+// CmdCancel bypasses the normal shutdown mechanism of SSHConn
+// (that is, calling Close) and cancels the process's context,
+// which usually results in SIGKILL being sent to the process.
+// Intended for integration tests, regular users shouldn't use it.
+func (conn *SSHConn) CmdCancel() {
+	conn.cmdCancel()
+}
+
+const bannerMessageLen = 31
+
+var messages = make(map[string][]byte)
+
+func mustMessage(str string) []byte {
+	if len(str) > bannerMessageLen {
+		panic("message length must be smaller than bannerMessageLen")
+	}
+	if _, ok := messages[str]; ok {
+		panic("duplicate message")
+	}
+	var buf bytes.Buffer
+	n, _ := buf.WriteString(str)
+	if n != len(str) {
+		panic("message must only contain ascii / 8-bit chars")
+	}
+	buf.Write(bytes.Repeat([]byte{0}, bannerMessageLen-n))
+	return buf.Bytes()
+}
+
+var banner_msg = mustMessage("SSDIRECTHCON_HELO")
+var proxy_error_msg = mustMessage("SSDIRECTHCON_PROXY_ERROR") /* FIXME irrelevant, was copy-pasta */
+var begin_msg = mustMessage("SSDIRECTHCON_BEGIN")
+
+type SSHError struct {
+	RWCError      error
+	WhileActivity string
+}
+
+// Error() will try to present a one-line error message unless ssh stderr output is longer than one line
+func (e *SSHError) Error() string {
+
+	exitErr, ok := e.RWCError.(*exec.ExitError)
+	if !ok {
+		return fmt.Sprintf("ssh: %s", e.RWCError)
+	}
+
+	ws := exitErr.ProcessState.Sys().(syscall.WaitStatus)
+	var wsmsg string
+	if ws.Exited() {
+		wsmsg = fmt.Sprintf("(exit status %d)", ws.ExitStatus())
+	} else {
+		wsmsg = fmt.Sprintf("(%s)", ws.Signal())
+	}
+
+	haveSSHMessage := len(exitErr.Stderr) > 0
+	sshOnelineStderr := false
+	if i := bytes.Index(exitErr.Stderr, []byte("\n")); i == len(exitErr.Stderr)-1 {
+		sshOnelineStderr = true
+	}
+	stderr := bytes.TrimSpace(exitErr.Stderr)
+
+	if haveSSHMessage {
+		if sshOnelineStderr {
+			return fmt.Sprintf("ssh: '%s' %s", stderr, wsmsg) // FIXME proper single-quoting
+		} else {
+			return fmt.Sprintf("ssh %s\n%s", wsmsg, stderr)
+		}
+	}
+
+	return fmt.Sprintf("ssh terminated without stderr output %s", wsmsg)
+
+}
+
+type ProtocolError struct {
+	What string
+}
+
+func (e ProtocolError) Error() string {
+	return e.What
+}
+
+// Dial connects to the remote endpoint where it expects a command executing Proxy().
+// Dial performs a handshake consisting of the exchange of banner messages before returning the connection.
+// If the handshake cannot be completed before dialCtx is Done(), the underlying ssh command is killed
+// and the dialCtx.Err() returned.
+// If the handshake completes, dialCtx's deadline does not affect the returned connection.
+//
+// Errors returned are either dialCtx.Err(), or intances of ProtocolError or *SSHError
+func Dial(dialCtx context.Context, endpoint Endpoint) (*SSHConn, error) {
+
+	sshCmd, sshArgs, sshEnv := endpoint.CmdArgs()
+	commandCtx, commandCancel := context.WithCancel(context.Background())
+	cmd := exec.CommandContext(commandCtx, sshCmd, sshArgs...)
+	cmd.Env = sshEnv
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, err
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, err
+	}
+
+	stderrBuf, err := circlog.NewCircularLog(1 << 15)
+	if err != nil {
+		panic(err) // wrong API usage
+	}
+	cmd.Stderr = stderrBuf
+
+	if err = cmd.Start(); err != nil {
+		return nil, err
+	}
+	cmdWaitErrOrIOErr := func(ioErr error, what string) *SSHError {
+		werr := cmd.Wait()
+		if werr, ok := werr.(*exec.ExitError); ok {
+			werr.Stderr = []byte(stderrBuf.String())
+			return &SSHError{werr, what}
+		}
+		return &SSHError{ioErr, what}
+	}
+
+	confErrChan := make(chan error, 1)
+	go func() {
+		defer close(confErrChan)
+		var buf bytes.Buffer
+		if _, err := io.CopyN(&buf, stdout, int64(len(banner_msg))); err != nil {
+			confErrChan <- cmdWaitErrOrIOErr(err, "read banner")
+			return
+		}
+		resp := buf.Bytes()
+		switch {
+		case bytes.Equal(resp, banner_msg):
+			break
+		case bytes.Equal(resp, proxy_error_msg):
+			_ = cmdWaitErrOrIOErr(nil, "")
+			confErrChan <- ProtocolError{"proxy error, check remote configuration"}
+			return
+		default:
+			_ = cmdWaitErrOrIOErr(nil, "")
+			confErrChan <- ProtocolError{fmt.Sprintf("unknown banner message: %v", resp)}
+			return
+		}
+		buf.Reset()
+		buf.Write(begin_msg)
+		if _, err := io.Copy(stdin, &buf); err != nil {
+			confErrChan <- cmdWaitErrOrIOErr(err, "send begin message")
+			return
+		}
+	}()
+
+	select {
+	case <-dialCtx.Done():
+
+		commandCancel()
+		// cancelling will make one of the calls in above goroutine fail,
+		// and the goroutine will send the error to confErrChan
+		//
+		// ignore the error and return the cancellation cause
+
+		// draining always terminates because we know the channel is always closed
+		for _ = range confErrChan {
+		}
+
+		// TODO collect stderr in this case
+		// can probably extend *SSHError for this but need to implement net.Error
+
+		return nil, dialCtx.Err()
+
+	case err := <-confErrChan:
+		if err != nil {
+			commandCancel()
+			return nil, err
+		}
+	}
+
+	return &SSHConn{
+		cmd:       cmd,
+		stdin:     stdin,
+		stdout:    stdout,
+		cmdCancel: commandCancel,
+	}, nil
+}
+
+type Connecter struct {
+	s        *yamux.Session
+	endpoint Endpoint
+}
+
+var _ transport.Connecter = (*Connecter)(nil)
+
+func NewConnecter(ctx context.Context, endpoint Endpoint) (*Connecter, error) {
+	conn, err := Dial(ctx, endpoint)
+	if err != nil {
+		return nil, err
+	}
+	s, err := yamux.Client(conn, nil)
+	if err != nil {
+		return nil, err
+	}
+	return &Connecter{
+		s:        s,
+		endpoint: endpoint,
+	}, nil
+}
+
+type fakeWire struct {
+	net.Conn
+}
+
+func (w *fakeWire) CloseWrite() error {
+	time.Sleep(1*time.Second) // HACKY
+	return fmt.Errorf("fakeWire does not support CloseWrite")
+}
+
+func (c *Connecter) Connect(ctx context.Context) (transport.Wire, error) {
+	conn, err := c.s.Open()
+	return &fakeWire{conn}, err
+}

zync/transport/sshdirect/error.go

@@ -0,0 +1,48 @@
+package sshdirect
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"syscall"
+)
+
+type timeouter interface {
+	Timeout() bool
+}
+
+var _ timeouter = &os.PathError{}
+
+type IOError struct {
+	Cause error
+}
+
+var _ net.Error = &IOError{}
+
+func (e IOError) GoString() string {
+	return fmt.Sprintf("ServeConnIOError:%#v", e.Cause)
+}
+
+func (e IOError) Error() string {
+	// following case found by experiment
+	if pathErr, ok := e.Cause.(*os.PathError); ok {
+		if pathErr.Err == syscall.EPIPE {
+			return fmt.Sprintf("netssh %s: %s (likely: connection reset by peer)",
+				pathErr.Op, pathErr.Err,
+			)
+		}
+		return fmt.Sprintf("netssh: %s: %s", pathErr.Op, pathErr.Err)
+	}
+	return fmt.Sprintf("netssh: %s", e.Cause.Error())
+}
+
+func (e IOError) Timeout() bool {
+	if to, ok := e.Cause.(timeouter); ok {
+		return to.Timeout()
+	}
+	return false
+}
+
+func (e IOError) Temporary() bool {
+	return false
+}

zync/transport/sshdirect/serve.go

@@ -0,0 +1,91 @@
+package sshdirect
+
+import (
+	"bytes"
+	"io"
+	"log"
+	"net"
+	"os"
+	"time"
+
+	"github.com/hashicorp/yamux"
+)
+
+type ServeConn struct {
+	stdin, stdout *os.File
+}
+
+var _ net.Conn = (*ServeConn)(nil)
+
+func ServeStdin() (net.Listener, error) {
+
+	conn := &ServeConn{
+		stdin:  os.Stdin,
+		stdout: os.Stdout,
+	}
+
+	var buf bytes.Buffer
+	buf.Write(banner_msg)
+	if _, err := io.Copy(conn, &buf); err != nil {
+		log.Printf("error sending confirm message: %s", err)
+		conn.Close()
+		return nil, err
+	}
+	buf.Reset()
+	if _, err := io.CopyN(&buf, conn, int64(len(begin_msg))); err != nil {
+		log.Printf("error reading begin message: %s", err)
+		conn.Close()
+		return nil, err
+	}
+
+	return yamux.Server(conn, nil)
+}
+
+func (c *ServeConn) Read(p []byte) (int, error) {
+	return c.stdin.Read(p)
+}
+
+func (c *ServeConn) Write(p []byte) (int, error) {
+	return c.stdout.Write(p)
+}
+
+func (f *ServeConn) Close() (err error) {
+	e1 := f.stdin.Close()
+	e2 := f.stdout.Close()
+	// FIXME merge errors
+	if e1 != nil {
+		return e1
+	}
+	return e2
+}
+
+func (f *ServeConn) SetReadDeadline(t time.Time) error {
+	return f.stdin.SetReadDeadline(t)
+}
+
+func (f *ServeConn) SetWriteDeadline(t time.Time) error {
+	return f.stdout.SetReadDeadline(t)
+}
+
+func (f *ServeConn) SetDeadline(t time.Time) error {
+	// try both...
+	werr := f.SetWriteDeadline(t)
+	rerr := f.SetReadDeadline(t)
+	if werr != nil {
+		return werr
+	}
+	if rerr != nil {
+		return rerr
+	}
+	return nil
+}
+
+type serveAddr struct{}
+
+const GoNetwork string = "sshdirect"
+
+func (serveAddr) Network() string { return GoNetwork }
+func (serveAddr) String() string  { return "???" }
+
+func (f *ServeConn) LocalAddr() net.Addr  { return serveAddr{} }
+func (f *ServeConn) RemoteAddr() net.Addr { return serveAddr{} }

zync/transport/transportlistenerfromnetlistener/transportlistenerfromnetlistener.go

@@ -0,0 +1,47 @@
+package transportlistenerfromnetlistener
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/zrepl/zrepl/transport"
+)
+
+type wrapFixed struct {
+	id string
+	l  net.Listener
+}
+
+var _ transport.AuthenticatedListener = (*wrapFixed)(nil)
+
+func WrapFixed(l net.Listener, identity string) transport.AuthenticatedListener {
+	return &wrapFixed{identity, l}
+}
+
+func (w *wrapFixed) Addr() net.Addr {
+	return w.l.Addr()
+}
+
+type fakeWire struct {
+	net.Conn
+}
+
+func (w *fakeWire) CloseWrite() error {
+	time.Sleep(1*time.Second) // HACKY
+	return fmt.Errorf("fakeWire does not support CloseWrite")
+}
+
+func (w *wrapFixed) Accept(ctx context.Context) (*transport.AuthConn, error) {
+	nc, err := w.l.Accept()
+	if err != nil {
+		return nil, err
+	}
+
+	return transport.NewAuthConn(&fakeWire{nc}, w.id), nil
+}
+
+func (w *wrapFixed) Close() error {
+	return w.l.Close()
+}

zync/zync.go

@@ -0,0 +1,333 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/url"
+	"os"
+	"os/signal"
+	"runtime"
+	"strconv"
+	"syscall"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/daemon/logging"
+	"github.com/zrepl/zrepl/daemon/logging/trace"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/replication"
+	"github.com/zrepl/zrepl/replication/logic"
+	"github.com/zrepl/zrepl/replication/logic/pdu"
+	"github.com/zrepl/zrepl/rpc"
+	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/zfs"
+	"github.com/zrepl/zrepl/zync/transport/sshdirect"
+	"github.com/zrepl/zrepl/zync/transport/transportlistenerfromnetlistener"
+)
+
+var flagStdinserver = flag.String("stdinserver", "", "")
+var flagStderrToFile = flag.String("stderrtofile", "", "")
+
+const (
+	modeSender   = "sender"
+	modeReceiver = "receiver"
+)
+
+func connecter(ctx context.Context, u *url.URL, mode, fs string) (transport.Connecter, error) {
+	switch u.Scheme {
+	case "ssh":
+		return connecterSSH(ctx, u, mode, fs)
+	default:
+		panic(fmt.Sprintf("unknown scheme %q", u.Scheme))
+	}
+}
+
+func connecterSSH(ctx context.Context, u *url.URL, mode, fs string) (transport.Connecter, error) {
+	var port uint16
+	if u.Port() == "" {
+		port = 22
+	} else {
+		portU64, err := strconv.ParseUint(u.Port(), 10, 16)
+		if err != nil {
+			return nil, errors.Wrap(err, "invalid port")
+		}
+		port = uint16(portU64)
+	}
+
+	fmt.Println(u.User)
+	idFilePath, hasIdFile := u.User.Password()
+	if hasIdFile {
+		_, err := ioutil.ReadFile(idFilePath)
+		if err != nil {
+			fmt.Println(err)
+			hasIdFile = false
+		}
+	}
+	if !hasIdFile {
+		return nil, errors.New("must set password to identity file path")
+	}
+
+	ep := sshdirect.Endpoint{
+		Host:         u.Hostname(),
+		Port:         port,
+		User:         u.User.Username(),
+		IdentityFile: idFilePath,
+		RunCommand:   []string{"zync", "-stderrtofile", "/tmp/zync_server.log", "-stdinserver", mode, fs},
+	}
+	return sshdirect.NewConnecter(ctx, ep)
+}
+
+func onefsfilter(osname string) zfs.DatasetFilter {
+	f, err := filters.DatasetMapFilterFromConfig(map[string]bool{
+		osname: true,
+	})
+	if err != nil {
+		panic(err)
+	}
+	return f
+}
+
+func makeEndpoint(mode, fs string) (interface{}, error) {
+	switch mode {
+	case modeSender:
+		sc := endpoint.SenderConfig{
+			FSF:     onefsfilter(fs),
+			Encrypt: &zfs.NilBool{B: true},
+			JobID:   endpoint.MustMakeJobID("sender"),
+		}
+		return endpoint.NewSender(sc), nil
+	case modeReceiver:
+		rpath, err := zfs.NewDatasetPath(fs)
+		if err != nil {
+			panic(err)
+		}
+		if err != nil {
+			panic(err)
+		}
+		return endpoint.NewReceiver(endpoint.ReceiverConfig{
+			JobID:                      endpoint.MustMakeJobID("receiver"),
+			AppendClientIdentity:       false,
+			RootWithoutClientComponent: rpath,
+		}), nil
+	default:
+		return nil, fmt.Errorf("unknown mode %q", mode)
+	}
+}
+
+func serve(ctx context.Context, handler rpc.Handler) error {
+
+	// copy-pasta from passive.go
+	ctxInterceptor := func(handlerCtx context.Context, info rpc.HandlerContextInterceptorData, handler func(ctx context.Context)) {
+		// the handlerCtx is clean => need to inherit logging and tracing config from job context
+		handlerCtx = logging.WithInherit(handlerCtx, ctx)
+		handlerCtx = trace.WithInherit(handlerCtx, ctx)
+
+		handlerCtx, endTask := trace.WithTaskAndSpan(handlerCtx, "handler", fmt.Sprintf("method=%q", info.FullMethod()))
+		defer endTask()
+		handler(handlerCtx)
+	}
+
+	l, err := sshdirect.ServeStdin()
+	if err != nil {
+		panic(err)
+	}
+
+	srv := rpc.NewServer(handler, rpc.GetLoggersOrPanic(ctx), ctxInterceptor)
+	srv.Serve(ctx, transportlistenerfromnetlistener.WrapFixed(l, "fakeclientidentitymustnotbeempty"))
+	return nil
+}
+
+func parseFSArg(ctx context.Context, mode, arg string) (interface{}, error) {
+	u, err := url.Parse(arg)
+	if err != nil {
+		return nil, err
+	}
+	if !u.IsAbs() {
+		return nil, fmt.Errorf("URL must be absolute, got %q", arg)
+	}
+	if u.Path[0] != '/' {
+		panic("impl error: expecting leading /")
+	}
+	fs := u.Path[1:]
+	switch u.Scheme {
+	case "local":
+		if u.Host != "" {
+			panic("hostname must be empty for 'local' scheme")
+		}
+		return makeEndpoint(mode, fs)
+	default:
+		cn, err := connecter(ctx, u, mode, fs)
+		if err != nil {
+			return nil, err
+		}
+		return rpc.NewClient(cn, rpc.GetLoggersOrPanic(ctx)), nil
+	}
+}
+
+// func parseEndpoints(s, r string) (sender, receiver logic.Endpoint, _ error) {
+// 	sUrl, err := url.Parse(s)
+// 	if err != nil {
+// 		return nil, nil, err
+// 	}
+// 	rUrl, err := url.Parse((r))
+// 	if err != nil {
+// 		return nil, nil, err
+// 	}
+// 	if !sUrl.IsAbs() || !rUrl.IsAbs() {
+// 		return nil, nil, fmt.Errorf("must have a scheme")
+// 	}
+
+// 	if sUrl.Scheme == "local" {&& rUrl.Scheme == "local" {
+// 		var err error
+// 		sender, err = makeEndpoint(modeSender, sUrl.Path)
+// 		if err == nil {
+// 			receiver, err = makeEndpoint(modeReceiver, rUrl.Path)
+// 		}
+// 		return sender, receiver, err
+// 	} else {
+
+// 	}
+// }
+
+func main() {
+
+	cancelSigs := make(chan os.Signal)
+	signal.Notify(cancelSigs, os.Interrupt, syscall.SIGTERM)
+
+	ctx := context.Background()
+	trace.WithTaskFromStackUpdateCtx(&ctx)
+
+	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(logger.NewStderrDebugLogger()))
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	go func() {
+		for {
+			select {
+			case <-cancelSigs:
+				cancel()
+			}
+		}
+	}()
+
+	flag.Parse()
+
+	if *flagStderrToFile != "" {
+		f, err := os.Create(*flagStderrToFile)
+		if err != nil {
+			panic(err)
+		}
+		syscall.Dup2(int(f.Fd()), int(os.Stderr.Fd()))
+		runtime.KeepAlive(f)
+		// enough?
+	}
+
+	if *flagStdinserver != "" {
+		if flag.NArg() != 1 {
+			panic("usage: -stdinserver MODE FS")
+		}
+		h, err := makeEndpoint(*flagStdinserver, flag.Arg(0))
+		if err != nil {
+			panic(err)
+		}
+		err = serve(ctx, h.(rpc.Handler))
+		if err != nil {
+			panic(err)
+		}
+		return
+	}
+
+	if flag.NArg() != 2 {
+		panic("usage: zync SENDER RECEIVER")
+	}
+
+	sender, err := parseFSArg(ctx, modeSender, flag.Arg(0))
+	if err != nil {
+		panic(err)
+	}
+
+	receiver, err := parseFSArg(ctx, modeReceiver, flag.Arg(1))
+	if err != nil {
+		panic(err)
+	}
+
+	pp := logic.PlannerPolicy{
+		EncryptedSend: logic.TriFromBool(true), // FIXME add flag
+		ReplicationConfig: pdu.ReplicationConfig{
+			Protection: &pdu.ReplicationConfigProtection{
+				Initial:     pdu.ReplicationGuaranteeKind_GuaranteeNothing,
+				Incremental: pdu.ReplicationGuaranteeKind_GuaranteeNothing,
+			},
+		},
+	}
+	p := logic.NewPlanner(nil, nil, sender.(logic.Sender), receiver.(logic.Receiver), pp)
+	_, wait := replication.Do(ctx, p)
+	defer wait(true)
+
+}
+
+func local() {
+
+	cancelSigs := make(chan os.Signal)
+	signal.Notify(cancelSigs, os.Interrupt, syscall.SIGTERM)
+
+	ctx := context.Background()
+	trace.WithTaskFromStackUpdateCtx(&ctx)
+
+	ctx = logging.WithLoggers(ctx, logging.SubsystemLoggersWithUniversalLogger(logger.NewStderrDebugLogger()))
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	go func() {
+		for {
+			select {
+			case <-cancelSigs:
+				cancel()
+			}
+		}
+	}()
+
+	sc := endpoint.SenderConfig{
+		FSF:     onefsfilter(os.Args[1]),
+		Encrypt: &zfs.NilBool{B: true},
+		JobID:   endpoint.MustMakeJobID("sender"),
+	}
+	s := endpoint.NewSender(sc)
+	pretty.Println(s)
+	rpath, err := zfs.NewDatasetPath(os.Args[2])
+	if err != nil {
+		panic(err)
+	}
+	r := endpoint.NewReceiver(endpoint.ReceiverConfig{
+		JobID:                      endpoint.MustMakeJobID("receiver"),
+		AppendClientIdentity:       false,
+		RootWithoutClientComponent: rpath,
+	})
+	pretty.Println(r)
+	pp := logic.PlannerPolicy{
+		EncryptedSend: logic.TriFromBool(sc.Encrypt.B),
+		ReplicationConfig: pdu.ReplicationConfig{
+			Protection: &pdu.ReplicationConfigProtection{
+				Initial:     pdu.ReplicationGuaranteeKind_GuaranteeNothing,
+				Incremental: pdu.ReplicationGuaranteeKind_GuaranteeNothing,
+			},
+		},
+	}
+	p := logic.NewPlanner(nil, nil, s, r, pp)
+	report, wait := replication.Do(ctx, p)
+	defer wait(true)
+
+	ticker := time.NewTicker(2 * time.Second)
+	for !wait(false) {
+		select {
+		case <-ticker.C:
+			// pretty.Println(report())
+		}
+	}
+	pretty.Println(report())
+}