REWORD ToString in log message

- donation links
- package overview

.circleci/config.yml

@@ -7,7 +7,9 @@ workflows:
       - build-1.12
       - build-1.13
       - build-latest
+      - test-build-in-docker
 jobs:
+
   # build-latest serves as the template
   # we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
   build-latest: &build-latest
@@ -52,10 +54,10 @@ jobs:
           paths:
             - "/usr/include/google/protobuf"
             
-      - run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev
+      - run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev gawk
       - run: ./lazy.sh devsetup
 
-      - run: make
+      - run: make zrepl-bin
       - run: make vet
       - run: make test
       - run: make lint
@@ -108,3 +110,30 @@ jobs:
     <<: *build-latest
     docker:
     - image: circleci/golang:1.13
+
+  # this job tries to mimic the build-in-docker instructions
+  # given in docs/installation.rst
+  #
+  # However, CircleCI doesn't support volume mounts, so we have to copy
+  # the source into the build-container by modifying the Dockerfile
+  test-build-in-docker:
+    description: Check that build-in-docker works
+    docker:
+      - image: circleci/golang:latest
+    environment:
+    working_directory: /go/src/github.com/zrepl/zrepl
+    steps:
+    - checkout
+    - setup_remote_docker
+    - run:
+        name: (hacky) circleci doesn't allow volume mounts, so copy src to container
+        command: echo "ADD . /src" >> build.Dockerfile
+    - run:
+        name: (hacky) commit modified Dockerfile to avoid failing git clean check in Makefile
+        command: git -c user.name='circleci' -c user.email='circleci@localhost' commit -m 'CIRCLECI modified Dockerfile with zrepl src' --author 'autoauthor <circleci@localhost>' -- build.Dockerfile
+    - run:
+        name: build the build image (build deps)
+        command: docker build -t zrepl_build -f build.Dockerfile .
+    - run:
+        name: try compiling
+        command: docker run -it zrepl_build make release

Makefile

@@ -1,5 +1,6 @@
 .PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
-.DEFAULT_GOAL := build
+.PHONY: release bins-all release-noarch
+.DEFAULT_GOAL := zrepl-bin
 
 ARTIFACTDIR := artifacts
 
@@ -12,97 +13,69 @@ ifndef _ZREPL_VERSION
         $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
     endif
 endif
+
 GO := go
+GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
+GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
+GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
+GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
+GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
+GO_ENV_VARS := GO111MODULE=on
 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
 GO_MOD_READONLY := -mod=readonly
 GO_BUILDFLAGS := $(GO_MOD_READONLY)
-GO_BUILD := GO111MODULE=on $(GO) build $(GO_BUILDFLAGS) -v -ldflags $(GO_LDFLAGS)
+GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
+GOLANGCI_LINT := golangci-lint
+ifneq ($(GOARM),)
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
+else
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
+endif
 
-# keep in sync with vet target
-RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-amd64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-arm64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-darwin-amd64
+.PHONY: printvars
+printvars:
+	@echo GOOS=$(GOOS)
+	@echo GOARCH=$(GOARCH)
+	@echo GOARM=$(GOARM)
 
-RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
-THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <($(GO) env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
 
-generate: #not part of the build, must do that manually
-	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
-	$(GO) generate $(GO_BUILDFLAGS) -x ./...
+##################### PRODUCING A RELEASE #############
+.PHONY: release wrapup-and-checksum check-git-clean sign clean
 
-format:
-	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
+release: clean
+	# no cross-platform support for target test
+	$(MAKE) test
+	$(MAKE) bins-all
+	$(MAKE) noarch
+	$(MAKE) wrapup-and-checksum
+	$(MAKE) check-git-clean
+ifeq (SIGN, 1)
+	$(make) sign
+endif
+	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
 
-lint:
-	golangci-lint run ./...
-
-build:
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
-
-test:
-	$(GO) test $(GO_BUILDFLAGS) ./...
-	# TODO compile the tests for each supported platform
-	# but `go test -c ./...` is not supported
-
-vet:
-	$(GO) vet $(GO_BUILDFLAGS) ./...
-	# for each supported platform to cover conditional compilation
-	# (keep in sync with RELEASE_BINS)
-	GOOS=freebsd	GOARCH=amd64 	$(GO) vet $(GO_BUILDFLAGS) ./...
-	GOOS=linux		GOARCH=amd64 	$(GO) vet $(GO_BUILDFLAGS) ./...
-	GOOS=linux		GOARCH=arm64 	$(GO) vet $(GO_BUILDFLAGS) ./...
-	GOOS=darwin		GOARCH=amd64 	$(GO) vet $(GO_BUILDFLAGS) ./...
-
-ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
-ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
-$(ARTIFACTDIR)/zrepl_platformtest:
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl_platformtest" ./platformtest/harness
-platformtest: $(ARTIFACTDIR)/zrepl_platformtest
-	"$(ARTIFACTDIR)/zrepl_platformtest" -poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" -imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)"
-
-$(ARTIFACTDIR):
-	mkdir -p "$@"
-
-$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
-	mkdir -p "$@"
-
-$(ARTIFACTDIR)/bash_completion: $(RELEASE_BINS)
-	artifacts/$(THIS_PLATFORM_RELEASE_BIN) bashcomp "$@"
-
-.PHONY: $(ARTIFACTDIR)/go_version.txt
-$(ARTIFACTDIR)/go_version.txt:
-	$(GO) version > $@
-
-docs: $(ARTIFACTDIR)/docs
-	make -C docs \
-		html \
-		BUILDDIR=../artifacts/docs \
-
-docs-clean:
-	make -C docs \
-		clean \
-		BUILDDIR=../artifacts/docs
-
-.PHONY: $(RELEASE_BINS)
-# TODO: two wildcards possible
-$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%: generate $(ARTIFACTDIR) vet test lint
-	STEM=$*; GOOS="$${STEM%%-*}"; GOARCH="$${STEM##*-}"; export GOOS GOARCH; \
-		$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$$GOOS-$$GOARCH"
-
-$(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
+# expects `release` target to have run before
+NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
+wrapup-and-checksum:
+	rm -f $(NOARCH_TARBALL)
 	tar --mtime='1970-01-01' --sort=name \
 		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
-		-acf $@ \
+		--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
+		--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
+		-acf $(NOARCH_TARBALL) \
 		$(ARTIFACTDIR)/docs/html \
 		$(ARTIFACTDIR)/bash_completion \
-		$(ARTIFACTDIR)/go_version.txt
-
-release: $(RELEASE_BINS) $(RELEASE_NOARCH)
+		$(ARTIFACTDIR)/go_env.txt \
+		dist \
+		config/samples
 	rm -rf "$(ARTIFACTDIR)/release"
 	mkdir -p "$(ARTIFACTDIR)/release"
-	cp $^ "$(ARTIFACTDIR)/release"
+	cp -l $(ARTIFACTDIR)/zrepl-* \
+		$(ARTIFACTDIR)/platformtest-* \
+		"$(ARTIFACTDIR)/release"
 	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
+
+check-git-clean:
 	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
 	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
         echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
@@ -115,5 +88,101 @@ release: $(RELEASE_BINS) $(RELEASE_NOARCH)
 		fi; \
 	fi;
 
+sign:
+	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+		--armor \
+		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
+
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"
+
+##################### BINARIES #####################
+.PHONY: bins-all lint test vet zrepl-bin platformtest-bin
+
+BINS_ALL_TARGETS := zrepl-bin platformtest-bin vet lint
+GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
+bins-all:
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
+ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
+else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
+	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
+else
+	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
+endif
+
+lint:
+	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
+test:
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) ./...
+vet:
+	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
+
+zrepl-bin:
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
+
+platformtest-bin:
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" ./platformtest/harness
+
+##################### DEV TARGETS #####################
+# not part of the build, must do that manually
+.PHONY: generate format platformtest
+
+generate:
+	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
+	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
+
+format:
+	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
+
+ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
+ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
+ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
+ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
+# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
+ZREPL_PLATFORMTEST_ARGS := 
+platformtest: # do not track dependency on platformtest-bin to allow build of platformtest outside of test VM
+	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
+	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
+	"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
+		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
+		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
+		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
+		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
+		$(ZREPL_PLATFORMTEST_ARGS)
+
+##################### NOARCH #####################
+.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
+
+
+$(ARTIFACTDIR):
+	mkdir -p "$@"
+$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
+	mkdir -p "$@"
+
+noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_env.txt docs
+	# pass
+
+$(ARTIFACTDIR)/bash_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) bashcomp "$@"
+
+$(ARTIFACTDIR)/go_env.txt:
+	$(GO_ENV_VARS) $(GO) env > $@
+
+docs: $(ARTIFACTDIR)/docs
+	make -C docs \
+		html \
+		BUILDDIR=../artifacts/docs \
+
+docs-clean:
+	make -C docs \
+		clean \
+		BUILDDIR=../artifacts/docs

README.md

@@ -1,8 +1,9 @@
 [![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
 [![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
 [![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
-[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
+[![Donate via Patreon](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow)](https://www.patreon.com/zrepl)
 [![Donate via Liberapay](https://img.shields.io/liberapay/receives/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
+[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
 [![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
 
 # zrepl
@@ -23,6 +24,7 @@ zrepl is a one-stop ZFS backup & replication solution.
    If so, think of an expressive configuration example.
 2. Think of at least one use case that generalizes from your concrete application.
 3. Open an issue on GitHub with example conf & use case attached.
+4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
 
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
@@ -46,11 +48,8 @@ zrepl is written in [Go](https://golang.org) and uses [Go modules](https://githu
 The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
 
 To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
-
-### Overall Architecture
-
-The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
-Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
+`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
+If you just want to install the Go dependencies, run `./lazy.sh godep`.
 
 ### Project Structure
 
@@ -62,14 +61,15 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   └── samples
 ├── daemon                  # the implementation of `zrepl daemon` subcommand
 │   ├── filters
+│   ├── hooks               # snapshot hooks
 │   ├── job                 # job implementations
 │   ├── logging             # logging outlets + formatters
 │   ├── nethelpers
 │   ├── prometheus
 │   ├── pruner              # pruner implementation
 │   ├── snapper             # snapshotter implementation
-├── docs                    # sphinx-based documentation
 ├── dist                    # supplemental material for users & package maintainers
+├── docs                    # sphinx-based documentation
 │   ├── **/*.rst            # documentation in reStructuredText
 │   ├── sphinxconf
 │   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
@@ -78,6 +78,7 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   └── public_git          # checkout of zrepl.github.io managed by above shell script
 ├── endpoint                # implementation of replication endpoints (=> package replication)
 ├── logger                  # our own logger package
+├── platformtest            # test suite for our zfs abstractions (error classification, etc)
 ├── pruning                 # pruning rules (the logic, not the actual execution)
 │   └── retentiongrid
 ├── replication
@@ -92,14 +93,13 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   ├── transportmux        # TCP connecter and listener used to split control & data traffic
 │   └── versionhandshake    # replication protocol version handshake perfomed on newly established connections
 ├── tlsconf                 # abstraction for Go TLS server + client config
-├── transport               # transports implementation
+├── transport               # transport implementations
 │   ├── fromconfig
 │   ├── local
 │   ├── ssh
 │   ├── tcp
 │   └── tls
 ├── util
-├── vendor                  # managed by dep
 ├── version                 # abstraction for versions (filled during build by Makefile)
 └── zfs                     # zfs(8) wrappers
 ```
@@ -134,3 +134,15 @@ There will not be a big refactoring (an attempt was made, but it's destroying to
 
 However, new contributions & patches should fix naming without further notice in the commit message.
 
+### RPC debugging
+
+Optionally, there are various RPC-related environment varibles, that if set to something != `""` will produce additional debug output on stderr:
+
+https://github.com/zrepl/zrepl/blob/master/rpc/rpc_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/dataconn_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/stream/stream_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go#L11
+

build.Dockerfile

@@ -2,7 +2,8 @@ FROM golang:latest
 
 RUN apt-get update && apt-get install -y \
     python3-pip \
-    unzip
+    unzip \
+    gawk
 
 ADD build.installprotoc.bash ./
 RUN bash build.installprotoc.bash
@@ -22,7 +23,7 @@ RUN mkdir -p /.cache && chmod -R 0777 /.cache
 WORKDIR /src
 
 # Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
-ADD go.mod go.sum ./
+ADD build/ /tmp/build
 RUN /tmp/lazy.sh godep
 
 RUN chmod -R 0777 /go

build/build.go

@@ -1,23 +0,0 @@
-//+build windows
-
-// This package cannot actually be built, and since the Travis
-// tests run go test ./..., we have to avoid a build attempt.
-// Windows is not supported atm, so this works ¯\_(ツ)_/¯
-
-// This package is a pseudo-user of build-time dependencies for zrepl,
-// mostly for various code-generation tools.
-//
-// The imports are necessary to satisfy go dep
-package main
-
-import (
-	"fmt"
-
-	_ "github.com/alvaroloes/enumer"
-	_ "github.com/golang/protobuf/protoc-gen-go"
-	_ "golang.org/x/tools/cmd/stringer"
-)
-
-func main() {
-	fmt.Println("just a placeholder")
-}

build/go.mod

@@ -0,0 +1,51 @@
+module github.com/zrepl/zrepl/build
+
+go 1.12
+
+require (
+	github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
+	github.com/alvaroloes/enumer v1.1.1
+	github.com/fatih/color v1.7.0 // indirect
+	github.com/go-critic/go-critic v0.3.4 // indirect
+	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/golang/mock v1.2.0 // indirect
+	github.com/golang/protobuf v1.2.0
+	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
+	github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
+	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
+	github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
+	github.com/golangci/golangci-lint v1.17.1
+	github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
+	github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
+	github.com/golangci/misspell v0.3.4 // indirect
+	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/google/go-cmp v0.3.0 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/viper v1.3.2 // indirect
+	github.com/stretchr/testify v1.4.0 // indirect
+	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
+	golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
+	golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
+	mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
+	sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
+)
+
+// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
+replace (
+	github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
+	github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
+	github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
+	github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
+	github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
+	github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
+	github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
+	github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
+	github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
+	golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
+	mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
+)

build/go.sum

@@ -0,0 +1,280 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
+github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
+github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
+github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
+github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
+github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
+github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
+github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
+github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
+github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
+github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
+github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
+github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
+github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
+github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
+github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
+github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
+github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
+github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
+github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
+github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
+github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
+github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
+golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
+mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
+mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
+sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
+sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

build/tools.go

@@ -3,9 +3,9 @@
 package main
 
 import (
-	_ "golang.org/x/tools/cmd/stringer"
-	_ "github.com/golang/protobuf/protoc-gen-go"
 	_ "github.com/alvaroloes/enumer"
-	_ "golang.org/x/tools/cmd/goimports"
+	_ "github.com/golang/protobuf/protoc-gen-go"
 	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "golang.org/x/tools/cmd/goimports"
+	_ "golang.org/x/tools/cmd/stringer"
 )

client/holds.go

@@ -0,0 +1,87 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var (
+	HoldsCmd = &cli.Subcommand{
+		Use:   "holds",
+		Short: "manage holds & step bookmarks",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return holdsList
+		},
+	}
+)
+
+var holdsList = []*cli.Subcommand{
+	&cli.Subcommand{
+		Use:             "list [FSFILTER]",
+		Run:             doHoldsList,
+		NoRequireConfig: true,
+		Short: `
+FSFILTER SYNTAX:
+representation of a 'filesystems' filter statement on the command line
+		`,
+	},
+}
+
+func fsfilterFromCliArg(arg string) (zfs.DatasetFilter, error) {
+	mappings := strings.Split(arg, ",")
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return nil, thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return nil, fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	return f.AsFilter(), nil
+}
+
+func doHoldsList(sc *cli.Subcommand, args []string) error {
+	var err error
+	ctx := context.Background()
+
+	if len(args) > 1 {
+		return errors.New("this subcommand takes at most one argument")
+	}
+
+	var filter zfs.DatasetFilter
+	if len(args) == 0 {
+		filter = zfs.NoFilter()
+	} else {
+		filter, err = fsfilterFromCliArg(args[0])
+		if err != nil {
+			return errors.Wrap(err, "cannot parse filesystem filter args")
+		}
+	}
+
+	listing, err := endpoint.ListZFSHoldsAndBookmarks(ctx, filter)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("  ", "  ")
+	if err := enc.Encode(listing); err != nil {
+		panic(err)
+	}
+
+	return nil
+}

client/migrate.go

@@ -4,9 +4,13 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/fatih/color"
+	"github.com/kr/pretty"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/zfs"
 
 	"github.com/zrepl/zrepl/cli"
@@ -31,6 +35,13 @@ var migrations = []*cli.Subcommand{
 			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
 		},
 	},
+	&cli.Subcommand{
+		Use: "replication-cursor:v1-v2",
+		Run: doMigrateReplicationCursor,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
+		},
+	},
 }
 
 var migratePlaceholder0_1Args struct {
@@ -102,3 +113,161 @@ func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
 
 	return nil
 }
+
+var migrateReplicationCursorArgs struct {
+	dryRun bool
+}
+
+var bold = color.New(color.Bold)
+var succ = color.New(color.FgGreen)
+var fail = color.New(color.FgRed)
+
+var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
+
+func doMigrateReplicationCursor(sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+	jobs, err := job.JobsFromConfig(cfg)
+	if err != nil {
+		fmt.Printf("cannot parse config:\n%s\n\n", err)
+		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
+		return fmt.Errorf("exiting migration after error")
+	}
+
+	ctx := context.Background()
+
+	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
+	for i, j := range cfg.Jobs {
+		if jobs[i].Name() != j.Name() {
+			panic("implementation error")
+		}
+		switch j.Ret.(type) {
+		case *config.PushJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		case *config.SourceJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+	}
+
+	// scan all filesystems for v1 replication cursors
+
+	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "list filesystems")
+	}
+
+	var hadError bool
+	for _, fs := range fss {
+
+		bold.Printf("INSPECT FILESYTEM %q\n", fs.ToString())
+
+		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
+		if err == migrateReplicationCursorSkipSentinel {
+			bold.Printf("FILESYSTEM SKIPPED\n")
+		} else if err != nil {
+			hadError = true
+			fail.Printf("MIGRATION FAILED: %s\n", err)
+		} else {
+			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
+		}
+	}
+
+	if hadError {
+		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
+		return errors.Errorf("")
+	}
+	return nil
+}
+
+func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
+
+	var owningJob job.Job = nil
+	for _, job := range v1CursorJobs {
+		conf := job.SenderConfig()
+		if conf == nil {
+			continue
+		}
+		pass, err := conf.FSF.Filter(fs)
+		if err != nil {
+			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
+		}
+		if !pass {
+			continue
+		}
+		if owningJob != nil {
+			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
+		}
+		owningJob = job
+	}
+	if owningJob == nil {
+		fmt.Printf("no job's Filesystems filter matches\n")
+		return migrateReplicationCursorSkipSentinel
+	}
+	fmt.Printf("identified owning job %q\n", owningJob.Name())
+
+	versions, err := zfs.ZFSListFilesystemVersions(fs, nil)
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
+	}
+
+	var oldCursor *zfs.FilesystemVersion
+	for i, fsv := range versions {
+		if fsv.Type != zfs.Bookmark {
+			continue
+		}
+
+		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
+		if err != endpoint.ErrV1ReplicationCursor {
+			continue
+		}
+
+		if oldCursor != nil {
+			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
+			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
+		}
+
+		oldCursor = &versions[i]
+
+	}
+
+	if oldCursor == nil {
+		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
+		return migrateReplicationCursorSkipSentinel
+	}
+
+	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
+
+	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
+	if err != nil {
+		return errors.Wrapf(err, "get most recent v2 replication cursor")
+	}
+
+	if mostRecentNew == nil {
+		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
+	}
+
+	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
+
+	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
+		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
+	}
+
+	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
+	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
+
+	if migrateReplicationCursorArgs.dryRun {
+		succ.Printf("DRY RUN\n")
+	} else {
+		if err := zfs.ZFSDestroyFilesystemVersion(fs, oldCursor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

client/status.go

@@ -461,12 +461,23 @@ func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgress
 	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
 		// Draw global progress bar
 		// Progress: [---------------]
-		expected, replicated := latest.BytesSum()
+		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
 		rate, changeCount := history.Update(replicated)
+		eta := time.Duration(0)
+		if rate > 0 {
+			eta = time.Duration((expected-replicated)/rate) * time.Second
+		}
 		t.write("Progress: ")
 		t.drawBar(50, replicated, expected, changeCount)
 		t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
+		if eta != 0 {
+			t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+		}
 		t.newline()
+		if containsInvalidSizeEstimates {
+			t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
+			t.newline()
+		}
 
 		var maxFSLen int
 		for _, fs := range latest.Filesystems {
@@ -707,11 +718,20 @@ func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
 
 func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
 
-	expected, replicated := rep.BytesSum()
-	status := fmt.Sprintf("%s (step %d/%d, %s/%s)",
+	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
+	sizeEstimationImpreciseNotice := ""
+	if containsInvalidSizeEstimates {
+		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
+	}
+	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
+		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
+	}
+
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
 		strings.ToUpper(string(rep.State)),
 		rep.CurrentStep, len(rep.Steps),
 		ByteCountBinary(replicated), ByteCountBinary(expected),
+		sizeEstimationImpreciseNotice,
 	)
 
 	activeIndicator := " "
@@ -731,8 +751,17 @@ func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, m
 			if nextStep.IsIncremental() {
 				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
 			} else {
-				next = fmt.Sprintf("next: %s (full)", nextStep.Info.To)
+				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
 			}
+			attribs := []string{}
+
+			if nextStep.Info.Resumed {
+				attribs = append(attribs, "resumed")
+			}
+
+			attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
+
+			next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
 		} else {
 			next = "" // individual FSes may still be in planning state
 		}
@@ -755,3 +784,27 @@ func ByteCountBinary(b int64) string {
 	}
 	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
 }
+
+func humanizeDuration(duration time.Duration) string {
+	days := int64(duration.Hours() / 24)
+	hours := int64(math.Mod(duration.Hours(), 24))
+	minutes := int64(math.Mod(duration.Minutes(), 60))
+	seconds := int64(math.Mod(duration.Seconds(), 60))
+
+	var parts []string
+
+	force := false
+	chunks := []int64{days, hours, minutes, seconds}
+	for i, chunk := range chunks {
+		if force || chunk > 0 {
+			padding := 0
+			if force {
+				padding = 2
+			}
+			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
+			force = true
+		}
+	}
+
+	return strings.Join(parts, " ")
+}

client/testcmd.go

@@ -1,7 +1,10 @@
 package client
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
@@ -15,7 +18,7 @@ import (
 var TestCmd = &cli.Subcommand{
 	Use: "test",
 	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{testFilter, testPlaceholder}
+		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
 	},
 }
 
@@ -57,6 +60,8 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 		confFilter = j.Filesystems
 	case *config.PushJob:
 		confFilter = j.Filesystems
+	case *config.SnapJob:
+		confFilter = j.Filesystems
 	default:
 		return fmt.Errorf("job type %T does not have filesystems filter", j)
 	}
@@ -176,3 +181,32 @@ func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
 	}
 	return nil
 }
+
+var testDecodeResumeTokenArgs struct {
+	token string
+}
+
+var testDecodeResumeToken = &cli.Subcommand{
+	Use:   "decoderesumetoken --token TOKEN",
+	Short: "decode resume token",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
+	},
+	Run: runTestDecodeResumeTokenCmd,
+}
+
+func runTestDecodeResumeTokenCmd(subcommand *cli.Subcommand, args []string) error {
+	if testDecodeResumeTokenArgs.token == "" {
+		return fmt.Errorf("token argument must be specified")
+	}
+	token, err := zfs.ParseResumeToken(context.Background(), testDecodeResumeTokenArgs.token)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+	if err := enc.Encode(&token); err != nil {
+		panic(err)
+	}
+	return nil
+}

config/config.go

@@ -75,16 +75,42 @@ type SnapJob struct {
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
 }
 
+type SendOptions struct {
+	Encrypted bool `yaml:"encrypted"`
+}
+
+var _ yaml.Defaulter = (*SendOptions)(nil)
+
+func (l *SendOptions) SetDefault() {
+	*l = SendOptions{Encrypted: false}
+}
+
+type RecvOptions struct {
+	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
+	// Encrypted bool `yaml:"may_encrypted"`
+
+	// Future:
+	// Reencrypt bool `yaml:"reencrypt"`
+}
+
+var _ yaml.Defaulter = (*RecvOptions)(nil)
+
+func (l *RecvOptions) SetDefault() {
+	*l = RecvOptions{}
+}
+
 type PushJob struct {
 	ActiveJob    `yaml:",inline"`
 	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
 }
 
 type PullJob struct {
 	ActiveJob `yaml:",inline"`
 	RootFS    string                   `yaml:"root_fs"`
 	Interval  PositiveDurationOrManual `yaml:"interval"`
+	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
 }
 
 type PositiveDurationOrManual struct {
@@ -121,12 +147,14 @@ func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error
 type SinkJob struct {
 	PassiveJob `yaml:",inline"`
 	RootFS     string       `yaml:"root_fs"`
+	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
 }
 
 type SourceJob struct {
 	PassiveJob   `yaml:",inline"`
 	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
 }
 
 type FilesystemsFilter map[string]bool
@@ -247,12 +275,14 @@ type ServeCommon struct {
 type TCPServe struct {
 	ServeCommon    `yaml:",inline"`
 	Listen         string            `yaml:"listen,hostport"`
+	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
 	Clients        map[string]string `yaml:"clients"`
 }
 
 type TLSServe struct {
 	ServeCommon      `yaml:",inline"`
 	Listen           string        `yaml:"listen,hostport"`
+	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
 	Ca               string        `yaml:"ca"`
 	Cert             string        `yaml:"cert"`
 	Key              string        `yaml:"key"`
@@ -333,6 +363,7 @@ type MonitoringEnum struct {
 type PrometheusMonitoring struct {
 	Type           string `yaml:"type"`
 	Listen         string `yaml:"listen,hostport"`
+	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
 }
 
 type SyslogFacility syslog.Priority

config/config_jobid_test.go

@@ -0,0 +1 @@
+package config

config/config_send_test.go

@@ -0,0 +1,74 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSendOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	encrypted_false := `
+  send:
+    encrypted: false
+`
+
+	encrypted_true := `
+  send:
+    encrypted: true
+`
+
+	encrypted_unspecified := `
+  send: {}
+`
+
+	send_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("encrypted_false", func(t *testing.T) {
+		c = testValidConfig(t, fill(encrypted_false))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+	t.Run("encrypted_true", func(t *testing.T) {
+		c = testValidConfig(t, fill(encrypted_true))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("encrypted_unspecified", func(t *testing.T) {
+		c, err := testConfig(t, fill(encrypted_unspecified))
+		assert.Error(t, err)
+		assert.Nil(t, c)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c, err := testConfig(t, fill(send_not_specified))
+		assert.NoError(t, err)
+		assert.NotNil(t, c)
+	})
+
+}

config/samples/pull_ssh.yml

@@ -9,7 +9,7 @@ jobs:
     port: 22
     identity_file: /etc/zrepl/ssh/identity
     options: # optional, default [], `-o` arguments passed to ssh
-    - "Compression=on"
+    - "Compression=yes"
   root_fs: "pool2/backup_servers"
   interval: 10m
   pruning:

config/samples/push.yml

@@ -10,6 +10,8 @@ jobs:
       address: "backup-server.foo.bar:8888"
     snapshotting:
       type: manual
+    send:
+      encrypted: false
     pruning:
       keep_sender:
         - type: not_replicated

daemon/control.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/zrepl/zrepl/daemon/job"
 	"github.com/zrepl/zrepl/daemon/nethelpers"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/version"
@@ -44,6 +45,8 @@ func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInt
 
 func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
 
+func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 var promControl struct {
 	requestBegin    *prometheus.CounterVec
 	requestFinished *prometheus.HistogramVec

daemon/daemon.go

@@ -76,7 +76,7 @@ func Run(conf *config.Config) error {
 			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
 		}
 		if err != nil {
-			return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
+			return errors.Wrapf(err, "cannot build monitoring job #%d", i)
 		}
 		jobs.start(ctx, job, true)
 	}

daemon/filters/fsmapfilter_test.go

@@ -1,11 +1,11 @@
 package filters
 
 import (
+	"testing"
+
 	"github.com/zrepl/zrepl/zfs"
 )
 
-import "testing"
-
 func TestDatasetMapFilter(t *testing.T) {
 
 	type testCase struct {

daemon/job/active.go

@@ -2,6 +2,7 @@ package job
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
@@ -28,7 +29,7 @@ import (
 
 type ActiveSide struct {
 	mode      activeMode
-	name      string
+	name      endpoint.JobID
 	connecter transport.Connecter
 
 	prunerFactory *pruner.PrunerFactory
@@ -82,6 +83,7 @@ type activeMode interface {
 	DisconnectEndpoints()
 	SenderReceiver() (logic.Sender, logic.Receiver)
 	Type() Type
+	PlannerPolicy() logic.PlannerPolicy
 	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
 	SnapperReport() *snapper.Report
 	ResetConnectBackoff()
@@ -91,7 +93,8 @@ type modePush struct {
 	setupMtx      sync.Mutex
 	sender        *endpoint.Sender
 	receiver      *rpc.Client
-	fsfilter endpoint.FSFilter
+	senderConfig  *endpoint.SenderConfig
+	plannerPolicy *logic.PlannerPolicy
 	snapper       *snapper.PeriodicOrManual
 }
 
@@ -101,7 +104,7 @@ func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Con
 	if m.receiver != nil || m.sender != nil {
 		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
 	}
-	m.sender = endpoint.NewSender(m.fsfilter)
+	m.sender = endpoint.NewSender(*m.senderConfig)
 	m.receiver = rpc.NewClient(connecter, loggers)
 }
 
@@ -121,6 +124,8 @@ func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
 
 func (m *modePush) Type() Type { return TypePush }
 
+func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
 func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
 	m.snapper.Run(ctx, wakeUpCommon)
 }
@@ -137,13 +142,22 @@ func (m *modePush) ResetConnectBackoff() {
 	}
 }
 
-func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
+func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
 	m := &modePush{}
+
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannnot build filesystem filter")
 	}
-	m.fsfilter = fsf
+
+	m.senderConfig = &endpoint.SenderConfig{
+		FSF:     fsf,
+		Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
+		JobID:   jobID,
+	}
+	m.plannerPolicy = &logic.PlannerPolicy{
+		EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
+	}
 
 	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
@@ -155,8 +169,10 @@ func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error)
 type modePull struct {
 	setupMtx       sync.Mutex
 	receiver       *endpoint.Receiver
+	receiverConfig endpoint.ReceiverConfig
 	sender         *rpc.Client
 	rootFS         *zfs.DatasetPath
+	plannerPolicy  *logic.PlannerPolicy
 	interval       config.PositiveDurationOrManual
 }
 
@@ -166,7 +182,7 @@ func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Con
 	if m.receiver != nil || m.sender != nil {
 		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
 	}
-	m.receiver = endpoint.NewReceiver(m.rootFS, false)
+	m.receiver = endpoint.NewReceiver(m.receiverConfig)
 	m.sender = rpc.NewClient(connecter, loggers)
 }
 
@@ -186,6 +202,8 @@ func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
 
 func (*modePull) Type() Type { return TypePull }
 
+func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
 func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
 	if m.interval.Manual {
 		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
@@ -223,7 +241,7 @@ func (m *modePull) ResetConnectBackoff() {
 	}
 }
 
-func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
+func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.JobID) (m *modePull, err error) {
 	m = &modePull{}
 	m.interval = in.Interval
 
@@ -235,26 +253,56 @@ func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err
 		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
 	}
 
+	m.plannerPolicy = &logic.PlannerPolicy{
+		EncryptedSend: logic.DontCare,
+	}
+
+	m.receiverConfig = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: m.rootFS,
+		AppendClientIdentity:       false, // !
+		UpdateLastReceivedHold:     true,
+	}
+	if err := m.receiverConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver config")
+	}
+
 	return m, nil
 }
 
-func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {
+func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (j *ActiveSide, err error) {
+
+	j = &ActiveSide{}
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.PushJob:
+		j.mode, err = modePushFromConfig(g, v, j.name) // shadow
+	case *config.PullJob:
+		j.mode, err = modePullFromConfig(g, v, j.name) // shadow
+	default:
+		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
+	}
+	if err != nil {
+		return nil, err // no wrapping required
+	}
 
-	j = &ActiveSide{mode: mode}
-	j.name = in.Name
 	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "replication",
 		Name:        "state_time",
 		Help:        "seconds spent during replication",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"state"})
 	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "replication",
 		Name:        "bytes_replicated",
 		Help:        "number of bytes replicated from sender to receiver per filesystem",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"filesystem"})
 
 	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
@@ -267,7 +315,7 @@ func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *Act
 		Subsystem:   "pruning",
 		Name:        "time",
 		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"prune_side"})
 	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
 	if err != nil {
@@ -283,7 +331,7 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
 	registerer.MustRegister(j.promBytesReplicated)
 }
 
-func (j *ActiveSide) Name() string { return j.name }
+func (j *ActiveSide) Name() string { return j.name.String() }
 
 type ActiveSideStatus struct {
 	Replication                    *report.Report
@@ -318,6 +366,15 @@ func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
 	return pull.rootFS.Copy(), true
 }
 
+func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
+	push, ok := j.mode.(*modePush)
+	if !ok {
+		_ = j.mode.(*modePull) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return push.senderConfig
+}
+
 func (j *ActiveSide) Run(ctx context.Context) {
 	log := GetLogger(ctx)
 	ctx = logging.WithSubsystemLoggers(ctx, log)
@@ -383,7 +440,7 @@ func (j *ActiveSide) do(ctx context.Context) {
 			*tasks = activeSideTasks{}
 			tasks.replicationCancel = repCancel
 			tasks.replicationReport, repWait = replication.Do(
-				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver),
+				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
 			)
 			tasks.state = ActiveSideReplicating
 		})

daemon/job/build_jobs.go

@@ -48,20 +48,12 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
 	// FIXME prettify this
 	switch v := in.Ret.(type) {
 	case *config.SinkJob:
-		m, err := modeSinkFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.SourceJob:
-		m, err := modeSourceFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
@@ -71,20 +63,12 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.PushJob:
-		m, err := modePushFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
+		j, err = activeSide(c, &v.ActiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.PullJob:
-		m, err := modePullFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
+		j, err = activeSide(c, &v.ActiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
@@ -104,6 +88,11 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
 	sort.Slice(rfss, func(i, j int) bool {
 		return strings.Compare(rfss[i], rfss[j]) == -1
 	})
+	// add tailing slash because of hierarchy-simulation
+	// rootfs/ is not root of rootfs2/
+	for i := 0; i < len(rfss); i++ {
+		rfss[i] += "/"
+	}
 	// idea:
 	//   no path in rfss must be prefix of another
 	//

daemon/job/build_jobs_test.go

@@ -1,9 +1,13 @@
 package job
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/config"
 )
 
 func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
@@ -18,6 +22,7 @@ func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
 		{false, []string{"a"}},
 		{false, []string{"some/path"}},
 		{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
+		{false, []string{"zroot/foo", "zroot/foobar"}},
 		{true, []string{"zroot/b", "zroot/b"}},
 		{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
 		{false, []string{"a/x", "b/x"}},
@@ -40,3 +45,66 @@ func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
 		}
 	}
 }
+
+func TestJobIDErrorHandling(t *testing.T) {
+	tmpl := `
+jobs:
+- name: %s
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type Case struct {
+		jobName string
+		valid   bool
+	}
+	cases := []Case{
+		{"validjobname", true},
+		{"valid with spaces", true},
+		{"invalid\twith\ttabs", false},
+		{"invalid#withdelimiter", false},
+		{"invalid@withdelimiter", false},
+		{"withnewline\\nmiddle", false},
+		{"withnewline\\n", false},
+		{"withslash/", false},
+		{"withslash/inthemiddle", false},
+		{"/", false},
+	}
+
+	for i := range cases {
+		t.Run(cases[i].jobName, func(t *testing.T) {
+			c := cases[i]
+
+			conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
+			require.NoError(t, err, "not expecting yaml-config to know about job ids")
+			require.NotNil(t, conf)
+			jobs, err := JobsFromConfig(conf)
+
+			if c.valid {
+				assert.NoError(t, err)
+				require.Len(t, jobs, 1)
+				assert.Equal(t, c.jobName, jobs[0].Name())
+			} else {
+				t.Logf("error: %s", err)
+				assert.Error(t, err)
+				assert.Nil(t, jobs)
+			}
+
+		})
+	}
+
+}

daemon/job/job.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/zfs"
 )
@@ -38,6 +39,7 @@ type Job interface {
 	// Jobs that return a subtree of the dataset hierarchy
 	// must return the root of that subtree as rfs and ok = true
 	OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
+	SenderConfig() *endpoint.SenderConfig
 }
 
 type Type string

daemon/job/passive.go

@@ -20,7 +20,7 @@ import (
 
 type PassiveSide struct {
 	mode   passiveMode
-	name   string
+	name   endpoint.JobID
 	listen transport.AuthenticatedListenerFactory
 }
 
@@ -32,43 +32,56 @@ type passiveMode interface {
 }
 
 type modeSink struct {
-	rootDataset *zfs.DatasetPath
+	receiverConfig endpoint.ReceiverConfig
 }
 
 func (m *modeSink) Type() Type { return TypeSink }
 
 func (m *modeSink) Handler() rpc.Handler {
-	return endpoint.NewReceiver(m.rootDataset, true)
+	return endpoint.NewReceiver(m.receiverConfig)
 }
 
 func (m *modeSink) RunPeriodic(_ context.Context)  {}
 func (m *modeSink) SnapperReport() *snapper.Report { return nil }
 
-func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) {
+func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
 	m = &modeSink{}
-	m.rootDataset, err = zfs.NewDatasetPath(in.RootFS)
+
+	rootDataset, err := zfs.NewDatasetPath(in.RootFS)
 	if err != nil {
 		return nil, errors.New("root dataset is not a valid zfs filesystem path")
 	}
-	if m.rootDataset.Length() <= 0 {
-		return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver
+
+	m.receiverConfig = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: rootDataset,
+		AppendClientIdentity:       true, // !
+		UpdateLastReceivedHold:     true,
 	}
+	if err := m.receiverConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver config")
+	}
+
 	return m, nil
 }
 
 type modeSource struct {
-	fsfilter zfs.DatasetFilter
+	senderConfig *endpoint.SenderConfig
 	snapper      *snapper.PeriodicOrManual
 }
 
-func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
+func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
 	// FIXME exact dedup of modePush
 	m = &modeSource{}
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannnot build filesystem filter")
 	}
-	m.fsfilter = fsf
+	m.senderConfig = &endpoint.SenderConfig{
+		FSF:     fsf,
+		Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
+		JobID:   jobID,
+	}
 
 	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
@@ -80,7 +93,7 @@ func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource
 func (m *modeSource) Type() Type { return TypeSource }
 
 func (m *modeSource) Handler() rpc.Handler {
-	return endpoint.NewSender(m.fsfilter)
+	return endpoint.NewSender(*m.senderConfig)
 }
 
 func (m *modeSource) RunPeriodic(ctx context.Context) {
@@ -91,9 +104,25 @@ func (m *modeSource) SnapperReport() *snapper.Report {
 	return m.snapper.Report()
 }
 
-func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
+func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}) (s *PassiveSide, err error) {
+
+	s = &PassiveSide{}
+
+	s.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.SinkJob:
+		s.mode, err = modeSinkFromConfig(g, v, s.name) // shadow
+	case *config.SourceJob:
+		s.mode, err = modeSourceFromConfig(g, v, s.name) // shadow
+	}
+	if err != nil {
+		return nil, err // no wrapping necessary
+	}
 
-	s = &PassiveSide{mode: mode, name: in.Name}
 	if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
 		return nil, errors.Wrap(err, "cannot build listener factory")
 	}
@@ -101,7 +130,7 @@ func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passive
 	return s, nil
 }
 
-func (j *PassiveSide) Name() string { return j.name }
+func (j *PassiveSide) Name() string { return j.name.String() }
 
 type PassiveStatus struct {
 	Snapper *snapper.Report
@@ -120,7 +149,16 @@ func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
 		_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
 		return nil, false
 	}
-	return sink.rootDataset.Copy(), true
+	return sink.receiverConfig.RootWithoutClientComponent.Copy(), true
+}
+
+func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
+	source, ok := j.mode.(*modeSource)
+	if !ok {
+		_ = j.mode.(*modeSink) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return source.senderConfig
 }
 
 func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}

daemon/job/snapjob.go

@@ -2,7 +2,6 @@ package job
 
 import (
 	"context"
-	"fmt"
 	"sort"
 
 	"github.com/pkg/errors"
@@ -20,7 +19,7 @@ import (
 )
 
 type SnapJob struct {
-	name     string
+	name     endpoint.JobID
 	fsfilter zfs.DatasetFilter
 	snapper  *snapper.PeriodicOrManual
 
@@ -31,7 +30,7 @@ type SnapJob struct {
 	pruner *pruner.Pruner
 }
 
-func (j *SnapJob) Name() string { return j.name }
+func (j *SnapJob) Name() string { return j.name.String() }
 
 func (j *SnapJob) Type() Type { return TypeSnap }
 
@@ -39,20 +38,23 @@ func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err er
 	j = &SnapJob{}
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
 	}
 	j.fsfilter = fsf
 
 	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
 	}
-	j.name = in.Name
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
 	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "pruning",
 		Name:        "time",
 		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"prune_side"})
 	j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
 	if err != nil {
@@ -84,6 +86,8 @@ func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
 	return nil, false
 }
 
+func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 func (j *SnapJob) Run(ctx context.Context) {
 	log := GetLogger(ctx)
 	ctx = logging.WithSubsystemLoggers(ctx, log)
@@ -133,9 +137,6 @@ type alwaysUpToDateReplicationCursorHistory struct {
 var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
 
 func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
-	if req.GetGet() == nil {
-		return nil, fmt.Errorf("unsupported ReplicationCursor request: SnapJob only supports GETting a (faked) cursor")
-	}
 	fsvReq := &pdu.ListFilesystemVersionsReq{
 		Filesystem: req.GetFilesystem(),
 	}
@@ -162,7 +163,12 @@ func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Cont
 func (j *SnapJob) doPrune(ctx context.Context) {
 	log := GetLogger(ctx)
 	ctx = logging.WithSubsystemLoggers(ctx, log)
-	sender := endpoint.NewSender(j.fsfilter)
+	sender := endpoint.NewSender(endpoint.SenderConfig{
+		JobID: j.name,
+		FSF:   j.fsfilter,
+		// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
+		Encrypt: &zfs.NilBool{B: true},
+	})
 	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
 	log.Info("start pruning")
 	j.pruner.Prune()

daemon/prometheus.go

@@ -10,20 +10,23 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
+	"github.com/zrepl/zrepl/util/tcpsock"
 	"github.com/zrepl/zrepl/zfs"
 )
 
 type prometheusJob struct {
 	listen   string
+	freeBind bool
 }
 
 func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
 	if _, _, err := net.SplitHostPort(in.Listen); err != nil {
 		return nil, err
 	}
-	return &prometheusJob{in.Listen}, nil
+	return &prometheusJob{in.Listen, in.ListenFreeBind}, nil
 }
 
 var prom struct {
@@ -46,6 +49,8 @@ func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.Type
 
 func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
 
+func (j *prometheusJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
 
 func (j *prometheusJob) Run(ctx context.Context) {
@@ -60,9 +65,10 @@ func (j *prometheusJob) Run(ctx context.Context) {
 
 	log := job.GetLogger(ctx)
 
-	l, err := net.Listen("tcp", j.listen)
+	l, err := tcpsock.Listen(j.listen, j.freeBind)
 	if err != nil {
 		log.WithError(err).Error("cannot listen")
+		return
 	}
 	go func() {
 		<-ctx.Done()

daemon/pruner/pruner.go

@@ -412,9 +412,6 @@ tfss_loop:
 
 		rcReq := &pdu.ReplicationCursorReq{
 			Filesystem: tfs.Path,
-			Op: &pdu.ReplicationCursorReq_Get{
-				Get: &pdu.ReplicationCursorReq_GetOp{},
-			},
 		}
 		rc, err := receiver.ReplicationCursor(ctx, rcReq)
 		if err != nil {

daemon/snapper/snapper.go

@@ -13,6 +13,7 @@ import (
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/hooks"
 	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -400,9 +401,19 @@ func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.Dat
 	return zfs.ZFSListMapping(ctx, mf)
 }
 
+var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
+
+// see docs/snapshotting.rst
 func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
+
+	const (
+		prioHasVersions int = iota
+		prioNoVersions
+	)
+
 	type snapTime struct {
 		ds   *zfs.DatasetPath
+		prio int // lower is higher
 		time time.Time
 	}
 
@@ -411,22 +422,75 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
 	}
 
 	snaptimes := make([]snapTime, 0, len(fss))
+	hardErrs := 0
 
 	now := time.Now()
 
-	log.Debug("examine filesystem state")
+	log.Debug("examine filesystem state to find sync point")
 	for _, d := range fss {
-
 		l := log.WithField("fs", d.ToString())
+		syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(l, now, interval, prefix, d)
+		if err == findSyncPointFSNoFilesystemVersionsErr {
+			snaptimes = append(snaptimes, snapTime{
+				ds:   d,
+				prio: prioNoVersions,
+				time: now,
+			})
+		} else if err != nil {
+			hardErrs++
+			l.WithError(err).Error("cannot determine optimal sync point for this filesystem")
+		} else {
+			l.WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
+			snaptimes = append(snaptimes, snapTime{
+				ds:   d,
+				prio: prioHasVersions,
+				time: syncPoint,
+			})
+		}
+	}
+
+	if hardErrs == len(fss) {
+		return time.Time{}, fmt.Errorf("hard errors in determining sync point for every matching filesystem")
+	}
+
+	if len(snaptimes) == 0 {
+		panic("implementation error: loop must either inc hardErrs or add result to snaptimes")
+	}
+
+	// sort ascending by (prio,time)
+	// => those filesystems with versions win over those without any
+	sort.Slice(snaptimes, func(i, j int) bool {
+		if snaptimes[i].prio == snaptimes[j].prio {
+			return snaptimes[i].time.Before(snaptimes[j].time)
+		}
+		return snaptimes[i].prio < snaptimes[j].prio
+	})
+
+	winnerSyncPoint := snaptimes[0].time
+	l := log.WithField("syncPoint", winnerSyncPoint.String())
+	l.Info("determined sync point")
+	if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
+		for _, st := range snaptimes {
+			if st.prio == prioNoVersions {
+				l.WithField("fs", st.ds.ToString()).Warn("filesystem will not be snapshotted until sync point")
+			}
+		}
+	}
+
+	return snaptimes[0].time, nil
+
+}
+
+var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
+
+func findSyncPointFSNextOptimalSnapshotTime(l Logger, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
 
 	fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
 	if err != nil {
-			l.WithError(err).Error("cannot list filesystem versions")
-			continue
+		return time.Time{}, errors.Wrap(err, "list filesystem versions")
 	}
 	if len(fsvs) <= 0 {
-			l.WithField("prefix", prefix).Debug("no filesystem versions with prefix")
-			continue
+		return time.Time{}, findSyncPointFSNoFilesystemVersionsErr
 	}
 
 	// Sort versions by creation
@@ -435,31 +499,12 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
 	})
 
 	latest := fsvs[len(fsvs)-1]
-		l.WithField("creation", latest.Creation).
-			Debug("found latest snapshot")
+	l.WithField("creation", latest.Creation).Debug("found latest snapshot")
 
 	since := now.Sub(latest.Creation)
 	if since < 0 {
-			l.WithField("snapshot", latest.Name).
-				WithField("creation", latest.Creation).
-				Error("snapshot is from the future")
-			continue
-		}
-		next := now
-		if since < interval {
-			next = latest.Creation.Add(interval)
-		}
-		snaptimes = append(snaptimes, snapTime{d, next})
+		return time.Time{}, fmt.Errorf("snapshot %q is from the future: creation=%q now=%q", latest.ToAbsPath(d), latest.Creation, now)
 	}
 
-	if len(snaptimes) == 0 {
-		snaptimes = append(snaptimes, snapTime{nil, now})
-	}
-
-	sort.Slice(snaptimes, func(i, j int) bool {
-		return snaptimes[i].time.Before(snaptimes[j].time)
-	})
-
-	return snaptimes[0].time, nil
-
+	return latest.Creation.Add(interval), nil
 }

dist/systemd/zrepl.service

@@ -4,8 +4,9 @@ Documentation=https://zrepl.github.io
 
 [Service]
 Type=simple
+ExecStartPre=/usr/local/bin/zrepl --config /etc/zrepl/zrepl.yml configcheck
 ExecStart=/usr/local/bin/zrepl --config /etc/zrepl/zrepl.yml daemon
-RuntimeDirectory=zrepl
+RuntimeDirectory=zrepl zrepl/stdinserver
 RuntimeDirectoryMode=0700
 
 ProtectSystem=strict
@@ -20,24 +21,13 @@ RestrictNamespaces=true
 RestrictRealtime=yes
 SystemCallArchitectures=native
 
-# BEGIN ProtectHome
-# DEBIAN STRETCH
 ProtectHome=read-only
-# FEDORA 28 / 29
-# ProtectHome=tmpfs
-# END ProtectHome
+# ProtectHome=tmpfs totally possible, not by default though because of Debian stretch
+
+# SystemCallFilter
+#   ~@privileged doesn't work with Ubuntu 18.04 ssh
+SystemCallFilter=~ @mount @cpu-emulation @keyring @module @obsolete @raw-io @debug @clock @resources
 
-# BEGIN SystemCallFilter
-## BEGIN DEBIAN STRETCH
-SystemCallFilter=~ @mount @cpu-emulation @keyring @module @obsolete @privileged @raw-io @debug @clock @resources
-## END DEBIAN STRETCH
-## BEGIN FEDORA 28/29 
-## Syscall blacklist (should be fairly stable)
-#SystemCallFilter=~ @mount @aio @cpu-emulation @keyring @memlock @module @obsolete @privileged @raw-io @reboot @setuid @swap @sync @timer @debug @clock @chown @resources
-## Syscall whitelist (not sure how stable)
-#SystemCallFilter=@default @file-system @process @basic-io @ipc @network-io @signal @io-event brk mprotect sched_getaffinity ioctl getrandom
-## END END FEDORA 28/29
-# END SystemCallFilter
 
 [Install]
 WantedBy=multi-user.target

docs/changelog.rst

@@ -27,6 +27,54 @@ We use the following annotations for classifying changes:
 * |bugfix| Change that fixes a bug, no regressions or incompatibilities expected.
 * |docs| Change to the documentation.
 
+0.3
+---
+
+This is a big one! Headlining features:
+
+* **Resumable Send & Recv Support**
+  No knobs required, automatically used where supported.
+* **Hold-Protected Send & Recv**
+  Automatic ZFS holds to ensure that we can always resume a replication step.
+* **Encrypted Send & Recv Support** for OpenZFS native encryption.
+  :ref:`Configurable <job-send-options>` at the job level, i.e., for all filesystems a job is responsible for.
+* **Receive-side hold on last received dataset**
+  The counterpart to the replication cursor bookmark on the send-side.
+  Ensures that incremental replication will always be possible between a sender and receiver.
+
+Actual changelog:
+
+* |break_config| **more restrictive job names than in prior zrepl versions**
+  Starting with this version, job names are going to be embedded into ZFS holds and bookmark names.
+
+* |break| |mig| replication cursor representation changed
+
+  * zrepl now manages the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` per job-filesystem tuple instead of a single replication cursor per filesystem.
+    In the future, this will permit multiple sending jobs to send from the same filesystems.
+  * ZFS does not allow bookmark renaming, thus we cannot migrate the old replication cursors.
+  * zrepl 0.3 will automatically create cursors in the new format for new replications, and warn if it still finds ones in the old format.
+  * Run ``zrepl migrate replication-cursor:v1-v2`` to safely destroy old-format cursors.
+    The migration will ensure that only those old-format cursors are destroyed that have been superseeded by new-format cursors.
+
+* |bugfix| missing logger context vars in control connection handlers
+* |bugfix| improved error messages on ``zfs send`` errors
+* |feature| New option ``listen_freebind`` (tcp, tls, prometheus listener)
+* |bugfix| |docs| snapshotting: clarify sync-up behavior and warn about filesystems
+  that will not be snapshotted until the sync-up phase is over
+* |docs| Document new replication features in the :ref:`config overview <overview-how-replication-works>` and :repomasterlink:`replication/design.md`.
+* **[MAINTAINER NOTICE]** New platform tests in this version, please make sure you run them for your distro!
+
+0.2.1
+-----
+
+* |feature| Illumos (and Solaris) compatibility and binary builds (thanks, `MNX.io <https://mnx.io>`_ )
+* |feature| 32bit binaries for Linux and FreeBSD (untested, though)
+* |bugfix| better error messages in ``ssh+stdinserver`` transport
+* |bugfix| systemd + ``ssh+stdinserver``: automatically create ``/var/run/zrepl/stdinserver``
+* |bugfix| crash if Prometheus listening socket cannot be opened
+
+* [MAINTAINER NOTICE] ``Makefile`` refactoring, see :commit:`080f2c0`
+
 0.2
 ---
 

docs/configuration.rst

@@ -11,6 +11,7 @@ Configuration
     configuration/jobs
     configuration/transports
     configuration/filter_syntax
+    configuration/sendrecvoptions
     configuration/snapshotting
     configuration/prune
     configuration/logging

docs/configuration/filter_syntax.rst

@@ -5,7 +5,7 @@
 Filter Syntax
 =============
 
-For :ref:`source jobs <job-source>` and :ref:`push jobs <job-push>`, a filesystem filter must be defined (field ``filesystems``).
+For :ref:`source<job-source>`, :ref:`push<job-push>` and :ref:`snap<job-snap>` jobs, a filesystem filter must be defined (field ``filesystems``).
 A filter takes a filesystem path (in the ZFS filesystem hierarchy) as parameter and returns ``true`` (pass) or ``false`` (block).
 
 A filter is specified as a **YAML dictionary** with patterns as keys and booleans as values.

docs/configuration/jobs.rst

@@ -24,6 +24,8 @@ Job Type ``push``
       - |connect-transport|
     * - ``filesystems``
       - |filter-spec| for filesystems to be snapshotted and pushed to the sink
+    * - ``send``
+      - |send-options| 
     * - ``snapshotting``
       - |snapshotting-spec|
     * - ``pruning``
@@ -101,6 +103,8 @@ Job Type ``source``
       - |serve-transport|
     * - ``filesystems``
       - |filter-spec| for filesystems to be snapshotted and exposed to connecting clients
+    * - ``send``
+      - |send-options| 
     * - ``snapshotting``
       - |snapshotting-spec|
 

docs/configuration/monitoring.rst

@@ -14,6 +14,7 @@ Prometheus & Grafana
 
 zrepl can expose `Prometheus metrics <https://prometheus.io/docs/instrumenting/exposition_formats/>`_ via HTTP.
 The ``listen`` attribute is a `net.Listen <https://golang.org/pkg/net/#Listen>`_  string for tcp, e.g. ``:9091`` or ``127.0.0.1:9091``.
+The ``listen_freebind`` attribute is :ref:`explained here <listen-freebind-explanation>`.
 The Prometheues monitoring job appears in the ``zrepl control`` job list and may be specified **at most once**.
 
 zrepl also ships with an importable `Grafana <https://grafana.com>`_ dashboard that consumes the Prometheus metrics:
@@ -30,6 +31,7 @@ The dashboard also contains some advice on which metrics are important to monito
       monitoring:
         - type: prometheus
           listen: ':9091'
+          listen_freebind: true # optional, default false
 
 
 

docs/configuration/overview.rst

@@ -90,6 +90,8 @@ It uses the client identity for access control:
 .. TIP::
    The implementation of the ``sink`` job requires that the connecting client identities be a valid ZFS filesystem name components.
 
+.. _overview-how-replication-works:
+
 How Replication Works
 ---------------------
 
@@ -107,9 +109,8 @@ The following steps take place during replication and can be monitored using the
     * Per filesystem, compute a diff between sender and receiver snapshots
     * Build a list of replication steps
 
-      * If possible, use incremental sends (``zfs send -i``)
+      * If possible, use incremental and resumable sends (``zfs send -i``)
       * Otherwise, use full send of most recent snapshot on sender
-      * Give up on filesystems that cannot be replicated without data loss
 
   * Retry on errors that are likely temporary (i.e. network failures).
   * Give up on filesystems where a permanent error was received over RPC.
@@ -119,17 +120,23 @@ The following steps take place during replication and can be monitored using the
   * Perform replication steps in the following order:
     Among all filesystems with pending replication steps, pick the filesystem whose next replication step's snapshot is the oldest.
   * Create placeholder filesystems on the receiving side to mirror the dataset paths on the sender to ``root_fs/${client_identity}``.
-  * After a successful replication step, update the replication cursor bookmark (see below).
+  * Aquire send-side step-holds on the step's `from` and `to` snapshots.
+  * Perform the replication step.
+  * Move the **replication cursor** bookmark on the sending side (see below).
+  * Move the **last-received-hold** on the receiving side (see below).
+  * Release the send-side step-holds.
    
 The idea behind the execution order of replication steps is that if the sender snapshots all filesystems simultaneously at fixed intervals, the receiver will have all filesystems snapshotted at time ``T1`` before the first snapshot at ``T2 = T1 + $interval`` is replicated.
 
-.. _replication-cursor-bookmark:
+.. _replication-cursor-and-last-received-hold:
 
-The **replication cursor bookmark** ``#zrepl_replication_cursor`` is kept per filesystem on the sending side of a replication setup:
-It is a bookmark of the most recent successfully replicated snapshot to the receiving side.
-It is is used by the :ref:`not_replicated <prune-keep-not-replicated>` keep rule to identify all snapshots that have not yet been replicated to the receiving side.
-Regardless of whether that keep rule is used, the bookmark ensures that replication can always continue incrementally.
-Note that there is only one cursor bookmark per filesystem, which prohibits multiple jobs to replicate the same filesystem (:ref:`see below<jobs-multiple-jobs>`).
+**Replication cursor** bookmark and **last-received-hold** are managed by zrepl to ensure that future replications can always be done incrementally:
+the replication cursor is a send-side bookmark of the most recent successfully replicated snapshot,
+and the last-received-hold is a hold of that snapshot on the receiving side.
+The replication cursor has the format ``#zrepl_CUSOR_G_<GUID>_J_<JOBNAME>``.
+The last-received-hold tag has the format ``#zrepl_last_received_J_<JOBNAME>``.
+Encoding the job name in the names ensures that multiple sending jobs can replicate the same filesystem to different receivers without interference.
+The ``zrepl holds list`` provides a listing of all bookmarks and holds managed by zrepl.
 
 .. _replication-placeholder-property:
 
@@ -144,9 +151,13 @@ The ``zrepl test placeholder`` command can be used to check whether a filesystem
 .. ATTENTION::
 
     Currently, zrepl does not replicate filesystem properties.
-    Whe receiving a filesystem, it is never mounted (`-u` flag)  and `mountpoint=none` is set.
+    When receiving a filesystem, it is never mounted (`-u` flag)  and `mountpoint=none` is set.
     This is temporary and being worked on :issue:`24`.
 
+.. NOTE::
+
+    More details can be found in the design document :repomasterlink:`replication/design.md`.
+
 
 .. _jobs-multiple-jobs:
 
@@ -171,7 +182,7 @@ No Overlapping
 Jobs run independently of each other.
 If two jobs match the same filesystem with their ``filesystems`` filter, they will operate on that filesystem independently and potentially in parallel.
 For example, if job A prunes snapshots that job B is planning to replicate, the replication will fail because B asssumed the snapshot to still be present.
-More subtle race conditions can occur with the :ref:`replication cursor bookmark <replication-cursor-bookmark>`, which currently only exists once per filesystem.
+However, the next replication attempt will re-examine the situation from scratch and should work.
 
 N push jobs to 1 sink
 ~~~~~~~~~~~~~~~~~~~~~

docs/configuration/prune.rst

@@ -4,7 +4,7 @@ Pruning Policies
 ================
 
 In zrepl, *pruning* means *destroying snapshots*.
-Pruning must happen on both sides of a replication or the systems would inevitable run out of disk space at some point.
+Pruning must happen on both sides of a replication or the systems would inevitably run out of disk space at some point.
 
 Typically, the requirements to temporal resolution and maximum retention time differ per side.
 For example, when using zrepl to back up a busy database server, you will want high temporal resolution (snapshots every 10 min) for the last 24h in case of administrative disasters, but cannot afford to store them for much longer because you might have high turnover volume in the database.
@@ -66,7 +66,7 @@ Policy ``not_replicated``
 
 ``not_replicated`` keeps all snapshots that have not been replicated to the receiving side.
 It only makes sense to specify this rule on a sender (source or push job).
-The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark <replication-cursor-bookmark>` on the sending side.
+The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` on the sending side.
 
 .. _prune-keep-retention-grid:
 

docs/configuration/sendrecvoptions.rst

@@ -0,0 +1,45 @@
+.. include:: ../global.rst.inc
+
+
+Send & Recv Options
+===================
+
+.. _job-send-options:
+
+Send Options
+~~~~~~~~~~~~
+
+::
+   
+   jobs:
+   - type: push
+     filesystems: ...
+     send:
+       encrypted: true
+     ...
+
+:ref:`Source<job-source>` and :ref:`push<job-push>` jobs have an optional ``send`` configuration section.
+
+``encryption`` option
+---------------------
+
+The ``encryption`` variable controls whether the matched filesystems are sent as `OpenZFS native encryption <http://open-zfs.org/wiki/ZFS-Native_Encryption>`_ raw sends.
+More specificially, if ``encryption=true``, zrepl
+
+* checks for any of the filesystems matched by ``filesystems`` whether the ZFS ``encryption`` property indicates that the filesystem is actually encrypted with ZFS native encryption and
+* invokes the ``zfs send`` subcommand with the ``-w`` option (raw sends) and
+* expects the receiving side to support OpenZFS native encryption (recv will fail otherwise)
+
+Filesystems matched by ``filesystems`` that are not encrypted are not sent and will cause error log messages.
+
+If ``encryption=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.
+
+.. _job-recv-options:
+
+Recv Options
+~~~~~~~~~~~~
+
+:ref:`Sink<job-sink>` and :ref:`pull<job-pull>` jobs have an optional ``recv`` configuration section.
+However, there are currently no variables to configure there.
+
+

docs/configuration/snapshotting.rst

@@ -9,8 +9,15 @@ The ``push``, ``source`` and ``snap`` jobs can automatically take periodic snaps
 The snapshot names are composed of a user-defined prefix followed by a UTC date formatted like ``20060102_150405_000``.
 We use UTC because it will avoid name conflicts when switching time zones or between summer and winter time.
 
+When a job is started, the snapshotter attempts to get the snapshotting rhythms of the matched ``filesystems`` in sync because snapshotting all filesystems at the same time results in a more consistent backup.
+To find that sync point, the most recent snapshot, made by the snapshotter, in any of the matched ``filesystems`` is used.
+A filesystem that does not have snapshots by the snapshotter has lower priority than filesystem that do, and thus might not be snapshotted (and replicated) until it is snapshotted at the next sync point.
+
 For ``push`` jobs, replication is automatically triggered after all filesystems have been snapshotted.
 
+Note that the ``zrepl signal wakeup JOB`` subcommand does not trigger snapshotting.
+
+
 ::
 
     jobs:
@@ -113,14 +120,11 @@ Most hook types take additional parameters, please refer to the respective subse
       ...
 
 
-The hook type ``command`` is the only currently supported hook type.
-Future versions of zrepl may support other hook types.
-The ``path`` to the hook executables must be absolute (e.g. ``/etc/zrepl/hooks/zrepl-notify.sh``).
+``command`` hooks take a ``path`` to an executable script or binary to be executed before and after the snapshot.
+``path`` must be absolute (e.g. ``/etc/zrepl/hooks/zrepl-notify.sh``).
 No arguments may be specified; create a wrapper script if zrepl must call an executable that requires arguments.
-zrepl will call the hook both before and after the snapshot, but with different values of the ``ZREPL_HOOKTYPE`` environment variable; see below.
 The process standard output is logged at level INFO. Standard error is logged at level WARN.
-
-zrepl sets a number of environment variables for the hook processes:
+The following environment variables are set:
 
 * ``ZREPL_HOOKTYPE``: either "pre_snapshot" or "post_snapshot"
 * ``ZREPL_FS``: the ZFS filesystem name being snapshotted

docs/configuration/transports.rst

@@ -50,12 +50,18 @@ Serve
       serve:
         type: tcp
         listen: ":8888"
+        listen_freebind: true # optional, default false
         clients: {
           "192.168.122.123" : "mysql01"
           "192.168.122.123" : "mx01"
         }
       ...
 
+.. _listen-freebind-explanation:
+
+``listen_freebind`` controls whether the socket is allowed to bind to non-local or unconfigured IP addresses (Linux ``IP_FREEBIND`` , FreeBSD ``IP_BINDANY``).
+Enable this option if you want to ``listen`` on a specific IP address that might not yet be configured when the zrepl daemon starts.
+
 Connect
 ~~~~~~~
 
@@ -101,6 +107,7 @@ Serve
         serve:
           type: tls
           listen: ":8888"
+          listen_freebind: true # optional, default false
           ca:   /etc/zrepl/ca.crt
           cert: /etc/zrepl/prod.fullchain
           key:  /etc/zrepl/prod.key
@@ -110,6 +117,7 @@ Serve
 
 The ``ca`` field specified the certificate authority used to validate client certificates.
 The ``client_cns`` list specifies a list of accepted client common names (which are also the client identities for this transport).
+The ``listen_freebind`` field is :ref:`explained here <listen-freebind-explanation>`.
 
 Connect
 ~~~~~~~
@@ -287,9 +295,9 @@ Connect
         user: root
         port: 22
         identity_file: /etc/zrepl/ssh/identity
-        options: # optional, default [], `-o` arguments passed to ssh
-        - "Compression=on"
-        dial_timeout: # optional, default 10s, max time.Duration until initial handshake is completed
+        # options: # optional, default [], `-o` arguments passed to ssh
+        # - "Compression=yes"
+        # dial_timeout: 10s # optional, default 10s, max time.Duration until initial handshake is completed
 
 The connecting zrepl daemon
 

docs/global.rst.inc

@@ -21,6 +21,8 @@
 
 .. |serve-transport| replace:: :ref:`serve specification<transport>`
 .. |connect-transport| replace:: :ref:`connect specification<transport>`
+.. |send-options| replace:: :ref:`send options<job-send-options>`, e.g. for encrypted sends
+.. |recv-options| replace:: :ref:`recv options<job-recv-options>`
 .. |snapshotting-spec| replace:: :ref:`snapshotting specification <job-snapshotting-spec>`
 .. |pruning-spec| replace:: :ref:`pruning specification <prune>`
 .. |filter-spec| replace:: :ref:`filter specification<pattern-filter>`

docs/implementation.rst

@@ -1,61 +0,0 @@
-.. _implementation_toc:
-
-Implementation Overview
-=======================
-
-.. WARNING::
-
-   Incomplete and possibly outdated.
-   Check out the :ref:`talks about zrepl <pr-talks>` at various conferences for up-to-date material.
-   Alternatively, have a `look at the source code <http://github.com/zrepl/zrepl>`_ ;)
-
-The following design aspects may convince you that ``zrepl`` is superior to a hacked-together shell script solution.
-Also check out the :ref:`talks about zrepl <pr-talks>` at various conferences.
-
-Testability & Performance
--------------------------
-
-zrepl is written in Go, a real programming language with type safety,
-reasonable performance, testing infrastructure and an (opinionated) idea of
-software engineering.
-
-* key parts & algorithms of zrepl are covered by unit tests (work in progress)
-* zrepl is noticably faster than comparable shell scripts
-
-
-RPC protocol
-------------
-
-While it is tempting to just issue a few ``ssh remote 'zfs send ...' | zfs recv``, this has a number of drawbacks:
-
-* The snapshot streams need to be compatible.
-* Communication is still unidirectional. Thus, you will most likely
-
-  * either not take advantage of advanced replication features such as *compressed send & recv*
-  * or issue additional ``ssh`` commands in advance to figure out what features are supported on the other side.
-
-* Advanced logic in shell scripts is ugly to read, poorly testable and a pain to maintain.
-
-zrepl takes a different approach:
-
-* Define an RPC protocol.
-* Establish an encrypted, authenticated, bidirectional communication channel.
-* Run daemons on both sides of the setup and let them talk to each other.
-
-This has several obvious benefits:
-
-* No blank root shell access is given to the other side.
-* An *authenticated* peer *requests* filesystem lists, snapshot streams, etc.
-* The server decides which filesystems it exposes to which peers.
-* The :ref:`transport mechanism <transport>` is decoupled from the remaining logic, which allows us to painlessly offer multiple transport mechanisms.
-
-Protocol Implementation
-~~~~~~~~~~~~~~~~~~~~~~~
-
-zrepl uses a custom RPC protol because, at the time of writing, existing solutions like gRPC do not provide efficient means to transport large amounts of data, whose size is unknown at send time (= zfs send streams).
-The package used is `github.com/problame/go-streamrpc <https://github.com/problame/go-streamrpc/tree/master>`_.
-
-Logging & Transparency
-----------------------
-
-zrepl comes with :ref:`rich, structured and configurable logging <logging>`, allowing administators to understand what the software is actually doing.

docs/index.rst

@@ -57,9 +57,11 @@ Main Features
   * Advanced replication features
 
     * [x] Automatic retries for temporary network errors
-    * [ ] Resumable send & receive
+    * [x] Automatic resumable send & receive
+    * [x] Automatic ZFS holds during send & receive
+    * [x] Automatic bookmark \& hold management for guaranteed incremental send & recv 
+    * [x] Encrypted raw send & receive to untrusted receivers (OpenZFS native encryption)
     * [ ] Compressed send & receive
-    * [ ] Raw encrypted send & receive
 
 * **Automatic snapshot management**
 
@@ -83,6 +85,7 @@ Main Features
 * **Maintainable implementation in Go**
 
   * [x] Cross platform
+  * [x] Dynamic feature checking
   * [x] Type safe & testable code
 
 
@@ -126,7 +129,6 @@ Table of Contents
    installation
    configuration
    usage
-   implementation
    pr
    changelog
    GitHub Repository & Issue Tracker <https://github.com/zrepl/zrepl>

docs/installation.rst

@@ -38,12 +38,56 @@ The following list may be incomplete, feel free to submit a PR with an update:
       - ``brew install zrepl``
       - Available on `homebrew <https://brew.sh>`_
     * - Arch Linux
-      - ``yay install zrepl-bin``
-      - Available on `AUR <https://aur.archlinux.org/packages/zrepl-bin>`_
+      - ``yay install zrepl``
+      - Available on `AUR <https://aur.archlinux.org/packages/zrepl>`_
+    * - Fedora
+      - ``dnf install zrepl``
+      - Available on `COPR <https://copr.fedorainfracloud.org/coprs/poettlerric/zrepl/>`_
+    * - CentOS/RHEL
+      - ``yum install zrepl``
+      - Available on `COPR <https://copr.fedorainfracloud.org/coprs/poettlerric/zrepl/>`_
+    * - Debian + Ubuntu
+      - ``apt install zrepl``
+      - APT repository config :ref:`see below <installation-apt-repos>`
+    * - OmniOS
+      - ``pkg install zrepl``
+      - Available since `r151030 <https://pkg.omniosce.org/r151030/extra/en/search.shtml?token=zrepl&action=Search>`_
+    * - Void Linux
+      - ``xbps-install zrepl``
+      - Available since `a88a2a4 <https://github.com/void-linux/void-packages/commit/a88a2a4d7bf56072dadf61ab56b8424e39155890>`_
     * - Others
       -
       - Use `binary releases`_ or build from source.
 
+.. _installation-apt-repos:
+
+Debian / Ubuntu APT repositories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We maintain APT repositories for Debian, Ubuntu and derivatives.
+The fingerprint of the signing key is ``E101 418F D3D6 FBCB 9D65  A62D 7086 99FC 5F2E BF16``.
+It is available at `<https://zrepl.cschwarz.com/apt/apt-key.asc>`_ .
+Please open an issue `in the packaging repository <https://github.com/zrepl/debian-binary-packaging>`_ if you encounter any issues with the repository.
+
+The following snippet configure the repository for your Debian or Ubuntu release:
+
+::
+
+    apt update && apt install curl gnupg lsb-release; \
+    ARCH="$(dpkg --print-architecture)"; \
+    CODENAME="$(lsb_release -i -s | tr '[:upper:]' '[:lower:]') $(lsb_release -c -s | tr '[:upper:]' '[:lower:]')"; \
+    echo "Using Distro and Codename: $CODENAME"; \
+    (curl https://zrepl.cschwarz.com/apt/apt-key.asc | apt-key add -) && \
+    (echo "deb [arch=$ARCH] https://zrepl.cschwarz.com/apt/$CODENAME main" > /etc/apt/sources.list.d/zrepl.list) && \
+    apt update
+
+
+.. NOTE::
+
+   Until zrepl reaches 1.0, all APT repositories will be updated to the latest zrepl release immediately.
+   This includes breaking changes between zrepl versions.
+   Use ``apt-mark hold zrepl`` to prevent upgrades of zrepl.
+
 Compile From Source
 ~~~~~~~~~~~~~~~~~~~
 
@@ -57,9 +101,9 @@ and serves as a reference for build dependencies and procedure:
 
 ::
 
-    git clone https://github.com/zrepl/zrepl.git
-    cd zrepl
-    sudo docker build -t zrepl_build -f build.Dockerfile .
+    git clone https://github.com/zrepl/zrepl.git && \
+    cd zrepl && \
+    sudo docker build -t zrepl_build -f build.Dockerfile . && \
     sudo docker run -it --rm \
         -v "${PWD}:/src" \
         --user "$(id -u):$(id -g)" \
@@ -92,4 +136,3 @@ What next?
 Read the :ref:`configuration chapter<configuration_toc>` and then continue with the :ref:`usage chapter<usage>`.
 
 **Reminder**: If you want a quick introduction, please read the :ref:`tutorial`.
-

docs/publish.sh

@@ -56,7 +56,6 @@ sphinx-versioning build \
    --show-banner \
    --whitelist-branches '^master$' \
    --whitelist-tags '(v)?\d+\.[1-9][0-9]*.\d+$' \
-   --whitelist-tags '(v)?0.2.0-rc.*$' \
    docs ./public_git \
    -- -c sphinxconf # older conf.py throw errors because they used
                     # version = subprocess.show_output(["git", "describe"])

docs/supporters.rst

@@ -18,12 +18,31 @@ For SEPA wire transfer and **commerical support**, please `contact Christian dir
 Supporters
 ==========
 
-We would like to thank the following people / organizations for supporting zrepl through monetary and other means:
+We would like to thank the following people and organizations for supporting zrepl through monetary and other means:
 
-* `Marshall Clyburn <https://github.com/mdclyburn>`_
-* `Ross Williams <https://github.com/overhacked>`_
-* Mike T.
-* `Justin Scholz <https://github.com/JMoVS>`_
-* `InsanePrawn <https://github.com/InsanePrawn>`_
-* `Ben Woods <https://www.freshports.org/sysutils/zrepl/>`_
-* `Anton Schirg <https://github.com/antonxy>`_
+.. |supporter-std| raw:: html
+
+   <div class="fa fa-star-o" style="width: 1em;"></div>
+
+.. |supporter-gold| raw:: html
+
+   <div class="fa fa-star" style="width: 1em;"></div>
+
+.. |supporter-code| raw:: html
+
+   <div class="fa fa-code" style="width: 1em;"></div>
+
+* |supporter-gold| `Mateusz Kwiatkowski (runhyve.app) <https://runhyve.app>`_
+* |supporter-std| `Gaelan D'costa <https://www.robot-disco.net>`_
+* |supporter-std| `Tenzin Lhakhang <https://confluence.tenzin.io>`_
+* |supporter-std| `Lapo Luchini <https://github.com/lapo-luchini>`_
+* |supporter-std| `F. Schmid <https://github.com/tdschmidl>`_
+* |supporter-gold| `MNX.io <https://mnx.io>`_
+* |supporter-std| `Marshall Clyburn <https://github.com/mdclyburn>`_
+* |supporter-code| `Ross Williams <https://github.com/overhacked>`_
+* |supporter-std| Mike T.
+* |supporter-code| `Justin Scholz <https://github.com/JMoVS>`_
+* |supporter-code| `InsanePrawn <https://github.com/InsanePrawn>`_
+* |supporter-code| `Ben Woods <https://www.freshports.org/sysutils/zrepl/>`_
+* |supporter-code| `Janis Streib <https://github.com/janisstreib>`_
+* |supporter-code| `Anton Schirg <https://github.com/antonxy>`_

docs/tutorial.rst

@@ -52,9 +52,9 @@ Follow the :ref:`OS-specific installation instructions <installation>` and come
 Generate TLS Certificates
 -------------------------
 
-We use the `TLS client authentication transport <transport-tcp+tlsclientauth>` to protect our data on the wire.
+We use the :ref:`TLS client authentication transport <transport-tcp+tlsclientauth>` to protect our data on the wire.
 To get things going quickly, we skip setting up a CA and generate two self-signed certificates as described :ref:`here <transport-tcp+tlsclientauth-2machineopenssl>`.
-Again, for convenience, We generate the key pairs on our local machine and distribute them using ssh:
+For convenience, we generate the key pairs on our local machine and distribute them using ssh:
 
 .. code-block:: bash
    :emphasize-lines: 6,13
@@ -119,7 +119,7 @@ We define a **push job** named ``prod_to_backups`` in ``/etc/zrepl/zrepl.yml`` o
 Configure server ``backups``
 ----------------------------
 
-We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`` on host ``prod`` : ::
+We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`` on host ``backups`` : ::
 
     jobs:
     - name: sink
@@ -138,7 +138,7 @@ We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`
 Apply Configuration Changes
 ---------------------------
 
-We use ``zrepl configcheck`` before to catch any configuration errors: no output indicates that everything is fine.
+We use ``zrepl configcheck`` to catch any configuration errors: no output indicates that everything is fine.
 If that is the case, restart the zrepl daemon on **both** ``prod`` and ``backups`` using ``service zrepl restart`` or ``systemctl restart zrepl``.
 
 
@@ -175,6 +175,3 @@ Congratulations, you have a working push backup. Where to go next?
 
 * Read more about :ref:`configuration format, options & job types <configuration_toc>`
 * Configure :ref:`logging <logging>` \& :ref:`monitoring <monitoring>`.
-* Learn about :ref:`implementation details <implementation_toc>` of zrepl.
-
-

endpoint/endpoint.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"sync"
 
 	"github.com/pkg/errors"
 
@@ -15,13 +16,39 @@ import (
 	"github.com/zrepl/zrepl/zfs"
 )
 
+type SenderConfig struct {
+	FSF     zfs.DatasetFilter
+	Encrypt *zfs.NilBool
+	JobID   JobID
+}
+
+func (c *SenderConfig) Validate() error {
+	c.JobID.MustValidate()
+	if err := c.Encrypt.Validate(); err != nil {
+		return errors.Wrap(err, "`Encrypt` field invalid")
+	}
+	if _, err := StepHoldTag(c.JobID); err != nil {
+		return fmt.Errorf("JobID cannot be used for hold tag: %s", err)
+	}
+	return nil
+}
+
 // Sender implements replication.ReplicationEndpoint for a sending side
 type Sender struct {
 	FSFilter zfs.DatasetFilter
+	encrypt  *zfs.NilBool
+	jobId    JobID
 }
 
-func NewSender(fsf zfs.DatasetFilter) *Sender {
-	return &Sender{FSFilter: fsf}
+func NewSender(conf SenderConfig) *Sender {
+	if err := conf.Validate(); err != nil {
+		panic("invalid config" + err.Error())
+	}
+	return &Sender{
+		FSFilter: conf.FSF,
+		encrypt:  conf.Encrypt,
+		jobId:    conf.JobID,
+	}
 }
 
 func (s *Sender) filterCheckFS(fs string) (*zfs.DatasetPath, error) {
@@ -49,10 +76,15 @@ func (s *Sender) ListFilesystems(ctx context.Context, r *pdu.ListFilesystemReq)
 	}
 	rfss := make([]*pdu.Filesystem, len(fss))
 	for i := range fss {
+		encEnabled, err := zfs.ZFSGetEncryptionEnabled(ctx, fss[i].ToString())
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot get filesystem encryption status")
+		}
 		rfss[i] = &pdu.Filesystem{
 			Path: fss[i].ToString(),
-			// FIXME: not supporting ResumeToken yet
+			// ResumeToken does not make sense from Sender
 			IsPlaceholder: false, // sender FSs are never placeholders
+			IsEncrypted:   encEnabled,
 		}
 	}
 	res := &pdu.ListFilesystemRes{Filesystems: rfss}
@@ -77,13 +109,86 @@ func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesyst
 
 }
 
+func (p *Sender) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	var err error
+
+	fs := r.GetFilesystem()
+	mostRecent, err := sendArgsFromPDUAndValidateExists(ctx, fs, r.GetSenderVersion())
+	if err != nil {
+		msg := "HintMostRecentCommonAncestor rpc with nonexistent most recent version"
+		getLogger(ctx).WithField("fs", fs).WithField("hinted_most_recent", fmt.Sprintf("%#v", mostRecent)).
+			Warn(msg)
+		return nil, errors.Wrap(err, msg)
+	}
+
+	// move replication cursor to this position
+	_, err = MoveReplicationCursor(ctx, fs, mostRecent, p.jobId)
+	if err == zfs.ErrBookmarkCloningNotSupported {
+		getLogger(ctx).Debug("not creating replication cursor from bookmark because ZFS does not support it")
+		// fallthrough
+	} else if err != nil {
+		return nil, errors.Wrap(err, "cannot set replication cursor to hinted version")
+	}
+
+	// cleanup previous steps
+	if err := ReleaseStepAll(ctx, fs, mostRecent, p.jobId); err != nil {
+		return nil, errors.Wrap(err, "cannot cleanup prior invocation's step holds and bookmarks")
+	}
+
+	return &pdu.HintMostRecentCommonAncestorRes{}, nil
+}
+
 var maxConcurrentZFSSendSemaphore = semaphore.New(envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_SEND", 10))
 
+func uncheckedSendArgsFromPDU(fsv *pdu.FilesystemVersion) *zfs.ZFSSendArgVersion {
+	if fsv == nil {
+		return nil
+	}
+	return &zfs.ZFSSendArgVersion{RelName: fsv.GetRelName(), GUID: fsv.Guid}
+}
+
+func sendArgsFromPDUAndValidateExists(ctx context.Context, fs string, fsv *pdu.FilesystemVersion) (*zfs.ZFSSendArgVersion, error) {
+	v := uncheckedSendArgsFromPDU(fsv)
+	if v == nil {
+		return nil, errors.New("must not be nil")
+	}
+	if err := v.ValidateExists(ctx, fs); err != nil {
+		return nil, err
+	}
+	return v, nil
+}
+
 func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+
 	_, err := s.filterCheckFS(r.Filesystem)
 	if err != nil {
 		return nil, nil, err
 	}
+	switch r.Encrypted {
+	case pdu.Tri_DontCare:
+		// use s.encrypt setting
+		// ok, fallthrough outer
+	case pdu.Tri_False:
+		if s.encrypt.B {
+			return nil, nil, errors.New("only encrytped sends allowed (send -w + encryption!= off), but unencrytped send requested")
+		}
+		// fallthrough outer
+	case pdu.Tri_True:
+		if !s.encrypt.B {
+			return nil, nil, errors.New("only unencrypted sends allowed, but encrypted send requested")
+		}
+		// fallthrough outer
+	default:
+		return nil, nil, fmt.Errorf("unknown pdu.Tri variant %q", r.Encrypted)
+	}
+
+	sendArgs := zfs.ZFSSendArgs{
+		FS:          r.Filesystem,
+		From:        uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
+		To:          uncheckedSendArgsFromPDU(r.GetTo()),   // validated by zfs.ZFSSendDry / zfs.ZFSSend
+		Encrypted:   s.encrypt,
+		ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
+	}
 
 	getLogger(ctx).Debug("acquire concurrent send semaphore")
 	// TODO use try-acquire and fail with resource-exhaustion rpc status
@@ -95,28 +200,154 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.St
 	}
 	defer guard.Release()
 
-	si, err := zfs.ZFSSendDry(r.Filesystem, r.From, r.To, "")
+	si, err := zfs.ZFSSendDry(ctx, sendArgs)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Wrap(err, "zfs send dry failed")
 	}
 
+	// From now on, assume that sendArgs has been validated by ZFSSendDry
+	// (because validation invovles shelling out, it's actually a little expensive)
+
 	var expSize int64 = 0      // protocol says 0 means no estimate
 	if si.SizeEstimate != -1 { // but si returns -1 for no size estimate
 		expSize = si.SizeEstimate
 	}
-	res := &pdu.SendRes{ExpectedSize: expSize}
+	res := &pdu.SendRes{
+		ExpectedSize:    expSize,
+		UsedResumeToken: r.ResumeToken != "",
+	}
 
 	if r.DryRun {
 		return res, nil, nil
 	}
 
-	streamCopier, err := zfs.ZFSSend(ctx, r.Filesystem, r.From, r.To, "")
+	// update replication cursor
+	if sendArgs.From != nil {
+		// For all but the first replication, this should always be a no-op because SendCompleted already moved the cursor
+		_, err = MoveReplicationCursor(ctx, sendArgs.FS, sendArgs.From, s.jobId)
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			getLogger(ctx).Debug("not creating replication cursor from bookmark because ZFS does not support it")
+			// fallthrough
+		} else if err != nil {
+			return nil, nil, errors.Wrap(err, "cannot set replication cursor to `from` version before starting send")
+		}
+	}
+
+	// make sure `From` doesn't go away in order to make this step resumable
+	if sendArgs.From != nil {
+		err := HoldStep(ctx, sendArgs.FS, sendArgs.From, s.jobId)
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			getLogger(ctx).Debug("not creating step bookmark because ZFS does not support it")
+			// fallthrough
+		} else if err != nil {
+			return nil, nil, errors.Wrap(err, "cannot create step bookmark")
+		}
+	}
+	// make sure `To` doesn't go away in order to make this step resumable
+	err = HoldStep(ctx, sendArgs.FS, sendArgs.To, s.jobId)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Wrapf(err, "cannot hold `to` version %q before starting send", sendArgs.To.RelName)
+	}
+
+	// step holds & replication cursor released / moved forward in s.SendCompleted => s.moveCursorAndReleaseSendHolds
+
+	streamCopier, err := zfs.ZFSSend(ctx, sendArgs)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "zfs send failed")
 	}
 	return res, streamCopier, nil
 }
 
+func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	orig := r.GetOriginalReq() // may be nil, always use proto getters
+	fs := orig.GetFilesystem()
+
+	var err error
+	var from *zfs.ZFSSendArgVersion
+	if orig.GetFrom() != nil {
+		from, err = sendArgsFromPDUAndValidateExists(ctx, fs, orig.GetFrom()) // no shadow
+		if err != nil {
+			return nil, errors.Wrap(err, "validate `from` exists")
+		}
+	}
+	to, err := sendArgsFromPDUAndValidateExists(ctx, fs, orig.GetTo())
+	if err != nil {
+		return nil, errors.Wrap(err, "validate `to` exists")
+	}
+
+	log := getLogger(ctx).WithField("to_guid", to.GUID).
+		WithField("fs", fs).
+		WithField("to", to.RelName)
+	if from != nil {
+		log = log.WithField("from", from.RelName).WithField("from_guid", from.GUID)
+	}
+
+	log.Debug("move replication cursor to most recent common version")
+	destroyedCursors, err := MoveReplicationCursor(ctx, fs, to, p.jobId)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			log.Debug("not setting replication cursor, bookmark cloning not supported")
+		} else {
+			msg := "cannot move replication cursor, keeping hold on `to` until successful"
+			log.WithError(err).Error(msg)
+			err = errors.Wrap(err, msg)
+			// it is correct to not release the hold if we can't move the cursor!
+			return &pdu.SendCompletedRes{}, err
+		}
+	} else {
+		log.Info("successfully moved replication cursor")
+	}
+
+	// kick off releasing of step holds / bookmarks
+	// if we fail to release them, don't bother the caller:
+	// they are merely an implementation detail on the sender for better resumability
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		log.Debug("release step-hold of or step-bookmark on `to`")
+		err = ReleaseStep(ctx, fs, to, p.jobId)
+		if err != nil {
+			log.WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `to`")
+		} else {
+			log.Info("successfully released step-holds on or destroyed step-bookmark of `to`")
+		}
+
+	}()
+	go func() {
+		defer wg.Done()
+		if from == nil {
+			return
+		}
+		log.Debug("release step-hold of or step-bookmark on `from`")
+		err := ReleaseStep(ctx, fs, from, p.jobId)
+		if err != nil {
+			if dne, ok := err.(*zfs.DatasetDoesNotExist); ok {
+				// If bookmark cloning is not supported, `from` might be the old replication cursor
+				// and thus have already been destroyed by MoveReplicationCursor above
+				// In that case, nonexistence of `from` is not an error, otherwise it is.
+				fsp, err := zfs.NewDatasetPath(fs)
+				if err != nil {
+					panic(err) // fs has been validated multiple times above
+				}
+				for _, fsv := range destroyedCursors {
+					if fsv.ToAbsPath(fsp) == dne.Path {
+						log.Info("`from` was a replication cursor and has already been destroyed")
+						return
+					}
+				}
+				// fallthrough
+			}
+			log.WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `from`")
+		} else {
+			log.Info("successfully released step-holds on or destroyed step-bookmark of `from`")
+		}
+	}()
+	wg.Wait()
+
+	return &pdu.SendCompletedRes{}, nil
+}
+
 func (p *Sender) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) {
 	dp, err := p.filterCheckFS(req.Filesystem)
 	if err != nil {
@@ -146,9 +377,7 @@ func (p *Sender) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCurs
 		return nil, err
 	}
 
-	switch op := req.Op.(type) {
-	case *pdu.ReplicationCursorReq_Get:
-		cursor, err := zfs.ZFSGetReplicationCursor(dp)
+	cursor, err := GetMostRecentReplicationCursorOfJob(ctx, dp.ToString(), p.jobId)
 	if err != nil {
 		return nil, err
 	}
@@ -156,15 +385,6 @@ func (p *Sender) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCurs
 		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
 	}
 	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: cursor.Guid}}, nil
-	case *pdu.ReplicationCursorReq_Set:
-		guid, err := zfs.ZFSSetReplicationCursor(dp, op.Set.Snapshot)
-		if err != nil {
-			return nil, err
-		}
-		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: guid}}, nil
-	default:
-		return nil, errors.Errorf("unknown op %T", op)
-	}
 }
 
 func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error) {
@@ -183,21 +403,41 @@ type FSMap interface { // FIXME unused
 	AsFilter() FSFilter
 }
 
+type ReceiverConfig struct {
+	JobID JobID
+
+	RootWithoutClientComponent *zfs.DatasetPath // TODO use
+	AppendClientIdentity       bool
+
+	UpdateLastReceivedHold bool
+}
+
+func (c *ReceiverConfig) copyIn() {
+	c.RootWithoutClientComponent = c.RootWithoutClientComponent.Copy()
+}
+
+func (c *ReceiverConfig) Validate() error {
+	c.JobID.MustValidate()
+	if c.RootWithoutClientComponent.Length() <= 0 {
+		return errors.New("RootWithoutClientComponent must not be an empty dataset path")
+	}
+	return nil
+}
+
 // Receiver implements replication.ReplicationEndpoint for a receiving side
 type Receiver struct {
-	rootWithoutClientComponent *zfs.DatasetPath
-	appendClientIdentity       bool
+	conf ReceiverConfig // validated
 
 	recvParentCreationMtx *chainlock.L
 }
 
-func NewReceiver(rootDataset *zfs.DatasetPath, appendClientIdentity bool) *Receiver {
-	if rootDataset.Length() <= 0 {
-		panic(fmt.Sprintf("root dataset must not be an empty path: %v", rootDataset))
+func NewReceiver(config ReceiverConfig) *Receiver {
+	config.copyIn()
+	if err := config.Validate(); err != nil {
+		panic(err)
 	}
 	return &Receiver{
-		rootWithoutClientComponent: rootDataset.Copy(),
-		appendClientIdentity:       appendClientIdentity,
+		conf:                  config,
 		recvParentCreationMtx: chainlock.New(),
 	}
 }
@@ -221,8 +461,8 @@ func clientRoot(rootFS *zfs.DatasetPath, clientIdentity string) (*zfs.DatasetPat
 }
 
 func (s *Receiver) clientRootFromCtx(ctx context.Context) *zfs.DatasetPath {
-	if !s.appendClientIdentity {
-		return s.rootWithoutClientComponent.Copy()
+	if !s.conf.AppendClientIdentity {
+		return s.conf.RootWithoutClientComponent.Copy()
 	}
 
 	clientIdentity, ok := ctx.Value(ClientIdentityKey).(string)
@@ -230,7 +470,7 @@ func (s *Receiver) clientRootFromCtx(ctx context.Context) *zfs.DatasetPath {
 		panic(fmt.Sprintf("ClientIdentityKey context value must be set"))
 	}
 
-	clientRoot, err := clientRoot(s.rootWithoutClientComponent, clientIdentity)
+	clientRoot, err := clientRoot(s.conf.RootWithoutClientComponent, clientIdentity)
 	if err != nil {
 		panic(fmt.Sprintf("ClientIdentityContextKey must have been validated before invoking Receiver: %s", err))
 	}
@@ -282,8 +522,27 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
 			err := errors.Errorf("inconsistent placeholder state: filesystem %q must exist in this context", a.ToString())
 			return nil, err
 		}
+		token, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, a)
+		if err != nil {
+			l.WithError(err).Error("cannot get receive resume token")
+			return nil, err
+		}
+		encEnabled, err := zfs.ZFSGetEncryptionEnabled(ctx, a.ToString())
+		if err != nil {
+			l.WithError(err).Error("cannot get encryption enabled status")
+			return nil, err
+		}
+		l.WithField("receive_resume_token", token).Debug("receive resume token")
+
 		a.TrimPrefix(root)
-		fss = append(fss, &pdu.Filesystem{Path: a.ToString(), IsPlaceholder: ph.IsPlaceholder})
+
+		fs := &pdu.Filesystem{
+			Path:          a.ToString(),
+			IsPlaceholder: ph.IsPlaceholder,
+			ResumeToken:   token,
+			IsEncrypted:   encEnabled,
+		}
+		fss = append(fss, fs)
 	}
 	if len(fss) == 0 {
 		getLogger(ctx).Debug("no filesystems found")
@@ -344,7 +603,18 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 	root := s.clientRootFromCtx(ctx)
 	lp, err := subroot{root}.MapToLocal(req.Filesystem)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "`Filesystem` invalid")
+	}
+
+	to := uncheckedSendArgsFromPDU(req.GetTo())
+	if to == nil {
+		return nil, errors.New("`To` must not be nil")
+	}
+	if err := to.ValidateInMemory(lp.ToString()); err != nil {
+		return nil, errors.Wrap(err, "`To` invalid")
+	}
+	if !to.IsSnapshot() {
+		return nil, errors.New("`To` must be a snapshot")
 	}
 
 	// create placeholder parent filesystems as appropriate
@@ -380,11 +650,11 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 			}
 
 			if !ph.FSExists {
-				if s.rootWithoutClientComponent.HasPrefix(v.Path) {
+				if s.conf.RootWithoutClientComponent.HasPrefix(v.Path) {
 					if v.Path.Length() == 1 {
 						visitErr = fmt.Errorf("pool %q not imported", v.Path.ToString())
 					} else {
-						visitErr = fmt.Errorf("root_fs %q does not exist", s.rootWithoutClientComponent.ToString())
+						visitErr = fmt.Errorf("root_fs %q does not exist", s.conf.RootWithoutClientComponent.ToString())
 					}
 					getLogger(ctx).WithError(visitErr).Error("placeholders are only created automatically below root_fs")
 					return false
@@ -422,6 +692,17 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 		}
 	}
 
+	if req.ClearResumeToken && ph.FSExists {
+		if err := zfs.ZFSRecvClearResumeToken(lp.ToString()); err != nil {
+			return nil, errors.Wrap(err, "cannot clear resume token")
+		}
+	}
+
+	recvOpts.SavePartialRecvState, err = zfs.ResumeRecvSupported(ctx, lp)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot determine whether we can use resumable send & recv")
+	}
+
 	getLogger(ctx).Debug("acquire concurrent recv semaphore")
 	// TODO use try-acquire and fail with resource-exhaustion rpc status
 	// => would require handling on the client-side
@@ -434,13 +715,30 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 
 	getLogger(ctx).WithField("opts", fmt.Sprintf("%#v", recvOpts)).Debug("start receive command")
 
-	if err := zfs.ZFSRecv(ctx, lp.ToString(), receive, recvOpts); err != nil {
+	snapFullPath := to.FullPath(lp.ToString())
+	if err := zfs.ZFSRecv(ctx, lp.ToString(), to, receive, recvOpts); err != nil {
 		getLogger(ctx).
 			WithError(err).
 			WithField("opts", recvOpts).
 			Error("zfs receive failed")
 		return nil, err
 	}
+
+	// validate that we actually received what the sender claimed
+	if err := to.ValidateExists(ctx, lp.ToString()); err != nil {
+		msg := "receive request's `To` version does not match what we received in the stream"
+		getLogger(ctx).WithError(err).WithField("snap", snapFullPath).Error(msg)
+		getLogger(ctx).Error("aborting recv request, but keeping received snapshot for inspection")
+		return nil, errors.Wrap(err, msg)
+	}
+
+	if s.conf.UpdateLastReceivedHold {
+		getLogger(ctx).Debug("move last-received-hold")
+		if err := MoveLastReceivedHold(ctx, lp.ToString(), *to, s.conf.JobID); err != nil {
+			return nil, errors.Wrap(err, "cannot move last-received-hold")
+		}
+	}
+
 	return &pdu.ReceiveRes{}, nil
 }
 
@@ -453,6 +751,19 @@ func (s *Receiver) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapsho
 	return doDestroySnapshots(ctx, lp, req.Snapshots)
 }
 
+func (p *Receiver) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	// we don't move last-received-hold as part of this hint
+	// because that wouldn't give us any benefit wrt resumability.
+	//
+	// Other reason: the replication logic that issues this RPC would require refactoring
+	// to include the receiver's FilesystemVersion in the request)
+	return &pdu.HintMostRecentCommonAncestorRes{}, nil
+}
+
+func (p *Receiver) SendCompleted(context.Context, *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	return &pdu.SendCompletedRes{}, nil
+}
+
 func doDestroySnapshots(ctx context.Context, lp *zfs.DatasetPath, snaps []*pdu.FilesystemVersion) (*pdu.DestroySnapshotsRes, error) {
 	reqs := make([]*zfs.DestroySnapOp, len(snaps))
 	ress := make([]*pdu.DestroySnapshotRes, len(snaps))

endpoint/endpoint_zfs.go

@@ -0,0 +1,503 @@
+package endpoint
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"sort"
+
+	"github.com/kr/pretty"
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var stepHoldTagRE = regexp.MustCompile("^zrepl_STEP_J_(.+)")
+
+func StepHoldTag(jobid JobID) (string, error) {
+	return stepHoldTagImpl(jobid.String())
+}
+
+func stepHoldTagImpl(jobid string) (string, error) {
+	t := fmt.Sprintf("zrepl_STEP_J_%s", jobid)
+	if err := zfs.ValidHoldTag(t); err != nil {
+		return "", err
+	}
+	return t, nil
+}
+
+// err != nil always means that the bookmark is not a step bookmark
+func ParseStepHoldTag(tag string) (JobID, error) {
+	match := stepHoldTagRE.FindStringSubmatch(tag)
+	if match == nil {
+		return JobID{}, fmt.Errorf("parse hold tag: match regex %q", stepHoldTagRE)
+	}
+	jobID, err := MakeJobID(match[1])
+	if err != nil {
+		return JobID{}, errors.Wrap(err, "parse hold tag: invalid job id field")
+	}
+	return jobID, nil
+}
+
+const stepBookmarkNamePrefix = "zrepl_STEP"
+
+// v must be validated by caller
+func StepBookmarkName(fs string, guid uint64, id JobID) (string, error) {
+	return stepBookmarkNameImpl(fs, guid, id.String())
+}
+
+func stepBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
+	return makeJobAndGuidBookmarkName(stepBookmarkNamePrefix, fs, guid, jobid)
+}
+
+// name is the full bookmark name, including dataset path
+//
+// err != nil always means that the bookmark is not a step bookmark
+func ParseStepBookmarkName(fullname string) (guid uint64, jobID JobID, err error) {
+	guid, jobID, err = parseJobAndGuidBookmarkName(fullname, stepBookmarkNamePrefix)
+	if err != nil {
+		err = errors.Wrap(err, "parse step bookmark name") // no shadow!
+	}
+	return guid, jobID, err
+}
+
+const replicationCursorBookmarkNamePrefix = "zrepl_CURSOR"
+
+func ReplicationCursorBookmarkName(fs string, guid uint64, id JobID) (string, error) {
+	return replicationCursorBookmarkNameImpl(fs, guid, id.String())
+}
+
+func replicationCursorBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
+	return makeJobAndGuidBookmarkName(replicationCursorBookmarkNamePrefix, fs, guid, jobid)
+}
+
+var ErrV1ReplicationCursor = fmt.Errorf("bookmark name is a v1-replication cursor")
+
+//err != nil always means that the bookmark is not a valid replication bookmark
+//
+// Returns ErrV1ReplicationCursor as error if the bookmark is a v1 replication cursor
+func ParseReplicationCursorBookmarkName(fullname string) (uint64, JobID, error) {
+
+	// check for legacy cursors
+	{
+		if err := zfs.EntityNamecheck(fullname, zfs.EntityTypeBookmark); err != nil {
+			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name")
+		}
+		_, _, name, err := zfs.DecomposeVersionString(fullname)
+		if err != nil {
+			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name: decompose version string")
+		}
+		const V1ReplicationCursorBookmarkName = "zrepl_replication_cursor"
+		if name == V1ReplicationCursorBookmarkName {
+			return 0, JobID{}, ErrV1ReplicationCursor
+		}
+	}
+
+	guid, jobID, err := parseJobAndGuidBookmarkName(fullname, replicationCursorBookmarkNamePrefix)
+	if err != nil {
+		err = errors.Wrap(err, "parse replication cursor bookmark name") // no shadow
+	}
+	return guid, jobID, err
+}
+
+// may return nil for both values, indicating there is no cursor
+func GetMostRecentReplicationCursorOfJob(ctx context.Context, fs string, jobID JobID) (*zfs.FilesystemVersion, error) {
+	fsp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		return nil, err
+	}
+	candidates, err := GetReplicationCursors(ctx, fsp, jobID)
+	if err != nil || len(candidates) == 0 {
+		return nil, err
+	}
+
+	sort.Slice(candidates, func(i, j int) bool {
+		return candidates[i].CreateTXG < candidates[j].CreateTXG
+	})
+
+	mostRecent := candidates[len(candidates)-1]
+	return &mostRecent, nil
+}
+
+func GetReplicationCursors(ctx context.Context, fs *zfs.DatasetPath, jobID JobID) ([]zfs.FilesystemVersion, error) {
+
+	listOut := &ListHoldsAndBookmarksOutput{}
+	if err := listZFSHoldsAndBookmarksImplFS(ctx, listOut, fs); err != nil {
+		return nil, errors.Wrap(err, "get replication cursor: list bookmarks and holds")
+	}
+
+	if len(listOut.V1ReplicationCursors) > 0 {
+		getLogger(ctx).WithField("bookmark", pretty.Sprint(listOut.V1ReplicationCursors)).
+			Warn("found v1-replication cursor bookmarks, consider running migration 'replication-cursor:v1-v2' after successful replication with this zrepl version")
+	}
+
+	candidates := make([]zfs.FilesystemVersion, 0)
+	for _, v := range listOut.ReplicationCursorBookmarks {
+		zv := zfs.ZFSSendArgVersion{
+			RelName: "#" + v.Name,
+			GUID:    v.Guid,
+		}
+		if err := zv.ValidateExists(ctx, v.FS); err != nil {
+			getLogger(ctx).WithError(err).WithField("bookmark", zv.FullPath(v.FS)).
+				Error("found invalid replication cursor bookmark")
+			continue
+		}
+		candidates = append(candidates, v.v)
+	}
+
+	return candidates, nil
+}
+
+// `target` is validated before replication cursor is set. if validation fails, the cursor is not moved.
+//
+// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
+func MoveReplicationCursor(ctx context.Context, fs string, target *zfs.ZFSSendArgVersion, jobID JobID) (destroyedCursors []zfs.FilesystemVersion, err error) {
+
+	if !target.IsSnapshot() {
+		return nil, zfs.ErrBookmarkCloningNotSupported
+	}
+
+	snapProps, err := target.ValidateExistsAndGetCheckedProps(ctx, fs)
+	if err != nil {
+		return nil, errors.Wrapf(err, "invalid replication cursor target %q (guid=%v)", target.RelName, target.GUID)
+	}
+
+	bookmarkname, err := ReplicationCursorBookmarkName(fs, snapProps.Guid, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "determine replication cursor name")
+	}
+
+	// idempotently create bookmark (guid is encoded in it, hence we'll most likely add a new one
+	// cleanup the old one afterwards
+
+	err = zfs.ZFSBookmark(fs, *target, bookmarkname)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			return nil, err // TODO go1.13 use wrapping
+		}
+		return nil, errors.Wrapf(err, "cannot create bookmark")
+	}
+
+	destroyedCursors, err = DestroyObsoleteReplicationCursors(ctx, fs, target, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "destroy obsolete replication cursors")
+	}
+
+	return destroyedCursors, nil
+}
+
+func DestroyObsoleteReplicationCursors(ctx context.Context, fs string, target *zfs.ZFSSendArgVersion, jobID JobID) (destroyed []zfs.FilesystemVersion, err error) {
+	return destroyBookmarksOlderThan(ctx, fs, target, jobID, func(shortname string) (accept bool) {
+		_, parsedID, err := ParseReplicationCursorBookmarkName(fs + "#" + shortname)
+		return err == nil && parsedID == jobID
+	})
+}
+
+// idempotently hold / step-bookmark `version`
+//
+// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
+func HoldStep(ctx context.Context, fs string, v *zfs.ZFSSendArgVersion, jobID JobID) error {
+	if err := v.ValidateExists(ctx, fs); err != nil {
+		return err
+	}
+	if v.IsSnapshot() {
+
+		tag, err := StepHoldTag(jobID)
+		if err != nil {
+			return errors.Wrap(err, "step hold tag")
+		}
+
+		if err := zfs.ZFSHold(ctx, fs, *v, tag); err != nil {
+			return errors.Wrap(err, "step hold: zfs")
+		}
+
+		return nil
+	}
+
+	v.MustBeBookmark()
+
+	bmname, err := StepBookmarkName(fs, v.GUID, jobID)
+	if err != nil {
+		return errors.Wrap(err, "create step bookmark: determine bookmark name")
+	}
+	// idempotently create bookmark
+	err = zfs.ZFSBookmark(fs, *v, bmname)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			// TODO we could actually try to find a local snapshot that has the requested GUID
+			// 		however, the replication algorithm prefers snapshots anyways, so this quest
+			// 		is most likely not going to be successful. Also, there's the possibility that
+			//      the caller might want to filter what snapshots are eligibile, and this would
+			//      complicate things even further.
+			return err // TODO go1.13 use wrapping
+		}
+		return errors.Wrap(err, "create step bookmark: zfs")
+	}
+	return nil
+}
+
+// idempotently release the step-hold on v if v is a snapshot
+// or idempotently destroy the step-bookmark  of v if v is a bookmark
+//
+// note that this operation leaves v itself untouched, unless v is the step-bookmark itself, in which case v is destroyed
+//
+// returns an instance of *zfs.DatasetDoesNotExist if `v` does not exist
+func ReleaseStep(ctx context.Context, fs string, v *zfs.ZFSSendArgVersion, jobID JobID) error {
+
+	if err := v.ValidateExists(ctx, fs); err != nil {
+		return err
+	}
+
+	if v.IsSnapshot() {
+		tag, err := StepHoldTag(jobID)
+		if err != nil {
+			return errors.Wrap(err, "step release tag")
+		}
+
+		if err := zfs.ZFSRelease(ctx, tag, v.FullPath(fs)); err != nil {
+			return errors.Wrap(err, "step release: zfs")
+		}
+
+		return nil
+	}
+
+	v.MustBeBookmark()
+
+	bmname, err := StepBookmarkName(fs, v.GUID, jobID)
+	if err != nil {
+		return errors.Wrap(err, "step release: determine bookmark name")
+	}
+	// idempotently destroy bookmark
+
+	if err := zfs.ZFSDestroyIdempotent(bmname); err != nil {
+		return errors.Wrap(err, "step release: bookmark destroy: zfs")
+	}
+
+	return nil
+}
+
+// release {step holds, step bookmarks} earlier and including `mostRecent`
+func ReleaseStepAll(ctx context.Context, fs string, mostRecent *zfs.ZFSSendArgVersion, jobID JobID) error {
+
+	if err := mostRecent.ValidateInMemory(fs); err != nil {
+		return err
+	}
+
+	tag, err := StepHoldTag(jobID)
+	if err != nil {
+		return errors.Wrap(err, "step release all: tag")
+	}
+
+	err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, mostRecent.GUID, tag)
+	if err != nil {
+		return errors.Wrapf(err, "step release all: release holds older and including %q", mostRecent.FullPath(fs))
+	}
+
+	_, err = destroyBookmarksOlderThan(ctx, fs, mostRecent, jobID, func(shortname string) bool {
+		_, parsedId, parseErr := ParseStepBookmarkName(fs + "#" + shortname)
+		return parseErr == nil && parsedId == jobID
+	})
+	if err != nil {
+		return errors.Wrapf(err, "step release all: destroy bookmarks older than %q", mostRecent.FullPath(fs))
+	}
+
+	return nil
+}
+
+var lastReceivedHoldTagRE = regexp.MustCompile("^zrepl_last_received_J_(.+)$")
+
+// err != nil always means that the bookmark is not a step bookmark
+func ParseLastReceivedHoldTag(tag string) (JobID, error) {
+	match := lastReceivedHoldTagRE.FindStringSubmatch(tag)
+	if match == nil {
+		return JobID{}, errors.Errorf("parse last-received-hold tag: does not match regex %s", lastReceivedHoldTagRE.String())
+	}
+	jobId, err := MakeJobID(match[1])
+	if err != nil {
+		return JobID{}, errors.Wrap(err, "parse last-received-hold tag: invalid job id field")
+	}
+	return jobId, nil
+}
+
+func LastReceivedHoldTag(jobID JobID) (string, error) {
+	return lastReceivedHoldImpl(jobID.String())
+}
+
+func lastReceivedHoldImpl(jobid string) (string, error) {
+	tag := fmt.Sprintf("zrepl_last_received_J_%s", jobid)
+	if err := zfs.ValidHoldTag(tag); err != nil {
+		return "", err
+	}
+	return tag, nil
+}
+
+func MoveLastReceivedHold(ctx context.Context, fs string, to zfs.ZFSSendArgVersion, jobID JobID) error {
+	if err := to.ValidateExists(ctx, fs); err != nil {
+		return err
+	}
+	if err := zfs.EntityNamecheck(to.FullPath(fs), zfs.EntityTypeSnapshot); err != nil {
+		return err
+	}
+
+	tag, err := LastReceivedHoldTag(jobID)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: hold tag")
+	}
+
+	// we never want to be without a hold
+	// => hold new one before releasing old hold
+
+	err = zfs.ZFSHold(ctx, fs, to, tag)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: hold newly received")
+	}
+
+	err = zfs.ZFSReleaseAllOlderThanGUID(ctx, fs, to.GUID, tag)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: release older holds")
+	}
+
+	return nil
+}
+
+type ListHoldsAndBookmarksOutputBookmarkV1ReplicationCursor struct {
+	FS   string
+	Name string
+}
+
+type ListHoldsAndBookmarksOutput struct {
+	StepBookmarks []*ListHoldsAndBookmarksOutputBookmark
+	StepHolds     []*ListHoldsAndBookmarksOutputHold
+
+	ReplicationCursorBookmarks []*ListHoldsAndBookmarksOutputBookmark
+	V1ReplicationCursors       []*ListHoldsAndBookmarksOutputBookmarkV1ReplicationCursor
+	LastReceivedHolds          []*ListHoldsAndBookmarksOutputHold
+}
+
+type ListHoldsAndBookmarksOutputBookmark struct {
+	FS, Name string
+	Guid     uint64
+	JobID    JobID
+	v        zfs.FilesystemVersion
+}
+
+type ListHoldsAndBookmarksOutputHold struct {
+	FS            string
+	Snap          string
+	SnapGuid      uint64
+	SnapCreateTXG uint64
+	Tag           string
+	JobID         JobID
+}
+
+// List all holds and bookmarks managed by endpoint
+func ListZFSHoldsAndBookmarks(ctx context.Context, fsfilter zfs.DatasetFilter) (*ListHoldsAndBookmarksOutput, error) {
+
+	// initialize all fields so that JSON serializion of output looks pretty (see client/holds.go)
+	// however, listZFSHoldsAndBookmarksImplFS shouldn't rely on it
+	out := &ListHoldsAndBookmarksOutput{
+		StepBookmarks:              make([]*ListHoldsAndBookmarksOutputBookmark, 0),
+		StepHolds:                  make([]*ListHoldsAndBookmarksOutputHold, 0),
+		ReplicationCursorBookmarks: make([]*ListHoldsAndBookmarksOutputBookmark, 0),
+		V1ReplicationCursors:       make([]*ListHoldsAndBookmarksOutputBookmarkV1ReplicationCursor, 0),
+		LastReceivedHolds:          make([]*ListHoldsAndBookmarksOutputHold, 0),
+	}
+
+	fss, err := zfs.ZFSListMapping(ctx, fsfilter)
+	if err != nil {
+		return nil, errors.Wrap(err, "list filesystems")
+	}
+
+	for _, fs := range fss {
+		err := listZFSHoldsAndBookmarksImplFS(ctx, out, fs)
+		if err != nil {
+			return nil, errors.Wrapf(err, "list holds and bookmarks on %q", fs.ToString())
+		}
+	}
+	return out, nil
+}
+
+func listZFSHoldsAndBookmarksImplFS(ctx context.Context, out *ListHoldsAndBookmarksOutput, fs *zfs.DatasetPath) error {
+	fsvs, err := zfs.ZFSListFilesystemVersions(fs, nil)
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs)
+	}
+	for _, v := range fsvs {
+		switch v.Type {
+		case zfs.Bookmark:
+			listZFSHoldsAndBookmarksImplTryParseBookmark(ctx, out, fs, v)
+		case zfs.Snapshot:
+			holds, err := zfs.ZFSHolds(ctx, fs.ToString(), v.Name)
+			if err != nil {
+				return errors.Wrapf(err, "get holds of %q", v.ToAbsPath(fs))
+			}
+			for _, tag := range holds {
+				listZFSHoldsAndBookmarksImplSnapshotTryParseHold(ctx, out, fs, v, tag)
+			}
+		default:
+			continue
+		}
+	}
+	return nil
+}
+
+// pure function, err != nil always indicates parsing error
+func listZFSHoldsAndBookmarksImplTryParseBookmark(ctx context.Context, out *ListHoldsAndBookmarksOutput, fs *zfs.DatasetPath, v zfs.FilesystemVersion) {
+	var err error
+
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+
+	fullname := v.ToAbsPath(fs)
+
+	bm := &ListHoldsAndBookmarksOutputBookmark{
+		FS: fs.ToString(), Name: v.Name, v: v,
+	}
+	bm.Guid, bm.JobID, err = ParseStepBookmarkName(fullname)
+	if err == nil {
+		out.StepBookmarks = append(out.StepBookmarks, bm)
+		return
+	}
+
+	bm.Guid, bm.JobID, err = ParseReplicationCursorBookmarkName(fullname)
+	if err == nil {
+		out.ReplicationCursorBookmarks = append(out.ReplicationCursorBookmarks, bm)
+		return
+	} else if err == ErrV1ReplicationCursor {
+		v1rc := &ListHoldsAndBookmarksOutputBookmarkV1ReplicationCursor{
+			FS: fs.ToString(), Name: v.Name,
+		}
+		out.V1ReplicationCursors = append(out.V1ReplicationCursors, v1rc)
+		return
+	}
+}
+
+// pure function, err != nil always indicates parsing error
+func listZFSHoldsAndBookmarksImplSnapshotTryParseHold(ctx context.Context, out *ListHoldsAndBookmarksOutput, fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) {
+	var err error
+
+	if v.Type != zfs.Snapshot {
+		panic("impl error")
+	}
+
+	hold := &ListHoldsAndBookmarksOutputHold{
+		FS:            fs.ToString(),
+		Snap:          v.Name,
+		SnapGuid:      v.Guid,
+		SnapCreateTXG: v.CreateTXG,
+		Tag:           holdTag,
+	}
+	hold.JobID, err = ParseStepHoldTag(holdTag)
+	if err == nil {
+		out.StepHolds = append(out.StepHolds, hold)
+		return
+	}
+
+	hold.JobID, err = ParseLastReceivedHoldTag(holdTag)
+	if err == nil {
+		out.LastReceivedHolds = append(out.LastReceivedHolds, hold)
+		return
+	}
+
+}

endpoint/endpoint_zfs_helpers.go

@@ -0,0 +1,109 @@
+package endpoint
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+	"strconv"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// returns the short name (no fs# prefix)
+func makeJobAndGuidBookmarkName(prefix string, fs string, guid uint64, jobid string) (string, error) {
+	bmname := fmt.Sprintf(prefix+"_G_%016x_J_%s", guid, jobid)
+	if err := zfs.EntityNamecheck(fmt.Sprintf("%s#%s", fs, bmname), zfs.EntityTypeBookmark); err != nil {
+		return "", err
+	}
+	return bmname, nil
+}
+
+var jobAndGuidBookmarkRE = regexp.MustCompile(`(.+)_G_([0-9a-f]{16})_J_(.+)$`)
+
+func parseJobAndGuidBookmarkName(fullname string, prefix string) (guid uint64, jobID JobID, _ error) {
+
+	if len(prefix) == 0 {
+		panic("prefix must not be empty")
+	}
+
+	if err := zfs.EntityNamecheck(fullname, zfs.EntityTypeBookmark); err != nil {
+		return 0, JobID{}, err
+	}
+
+	_, _, name, err := zfs.DecomposeVersionString(fullname)
+	if err != nil {
+		return 0, JobID{}, errors.Wrap(err, "decompose bookmark name")
+	}
+
+	match := jobAndGuidBookmarkRE.FindStringSubmatch(name)
+	if match == nil {
+		return 0, JobID{}, errors.Errorf("bookmark name does not match regex %q", jobAndGuidBookmarkRE.String())
+	}
+	if match[1] != prefix {
+		return 0, JobID{}, errors.Errorf("prefix component does not match: expected %q, got %q", prefix, match[1])
+	}
+
+	guid, err = strconv.ParseUint(match[2], 16, 64)
+	if err != nil {
+		return 0, JobID{}, errors.Wrapf(err, "parse guid component: %q", match[2])
+	}
+
+	jobID, err = MakeJobID(match[3])
+	if err != nil {
+		return 0, JobID{}, errors.Wrapf(err, "parse jobid component: %q", match[3])
+	}
+
+	return guid, jobID, nil
+}
+
+func destroyBookmarksOlderThan(ctx context.Context, fs string, mostRecent *zfs.ZFSSendArgVersion, jobID JobID, filter func(shortname string) (accept bool)) (destroyed []zfs.FilesystemVersion, err error) {
+	if filter == nil {
+		panic(filter)
+	}
+
+	fsp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid filesystem path")
+	}
+
+	mostRecentProps, err := mostRecent.ValidateExistsAndGetCheckedProps(ctx, fs)
+	if err != nil {
+		return nil, errors.Wrap(err, "validate most recent version argument")
+	}
+
+	stepBookmarks, err := zfs.ZFSListFilesystemVersions(fsp, zfs.FilterFromClosure(
+		func(t zfs.VersionType, name string) (accept bool, err error) {
+			if t != zfs.Bookmark {
+				return false, nil
+			}
+			return filter(name), nil
+		}))
+	if err != nil {
+		return nil, errors.Wrap(err, "list bookmarks")
+	}
+
+	// cut off all bookmarks prior to mostRecent's CreateTXG
+	var destroy []zfs.FilesystemVersion
+	for _, v := range stepBookmarks {
+		if v.Type != zfs.Bookmark {
+			panic("implementation error")
+		}
+		if !filter(v.Name) {
+			panic("inconsistent filter result")
+		}
+		if v.CreateTXG < mostRecentProps.CreateTXG {
+			destroy = append(destroy, v)
+		}
+	}
+
+	// FIXME use batch destroy, must adopt code to handle bookmarks
+	for _, v := range destroy {
+		if err := zfs.ZFSDestroyIdempotent(v.ToAbsPath(fsp)); err != nil {
+			return nil, errors.Wrap(err, "destroy bookmark")
+		}
+	}
+
+	return destroy, nil
+}

endpoint/endpoint_zfs_helpers_test.go

@@ -0,0 +1,51 @@
+package endpoint
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseJobAndGuidBookmarkName(t *testing.T) {
+
+	type Case struct {
+		input     string
+		expectErr bool
+
+		guid  uint64
+		jobid string
+	}
+
+	cases := []Case{
+		{
+			`p1/sync#zrepl_CURSOR_G_932f3a7089080ce2_J_push with legitimate name`,
+			false, 0x932f3a7089080ce2, "push with legitimate name",
+		},
+		{
+			input:     `p1/sync#zrepl_CURSOR_G_932f3a7089_J_push with legitimate name`,
+			expectErr: true,
+		},
+		{
+			input:     `p1/sync#zrepl_CURSOR_G_932f3a7089080ce2_J_push with il\tlegitimate name`,
+			expectErr: true,
+		},
+		{
+			input:     `p1/sync#otherprefix_G_932f3a7089080ce2_J_push with legitimate name`,
+			expectErr: true,
+		},
+	}
+
+	for i := range cases {
+		t.Run(cases[i].input, func(t *testing.T) {
+			guid, jobid, err := parseJobAndGuidBookmarkName(cases[i].input, replicationCursorBookmarkNamePrefix)
+			if cases[i].expectErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, cases[i].guid, guid)
+				assert.Equal(t, MustMakeJobID(cases[i].jobid), jobid)
+			}
+		})
+	}
+
+}

endpoint/jobid.go

@@ -0,0 +1,78 @@
+package endpoint
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// An instance of this type returned by MakeJobID guarantees
+// that that instance's JobID.String() can be used in a ZFS dataset name and hold tag.
+type JobID struct {
+	jid string
+}
+
+func MakeJobID(s string) (JobID, error) {
+	if len(s) == 0 {
+		return JobID{}, fmt.Errorf("must not be empty string")
+	}
+
+	if err := zfs.ComponentNamecheck(s); err != nil {
+		return JobID{}, errors.Wrap(err, "muse be usable as a dataset path component")
+	}
+
+	if _, err := stepBookmarkNameImpl("pool/ds", 0xface601d, s); err != nil {
+		// note that this might still fail due to total maximum name length, but we can't enforce that
+		return JobID{}, errors.Wrap(err, "must be usable for a step bookmark")
+	}
+
+	if _, err := stepHoldTagImpl(s); err != nil {
+		return JobID{}, errors.Wrap(err, "must be usable for a step hold tag")
+	}
+
+	if _, err := lastReceivedHoldImpl(s); err != nil {
+		return JobID{}, errors.Wrap(err, "must be usabel as a last-recieved-hold tag")
+	}
+
+	// FIXME replication cursor bookmark name
+
+	_, err := zfs.NewDatasetPath(s)
+	if err != nil {
+		return JobID{}, fmt.Errorf("must be usable in a ZFS dataset path: %s", err)
+	}
+
+	return JobID{s}, nil
+}
+
+func MustMakeJobID(s string) JobID {
+	jid, err := MakeJobID(s)
+	if err != nil {
+		panic(err)
+	}
+	return jid
+}
+
+func (j JobID) expectInitialized() {
+	if j.jid == "" {
+		panic("use of unitialized JobID")
+	}
+}
+
+func (j JobID) String() string {
+	j.expectInitialized()
+	return j.jid
+}
+
+var _ json.Marshaler = JobID{}
+var _ json.Unmarshaler = (*JobID)(nil)
+
+func (j JobID) MarshalJSON() ([]byte, error) { return json.Marshal(j.jid) }
+
+func (j *JobID) UnmarshalJSON(b []byte) error {
+	return json.Unmarshal(b, &j.jid)
+}
+
+func (j JobID) MustValidate() { j.expectInitialized() }

go.mod

@@ -3,80 +3,35 @@ module github.com/zrepl/zrepl
 go 1.12
 
 require (
-	github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
-	github.com/alvaroloes/enumer v1.1.1
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/fatih/color v1.7.0
-	github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff // indirect
 	github.com/gdamore/tcell v1.2.0
-	github.com/go-critic/go-critic v0.3.4 // indirect
-	github.com/go-logfmt/logfmt v0.3.0
+	github.com/go-logfmt/logfmt v0.4.0
 	github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/mock v1.2.0 // indirect
-	github.com/golang/protobuf v1.2.0
-	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
-	github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
-	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
-	github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
-	github.com/golangci/golangci-lint v1.17.1
-	github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
-	github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
-	github.com/golangci/misspell v0.3.4 // indirect
-	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/golang/protobuf v1.3.2
 	github.com/google/uuid v1.1.1
 	github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8
-	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 // indirect
 	github.com/kr/pretty v0.1.0
 	github.com/lib/pq v1.2.0
-	github.com/mattn/go-isatty v0.0.3
-	github.com/matttproud/golang_protobuf_extensions v1.0.0 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.4 // indirect
+	github.com/mattn/go-isatty v0.0.8
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // // go1.12 mod tidy adds this dependency as 'indirect', but go1.13 mod tidy removes it if the trailing comment is 'indirect' => add this comment to make the build work without changing go.mod on both go1.12 and go1.13
+	github.com/modern-go/reflect2 v1.0.1 // go1.12 mod tidy adds this dependency as 'indirect', but go1.13 mod tidy removes it if the trailing comment is 'indirect' => add this comment to make the build work without changing go.mod on both go1.12 and go1.13
 	github.com/montanaflynn/stats v0.5.0
-	github.com/pkg/errors v0.8.0
+	github.com/onsi/ginkgo v1.10.2 // indirect
+	github.com/onsi/gomega v1.7.0 // indirect
+	github.com/pkg/errors v0.8.1
 	github.com/pkg/profile v1.2.1
-	github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284
-	github.com/prometheus/client_golang v0.0.0-20180410130117-e11c6ff8170b
-	github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 // indirect
-	github.com/prometheus/common v0.0.0-20180413074202-d0f7cd64bda4 // indirect
-	github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d // indirect
-	github.com/sergi/go-diff v1.0.0 // indirect
-	github.com/sirupsen/logrus v1.4.0 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
+	github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7
+	github.com/prometheus/client_golang v1.2.1
+	github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 // go1.12 thinks it needs this
 	github.com/spf13/cobra v0.0.2
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.3.2 // indirect
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
-	github.com/theckman/goconstraint v1.11.0 // indirect
 	github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d
-	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
-	github.com/yudai/pp v2.0.1+incompatible // indirect
-	github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f
-	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
-	golang.org/x/net v0.0.0-20190313220215-9f648a60d977
+	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // go1.12 thinks it needs this
+	github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd
+	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756
-	golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
-	google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 // indirect
+	golang.org/x/sys v0.0.0-20191026070338-33540a1f6037
 	google.golang.org/grpc v1.17.0
-	mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
-	sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
-)
-
-// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
-replace (
-	github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
-	github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
-	github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
-	github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
-	github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
-	github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
-	github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
-	github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
-	github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
-	golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
-	mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
 )

go.sum

@@ -1,17 +1,23 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
-github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
 github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
-github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
@@ -30,107 +36,82 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
 github.com/gdamore/tcell v1.2.0 h1:ikixzsxc8K8o3V2/CEmyoEW8mJZaNYQQ3NP3VIQdUe4=
 github.com/gdamore/tcell v1.2.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
-github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
 github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
-github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
 github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4 h1:0suja/iKSDbEIYLbrS/8C7iArJiWpgCNcR+zwAHu7Ig=
 github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
-github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
-github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
 github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
-github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
 github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
 github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
-github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
 github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
 github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
-github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
 github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
 github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
 github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
-github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
 github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
-github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
 github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
-github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
 github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
 github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
-github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
 github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
 github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
-github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
-github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
-github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
-github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
 github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
-github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
-github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
 github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
 github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
-github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
 github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
 github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
 github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
-github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
 github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
 github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
-github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
 github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
-github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
 github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
 github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
 github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
-github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
 github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
-github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -138,6 +119,9 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8 h1:+dKzeuiDYbD/Cfi/sEkkNrs6z4/WZ8ZIR8dAbUpjXbU=
 github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -149,7 +133,6 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -164,102 +147,125 @@ github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQ
 github.com/lucasb-eyer/go-colorful v1.0.2 h1:mCMFu6PgSozg9tDNMMK3g18oJBX7oYGrC09mS6CXfO4=
 github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
 github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
-github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLzM9Y858MNGCOACTvCW9TSAc=
 github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
 github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk=
 github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
-github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
 github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
+github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
 github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284 h1:GX8YFBfSL6GlKymQZOsQllPbxVamfEYpis7oaJLXYZA=
 github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284/go.mod h1:XMR9fjP5/wMyFK0Ifk9ZHDFem4Rukm+ChvoYr2nxrgw=
-github.com/prometheus/client_golang v0.0.0-20180410130117-e11c6ff8170b h1:ymnAeobXUW0sfmt5n8euMzW4vgdQvZNTagdYMcpK63g=
+github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1 h1:HH8yzlaZq/A8xdJaSj/eu32yFFySWAJMTmOQPAGwfYg=
+github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1/go.mod h1:JRs3nyBjvOv/9YHC+Vlw9z1Jfzl5s5t0BNnTzmclj1c=
+github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7 h1:Oik8tLrLhoMq4fduDRuNNOAQ+q0M0dXkjAYLUf4+mAc=
+github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7/go.mod h1:Ba6RaFpK1+IHWs1wLZ6sCBuXkt4iAVLeOG5GinXHusM=
 github.com/prometheus/client_golang v0.0.0-20180410130117-e11c6ff8170b/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 h1:cLL6NowurKLMfCeQy4tIeph12XNQWgANCNvdyrOYKV4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20180413074202-d0f7cd64bda4 h1:rPmL0pCWefiqV7RjMYuBUrLbMJmNVjRKz82caELPTUQ=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180413074202-d0f7cd64bda4/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d h1:RCcsxyRr6+/pLg6wr0cUjPovhEhSNOtPh0SOz6u3hGU=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc=
+github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
-github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
-github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
 github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
 github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
-github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/theckman/goconstraint v1.11.0 h1:oBUwN5wpE4dwyPhRGraEgJsFTr+JtLWiDnaJZJeeXI0=
 github.com/theckman/goconstraint v1.11.0/go.mod h1:zkCR/f2kOULTk/h1ujgyB9BlCNLaqlQ6GN2Zl4mg81g=
-github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
 github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@@ -273,15 +279,15 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3Ifn
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
-github.com/zrepl/yaml-config v0.0.0-20190903123849-809f4603401a h1:l37btErUqI1yNmt5mf8kzglbeQfo32+TRacnxthiuck=
-github.com/zrepl/yaml-config v0.0.0-20190903123849-809f4603401a/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
 github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f h1:3MuiGfgMHCSwKUcsuI7ODbi50j+evTB7SsoOBMNC5Fk=
 github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
+github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd h1:SSo67WLS+99QESvbW8Meibz7zCrxshP71U9dH5KOCXM=
+github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
+github.com/zrepl/zrepl v0.2.0/go.mod h1:M3Zv2IGSO8iYpUjsZD6ayZ2LHy7zyMfzet9XatKOrZ8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -290,22 +296,33 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756 h1:9nuHUbU8dRnRRfj9KjWUVrJeoexdbeMjttk6Oh1rD10=
 golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -316,15 +333,15 @@ golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGm
 golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
 golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
 golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 h1:yvw+zsSmSM02Z5H3ZdEV7B7Ql7eFrjQTnmByJvK+3J8=
 google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
@@ -332,6 +349,7 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M
 google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -346,13 +364,9 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
-mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
-mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
 mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
-sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
 sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

lazy.sh

@@ -27,7 +27,8 @@ fi
 CHECKOUTPATH="${GOPATH}/src/github.com/zrepl/zrepl"
 
 godep() {
-    step "install build dependencies (versions pinned using go.mod and tools.go)"
+    step "install build dependencies (versions pinned in build/go.mod and build/tools.go)"
+    pushd "$(dirname "${BASH_SOURCE[0]}")"/build
     set -x
     export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
     go build -v -mod=readonly -o "$GOPATH/bin/stringer"      golang.org/x/tools/cmd/stringer
@@ -36,6 +37,7 @@ godep() {
     go build -v -mod=readonly -o "$GOPATH/bin/goimports"     golang.org/x/tools/cmd/goimports
     go build -v -mod=readonly -o "$GOPATH/bin/golangci-lint" github.com/golangci/golangci-lint/cmd/golangci-lint
     set +x
+    popd
     if ! type stringer || ! type protoc-gen-go || ! type enumer || ! type goimports || ! type golangci-lint; then
         echo "Installed dependencies but can't find them in \$PATH, adjust it to contain \$GOPATH/bin" 1>&2
         exit 1

main.go

@@ -17,6 +17,7 @@ func init() {
 	cli.AddSubcommand(client.PprofCmd)
 	cli.AddSubcommand(client.TestCmd)
 	cli.AddSubcommand(client.MigrateCmd)
+	cli.AddSubcommand(client.HoldsCmd)
 }
 
 func main() {

platformtest/harness/harness.go

@@ -4,11 +4,14 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"os"
 	"path/filepath"
+	"regexp"
 	"time"
 
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
@@ -16,17 +19,38 @@ import (
 	"github.com/zrepl/zrepl/platformtest/tests"
 )
 
-func main() {
+var bold = color.New(color.Bold)
+var boldRed = color.New(color.Bold, color.FgHiRed)
+var boldGreen = color.New(color.Bold, color.FgHiGreen)
 
-	var args struct {
+var args struct {
 	createArgs            platformtest.ZpoolCreateArgs
+	stopAndKeepPoolOnFail bool
+
+	run   string
+	runRE *regexp.Regexp
+}
+
+func main() {
+	if err := doMain(); err != nil {
+		os.Exit(1)
 	}
+}
+
+var exitWithErr = fmt.Errorf("exit with error")
+
+func doMain() error {
+
 	flag.StringVar(&args.createArgs.PoolName, "poolname", "", "")
 	flag.StringVar(&args.createArgs.ImagePath, "imagepath", "", "")
-	flag.Int64Var(&args.createArgs.ImageSize, "imagesize", 100*(1<<20), "")
-	flag.StringVar(&args.createArgs.Mountpoint, "mountpoint", "none", "")
+	flag.Int64Var(&args.createArgs.ImageSize, "imagesize", 200*(1<<20), "")
+	flag.StringVar(&args.createArgs.Mountpoint, "mountpoint", "", "")
+	flag.BoolVar(&args.stopAndKeepPoolOnFail, "failure.stop-and-keep-pool", false, "if a test case fails, stop test execution and keep pool as it was when the test failed")
+	flag.StringVar(&args.run, "run", "", "")
 	flag.Parse()
 
+	args.runRE = regexp.MustCompile(args.run)
+
 	outlets := logger.NewOutlets()
 	outlet, level, err := logging.ParseOutlet(config.LoggingOutletEnum{Ret: &config.StdoutLoggingOutlet{
 		LoggingOutletCommon: config.LoggingOutletCommon{
@@ -47,33 +71,132 @@ func main() {
 	ctx := platformtest.WithLogger(context.Background(), logger)
 	ex := platformtest.NewEx(logger)
 
-	bold := color.New(color.Bold)
+	type invocation struct {
+		runFunc tests.Case
+		result  *testCaseResult
+	}
+
+	invocations := make([]*invocation, 0, len(tests.Cases))
 	for _, c := range tests.Cases {
-		// ATTENTION future parallelism must pass c by value into closure!
-		err := func() error {
-			bold.Printf("BEGIN TEST CASE %s\n", c)
+		if args.runRE.MatchString(c.String()) {
+			invocations = append(invocations, &invocation{runFunc: c})
+		}
+	}
+
+	for _, inv := range invocations {
+
+		bold.Printf("BEGIN TEST CASE %s\n", inv.runFunc.String())
+
 		pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.createArgs)
 		if err != nil {
-				return errors.Wrap(err, "create test pool")
+			panic(errors.Wrap(err, "create test pool"))
 		}
-			defer func() {
-				if err := pool.Destroy(ctx, ex); err != nil {
-					fmt.Printf("error destroying test pool: %s", err)
-				}
-			}()
+
 		ctx := &platformtest.Context{
 			Context:     ctx,
 			RootDataset: filepath.Join(pool.Name(), "rootds"),
 		}
-			c(ctx)
-			bold.Printf("DONE  TEST CASE %s\n", c)
-			fmt.Println()
-			return nil
-		}()
-		if err != nil {
-			bold.Printf("TEST CASE FAILED WITH ERROR:\n")
-			fmt.Println(err)
+
+		res := runTestCase(ctx, ex, inv.runFunc)
+		inv.result = res
+		if res.failed {
+			fmt.Printf("%+v\n", res.failedStack) // print with stack trace
 		}
+
+		if res.failed && args.stopAndKeepPoolOnFail {
+			boldRed.Printf("STOPPING TEST RUN AT FAILING TEST PER USER REQUEST\n")
+			return exitWithErr
+		}
+
+		if err := pool.Destroy(ctx, ex); err != nil {
+			panic(fmt.Sprintf("error destroying test pool: %s", err))
+		}
+
+		if res.failed {
+			boldRed.Printf("TEST FAILED\n")
+		} else if res.skipped {
+			bold.Printf("TEST SKIPPED\n")
+		} else if res.succeeded {
+			boldGreen.Printf("TEST PASSED\n")
+		} else {
+			panic("unreachable")
+		}
+
+		fmt.Println()
+	}
+
+	var summary struct {
+		succ, fail, skip []*invocation
+	}
+	for _, inv := range invocations {
+		var bucket *[]*invocation
+		if inv.result.failed {
+			bucket = &summary.fail
+		} else if inv.result.skipped {
+			bucket = &summary.skip
+		} else if inv.result.succeeded {
+			bucket = &summary.succ
+		} else {
+			panic("unreachable")
+		}
+		*bucket = append(*bucket, inv)
+	}
+	printBucket := func(bucketName string, c *color.Color, bucket []*invocation) {
+		c.Printf("%s:", bucketName)
+		if len(bucket) == 0 {
+			fmt.Printf(" []\n")
+			return
+		}
+		fmt.Printf("\n")
+		for _, inv := range bucket {
+			fmt.Printf("  %s\n", inv.runFunc.String())
+		}
+	}
+	printBucket("PASSING TESTS", boldGreen, summary.succ)
+	printBucket("SKIPPED TESTS", bold, summary.skip)
+	printBucket("FAILED TESTS", boldRed, summary.fail)
+
+	if len(summary.fail) > 0 {
+		return errors.New("at least one test failed")
+	}
+	return nil
+}
+
+type testCaseResult struct {
+	// oneof
+	failed, skipped, succeeded bool
+
+	failedStack error // has stack inside, valid if failed=true
+}
+
+func runTestCase(ctx *platformtest.Context, ex platformtest.Execer, c tests.Case) *testCaseResult {
+
+	// run case
+	var paniced = false
+	var panicValue interface{} = nil
+	var panicStack error
+	func() {
+		defer func() {
+			if item := recover(); item != nil {
+				panicValue = item
+				paniced = true
+				panicStack = errors.Errorf("panic while running test: %v", panicValue)
+			}
+		}()
+		c(ctx)
+	}()
+
+	if paniced {
+		switch panicValue {
+		case platformtest.SkipNowSentinel:
+			return &testCaseResult{skipped: true}
+		case platformtest.FailNowSentinel:
+			return &testCaseResult{failed: true, failedStack: panicStack}
+		default:
+			return &testCaseResult{failed: true, failedStack: panicStack}
+		}
+	} else {
+		return &testCaseResult{succeeded: true}
 	}
 
 }

platformtest/platformtest.go

@@ -2,6 +2,7 @@ package platformtest
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -12,13 +13,26 @@ type Context struct {
 	RootDataset string
 }
 
+var FailNowSentinel = fmt.Errorf("platformtest: FailNow called on context")
+
+var SkipNowSentinel = fmt.Errorf("platformtest: SkipNow called on context")
+
 var _ assert.TestingT = (*Context)(nil)
 var _ require.TestingT = (*Context)(nil)
 
+func (c *Context) Logf(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	GetLog(c).Info(msg)
+}
+
 func (c *Context) Errorf(format string, args ...interface{}) {
-	getLog(c).Printf(format, args...)
+	GetLog(c).Printf(format, args...)
 }
 
 func (c *Context) FailNow() {
-	panic(nil)
+	panic(FailNowSentinel)
+}
+
+func (c *Context) SkipNow() {
+	panic(SkipNowSentinel)
 }

platformtest/platformtest_exec.go

@@ -32,7 +32,7 @@ func (e *ex) runNoOutput(expectSuccess bool, ctx context.Context, cmd string, ar
 	buf, _ := circlog.NewCircularLog(32 << 10)
 	ecmd.Stdout, ecmd.Stderr = buf, buf
 	err := ecmd.Run()
-	log.Printf("command output: %s", buf.String())
+	log.WithField("output", buf.String()).Debug("command output")
 	if _, ok := err.(*exec.ExitError); err != nil && !ok {
 		panic(err)
 	}

platformtest/platformtest_logging.go

@@ -19,6 +19,6 @@ func WithLogger(ctx context.Context, logger Logger) context.Context {
 	return ctx
 }
 
-func getLog(ctx context.Context) Logger {
+func GetLog(ctx context.Context) Logger {
 	return ctx.Value(contextKeyLogger).(Logger)
 }

platformtest/platformtest_ops.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
@@ -39,7 +40,7 @@ type DestroyRootOp struct {
 func (o *DestroyRootOp) Run(ctx context.Context, e Execer) error {
 	// early-exit if it doesn't exist
 	if err := e.RunExpectSuccessNoOutput(ctx, "zfs", "get", "-H", "name", o.Path); err != nil {
-		getLog(ctx).WithField("root_ds", o.Path).Info("assume root ds doesn't exist")
+		GetLog(ctx).WithField("root_ds", o.Path).Info("assume root ds doesn't exist")
 		return nil
 	}
 	return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", "-r", o.Path)
@@ -48,6 +49,7 @@ func (o *DestroyRootOp) Run(ctx context.Context, e Execer) error {
 type FSOp struct {
 	Op        Op
 	Path      string
+	Encrypted bool // only for Op=Add
 }
 
 func (o *FSOp) Run(ctx context.Context, e Execer) error {
@@ -57,7 +59,22 @@ func (o *FSOp) Run(ctx context.Context, e Execer) error {
 	case AssertNotExists:
 		return e.RunExpectFailureNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
 	case Add:
-		return e.RunExpectSuccessNoOutput(ctx, "zfs", "create", o.Path)
+		opts := []string{"create"}
+		if o.Encrypted {
+			const passphraseFilePath = "/tmp/zreplplatformtest.encryption.passphrase"
+			const passphrase = "foobar2342"
+			err := ioutil.WriteFile(passphraseFilePath, []byte(passphrase), 0600)
+			if err != nil {
+				panic(err)
+			}
+			opts = append(opts,
+				"-o", "encryption=on",
+				"-o", "keylocation=file:///"+passphraseFilePath,
+				"-o", "keyformat=passphrase",
+			)
+		}
+		opts = append(opts, o.Path)
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", opts...)
 	case Del:
 		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Path)
 	default:
@@ -94,7 +111,7 @@ func (o *RunOp) Run(ctx context.Context, e Execer) error {
 	cmd := exec.CommandContext(ctx, "/usr/bin/env", "bash", "-c", o.Script)
 	cmd.Env = os.Environ()
 	cmd.Env = append(cmd.Env, fmt.Sprintf("ROOTDS=%s", o.RootDS))
-	log := getLog(ctx).WithField("script", o.Script)
+	log := GetLog(ctx).WithField("script", o.Script)
 	log.Info("start script")
 	defer log.Info("script done")
 	output, err := cmd.CombinedOutput()
@@ -126,7 +143,7 @@ func Run(ctx context.Context, rk RunKind, rootds string, stmtsStr string) {
 	if err != nil {
 		panic(err)
 	}
-	execer := NewEx(getLog(ctx))
+	execer := NewEx(GetLog(ctx))
 	for _, s := range stmt {
 		err := s.Run(ctx, execer)
 		if err == nil {
@@ -249,7 +266,25 @@ nextLine:
 		if strings.ContainsAny(comps.Text(), "@") {
 			stmts = append(stmts, &SnapOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
 		} else {
-			stmts = append(stmts, &FSOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+			// FS
+			fs := comps.Text()
+			var encrypted bool = false
+			if op == Add {
+				if comps.Scan() {
+					t := comps.Text()
+					switch t {
+					case "encrypted":
+						encrypted = true
+					default:
+						panic(fmt.Sprintf("unexpected token %q", t))
+					}
+				}
+			}
+			stmts = append(stmts, &FSOp{
+				Op:        op,
+				Path:      fmt.Sprintf("%s/%s", rootds, fs),
+				Encrypted: encrypted,
+			})
 		}
 
 		if comps.Scan() {

platformtest/platformtest_zpool.go

@@ -2,10 +2,12 @@ package platformtest
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -28,8 +30,8 @@ func (a ZpoolCreateArgs) Validate() error {
 	if a.ImageSize < minImageSize {
 		return errors.Errorf("ImageSize must be > %v, got %v", minImageSize, a.ImageSize)
 	}
-	if a.Mountpoint != "none" {
-		return errors.Errorf("Mountpoint must be \"none\"")
+	if a.Mountpoint == "" || a.Mountpoint[0] != '/' {
+		return errors.Errorf("Mountpoint must be an absolute path to a directory")
 	}
 	if a.PoolName == "" {
 		return errors.Errorf("PoolName must not be emtpy")
@@ -68,7 +70,10 @@ func CreateOrReplaceZpool(ctx context.Context, e Execer, args ZpoolCreateArgs) (
 	image.Close()
 
 	// create the pool
-	err = e.RunExpectSuccessNoOutput(ctx, "zpool", "create", "-O", "mountpoint=none", args.PoolName, args.ImagePath)
+	err = e.RunExpectSuccessNoOutput(ctx, "zpool", "create", "-f",
+		"-O", fmt.Sprintf("mountpoint=%s", args.Mountpoint),
+		args.PoolName, args.ImagePath,
+	)
 	if err != nil {
 		return nil, errors.Wrap(err, "zpool create")
 	}

platformtest/tests/batchRelease.go

@@ -0,0 +1,154 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type rollupReleaseExpectTags struct {
+	Snap  string
+	Holds map[string]bool
+}
+
+func rollupReleaseTest(ctx *platformtest.Context, cb func(fs string) []rollupReleaseExpectTags) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+  "foo bar"
+	+  "foo bar@1"
+	+  "foo bar@2"
+	+  "foo bar@3"
+	+  "foo bar@4"
+	+  "foo bar@5"
+	+  "foo bar@6"
+	R  zfs hold zrepl_platformtest   "${ROOTDS}/foo bar@1"
+	R  zfs hold zrepl_platformtest_2 "${ROOTDS}/foo bar@2"
+	R  zfs hold zrepl_platformtest   "${ROOTDS}/foo bar@3"
+	R  zfs hold zrepl_platformtest   "${ROOTDS}/foo bar@5"
+	R  zfs hold zrepl_platformtest   "${ROOTDS}/foo bar@6"
+	R  zfs bookmark "${ROOTDS}/foo bar@5" "${ROOTDS}/foo bar#5"
+`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+
+	expTags := cb(fs)
+
+	for _, exp := range expTags {
+		holds, err := zfs.ZFSHolds(ctx, fs, exp.Snap)
+		if err != nil {
+			panic(err)
+		}
+		for _, h := range holds {
+			if e, ok := exp.Holds[h]; !ok || !e {
+				panic(fmt.Sprintf("tag %q on snap %q not expected", h, exp.Snap))
+			}
+		}
+	}
+
+}
+
+func RollupReleaseIncluding(ctx *platformtest.Context) {
+	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
+		guid5, err := zfs.ZFSGetGUID(fs, "@5")
+		require.NoError(ctx, err)
+
+		err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
+		require.NoError(ctx, err)
+
+		return []rollupReleaseExpectTags{
+			{"1", map[string]bool{}},
+			{"2", map[string]bool{"zrepl_platformtest_2": true}},
+			{"3", map[string]bool{}},
+			{"4", map[string]bool{}},
+			{"5", map[string]bool{}},
+			{"6", map[string]bool{"zrepl_platformtest": true}},
+		}
+	})
+}
+
+func RollupReleaseExcluding(ctx *platformtest.Context) {
+	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
+		guid5, err := zfs.ZFSGetGUID(fs, "@5")
+		require.NoError(ctx, err)
+
+		err = zfs.ZFSReleaseAllOlderThanGUID(ctx, fs, guid5, "zrepl_platformtest")
+		require.NoError(ctx, err)
+
+		return []rollupReleaseExpectTags{
+			{"1", map[string]bool{}},
+			{"2", map[string]bool{"zrepl_platformtest_2": true}},
+			{"3", map[string]bool{}},
+			{"4", map[string]bool{}},
+			{"5", map[string]bool{"zrepl_platformtest": true}},
+			{"6", map[string]bool{"zrepl_platformtest": true}},
+		}
+	})
+}
+
+func RollupReleaseMostRecentIsBookmarkWithoutSnapshot(ctx *platformtest.Context) {
+	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
+		guid5, err := zfs.ZFSGetGUID(fs, "#5")
+		require.NoError(ctx, err)
+
+		err = zfs.ZFSRelease(ctx, "zrepl_platformtest", fs+"@5")
+		require.NoError(ctx, err)
+
+		err = zfs.ZFSDestroy(fs + "@5")
+		require.NoError(ctx, err)
+
+		err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
+		require.NoError(ctx, err)
+
+		return []rollupReleaseExpectTags{
+			{"1", map[string]bool{}},
+			{"2", map[string]bool{"zrepl_platformtest_2": true}},
+			{"3", map[string]bool{}},
+			{"4", map[string]bool{}},
+			// {"5", map[string]bool{}}, doesn't exist
+			{"6", map[string]bool{"zrepl_platformtest": true}},
+		}
+	})
+}
+
+func RollupReleaseMostRecentIsBookmarkAndSnapshotStillExists(ctx *platformtest.Context) {
+	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
+		guid5, err := zfs.ZFSGetGUID(fs, "#5")
+		require.NoError(ctx, err)
+
+		err = zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, guid5, "zrepl_platformtest")
+		require.NoError(ctx, err)
+
+		return []rollupReleaseExpectTags{
+			{"1", map[string]bool{}},
+			{"2", map[string]bool{"zrepl_platformtest_2": true}},
+			{"3", map[string]bool{}},
+			{"4", map[string]bool{}},
+			{"5", map[string]bool{}},
+			{"6", map[string]bool{"zrepl_platformtest": true}},
+		}
+	})
+}
+
+func RollupReleaseMostRecentDoesntExist(ctx *platformtest.Context) {
+	rollupReleaseTest(ctx, func(fs string) []rollupReleaseExpectTags {
+
+		const nonexistentGuid = 0 // let's take our chances...
+		err := zfs.ZFSReleaseAllOlderAndIncludingGUID(ctx, fs, nonexistentGuid, "zrepl_platformtest")
+		require.Error(ctx, err)
+		require.Contains(ctx, err.Error(), "cannot find snapshot or bookmark with guid 0")
+
+		return []rollupReleaseExpectTags{
+			{"1", map[string]bool{"zrepl_platformtest": true}},
+			{"2", map[string]bool{"zrepl_platformtest_2": true}},
+			{"3", map[string]bool{"zrepl_platformtest": true}},
+			{"4", map[string]bool{"zrepl_platformtest": true}},
+			{"5", map[string]bool{"zrepl_platformtest": true}},
+			{"6", map[string]bool{"zrepl_platformtest": true}},
+		}
+	})
+}

platformtest/tests/getNonexistent.go

@@ -15,9 +15,6 @@ func GetNonexistent(ctx *platformtest.Context) {
 		+  "foo bar"
 		+  "foo bar@1"
 	`)
-	defer platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
-		DESTROYROOT
-	`)
 
 	// test raw
 	_, err := zfs.ZFSGetRawAnySource(fmt.Sprintf("%s/foo bar", ctx.RootDataset), []string{"name"})
@@ -51,4 +48,17 @@ func GetNonexistent(ctx *platformtest.Context) {
 		panic(err)
 	}
 
+	// test nonexistent bookmark
+	nonexistent = fmt.Sprintf("%s/foo bar#non existent", ctx.RootDataset)
+	props, err = zfs.ZFSGetRawAnySource(nonexistent, []string{"name"})
+	if err == nil {
+		panic(props)
+	}
+	dsne, ok = err.(*zfs.DatasetDoesNotExist)
+	if !ok {
+		panic(err)
+	} else if dsne.Path != nonexistent {
+		panic(err)
+	}
+
 }

platformtest/tests/helpers.go

@@ -0,0 +1,139 @@
+package tests
+
+import (
+	"io"
+	"math/rand"
+	"os"
+	"path"
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/util/limitio"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func sendArgVersion(fs, relName string) zfs.ZFSSendArgVersion {
+	guid, err := zfs.ZFSGetGUID(fs, relName)
+	if err != nil {
+		panic(err)
+	}
+	return zfs.ZFSSendArgVersion{
+		RelName: relName,
+		GUID:    guid,
+	}
+}
+
+func mustDatasetPath(fs string) *zfs.DatasetPath {
+	p, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	return p
+}
+
+func mustSnapshot(snap string) {
+	if err := zfs.EntityNamecheck(snap, zfs.EntityTypeSnapshot); err != nil {
+		panic(err)
+	}
+	comps := strings.Split(snap, "@")
+	if len(comps) != 2 {
+		panic(comps)
+	}
+	err := zfs.ZFSSnapshot(mustDatasetPath(comps[0]), comps[1], false)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func mustGetProps(entity string) zfs.ZFSPropCreateTxgAndGuidProps {
+	props, err := zfs.ZFSGetCreateTXGAndGuid(entity)
+	check(err)
+	return props
+}
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+var dummyDataRand = rand.New(rand.NewSource(99))
+
+func writeDummyData(path string, numBytes int64) {
+	r := io.LimitReader(dummyDataRand, numBytes)
+	d, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	check(err)
+	defer d.Close()
+	_, err = io.Copy(d, r)
+	check(err)
+}
+
+type dummySnapshotSituation struct {
+	sendFS       string
+	dummyDataLen int64
+	snapA        *zfs.ZFSSendArgVersion
+	snapB        *zfs.ZFSSendArgVersion
+}
+
+type resumeSituation struct {
+	sendArgs         zfs.ZFSSendArgs
+	recvOpts         zfs.RecvOptions
+	sendErr, recvErr error
+	recvErrDecoded   *zfs.RecvFailedWithResumeTokenErr
+}
+
+func makeDummyDataSnapshots(ctx *platformtest.Context, sendFS string) (situation dummySnapshotSituation) {
+
+	situation.sendFS = sendFS
+	sendFSMount, err := zfs.ZFSGetMountpoint(sendFS)
+	require.NoError(ctx, err)
+	require.True(ctx, sendFSMount.Mounted)
+
+	const dummyLen = int64(10 * (1 << 20))
+	situation.dummyDataLen = dummyLen
+
+	writeDummyData(path.Join(sendFSMount.Mountpoint, "dummy_data"), dummyLen)
+	mustSnapshot(sendFS + "@a snapshot")
+	snapA := sendArgVersion(sendFS, "@a snapshot")
+	situation.snapA = &snapA
+
+	writeDummyData(path.Join(sendFSMount.Mountpoint, "dummy_data"), dummyLen)
+	mustSnapshot(sendFS + "@b snapshot")
+	snapB := sendArgVersion(sendFS, "@b snapshot")
+	situation.snapB = &snapB
+
+	return situation
+}
+
+func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation, recvFS string, sendArgs zfs.ZFSSendArgs, recvOptions zfs.RecvOptions) *resumeSituation {
+
+	situation := &resumeSituation{}
+
+	situation.sendArgs = sendArgs
+	situation.recvOpts = recvOptions
+	require.True(ctx, recvOptions.SavePartialRecvState, "this method would be pointeless otherwise")
+	require.Equal(ctx, sendArgs.FS, src.sendFS)
+
+	copier, err := zfs.ZFSSend(ctx, sendArgs)
+	situation.sendErr = err
+	if err != nil {
+		return situation
+	}
+
+	limitedCopier := zfs.NewReadCloserCopier(limitio.ReadCloser(copier, src.dummyDataLen/2))
+	defer limitedCopier.Close()
+
+	require.NotNil(ctx, sendArgs.To)
+	err = zfs.ZFSRecv(ctx, recvFS, sendArgs.To, limitedCopier, recvOptions)
+	situation.recvErr = err
+	ctx.Logf("zfs recv exit with %T %s", err, err)
+
+	require.NotNil(ctx, err)
+	resumeErr, ok := err.(*zfs.RecvFailedWithResumeTokenErr)
+	require.True(ctx, ok)
+	situation.recvErrDecoded = resumeErr
+
+	return situation
+}

platformtest/tests/idempotentBookmark.go

@@ -0,0 +1,59 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentBookmark(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+		+  "foo bar@another snap"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+
+	asnap := sendArgVersion(fs, "@a snap")
+	anotherSnap := sendArgVersion(fs, "@another snap")
+
+	err := zfs.ZFSBookmark(fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	// do it again, should be idempotent
+	err = zfs.ZFSBookmark(fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	// should fail for another snapshot
+	err = zfs.ZFSBookmark(fs, anotherSnap, "a bookmark")
+	if err == nil {
+		panic(err)
+	}
+	if _, ok := err.(*zfs.BookmarkExists); !ok {
+		panic(fmt.Sprintf("has type %T", err))
+	}
+
+	// destroy the snapshot
+	if err := zfs.ZFSDestroy(fmt.Sprintf("%s@a snap", fs)); err != nil {
+		panic(err)
+	}
+
+	// do it again, should fail with special error type
+	err = zfs.ZFSBookmark(fs, asnap, "a bookmark")
+	if err == nil {
+		panic(err)
+	}
+	if _, ok := err.(*zfs.DatasetDoesNotExist); !ok {
+		panic(fmt.Sprintf("has type %T", err))
+	}
+
+}

platformtest/tests/idempotentDestroy.go

@@ -0,0 +1,75 @@
+package tests
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentDestroy(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	asnap := sendArgVersion(fs, "@a snap")
+	err := zfs.ZFSBookmark(fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	type testCase struct {
+		description, path string
+	}
+
+	cases := []testCase{
+		{"snapshot", fmt.Sprintf("%s@a snap", fs)},
+		{"bookmark", fmt.Sprintf("%s#a bookmark", fs)},
+		{"filesystem", fs},
+	}
+
+	for i := range cases {
+		func() {
+			c := cases[i]
+
+			log.Printf("SUBBEGIN testing idempotent destroy %q for path %q", c.description, c.path)
+
+			log.Println("destroy existing")
+			err = zfs.ZFSDestroy(c.path)
+			if err != nil {
+				panic(err)
+			}
+			log.Println("destroy again, non-idempotently, must error")
+			err = zfs.ZFSDestroy(c.path)
+			if _, ok := err.(*zfs.DatasetDoesNotExist); !ok {
+				panic(fmt.Sprintf("%T: %s", err, err))
+			}
+			log.Println("destroy again, idempotently, must not error")
+			err = zfs.ZFSDestroyIdempotent(c.path)
+			if err != nil {
+				panic(err)
+			}
+
+			log.Println("SUBEND")
+
+		}()
+	}
+
+	// also test idempotent destroy for cases where the parent dataset does not exist
+	err = zfs.ZFSDestroyIdempotent(fmt.Sprintf("%s/not foo bar@nonexistent snapshot", ctx.RootDataset))
+	if err != nil {
+		panic(err)
+	}
+
+	err = zfs.ZFSDestroyIdempotent(fmt.Sprintf("%s/not foo bar#nonexistent bookmark", ctx.RootDataset))
+	if err != nil {
+		panic(err)
+	}
+
+}

platformtest/tests/idempotentHold.go

@@ -0,0 +1,47 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentHold(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1"
+	`)
+	defer platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		R zfs release zrepl_platformtest "${ROOTDS}/foo bar@1"
+		-  "foo bar@1"
+		-  "foo bar"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	v1 := sendArgVersion(fs, "@1")
+
+	tag := "zrepl_platformtest"
+	err := zfs.ZFSHold(ctx, fs, v1, tag)
+	if err != nil {
+		panic(err)
+	}
+
+	err = zfs.ZFSHold(ctx, fs, v1, tag)
+	if err != nil {
+		panic(err)
+	}
+
+	vnonexistent := zfs.ZFSSendArgVersion{
+		RelName: "@nonexistent",
+		GUID:    0xbadf00d,
+	}
+	err = zfs.ZFSHold(ctx, fs, vnonexistent, tag)
+	if err == nil {
+		panic("still expecting error for nonexistent snapshot")
+	}
+
+}

platformtest/tests/replicationCursor.go

@@ -0,0 +1,68 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func ReplicationCursor(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@1 with space" "${ROOTDS}/foo bar#1 with space"
+		+  "foo bar@2 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@1 with space" "${ROOTDS}/foo bar#2 with space"
+		+  "foo bar@3 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@1 with space" "${ROOTDS}/foo bar#3 with space"
+	`)
+
+	jobid := endpoint.MustMakeJobID("zreplplatformtest")
+
+	ds, err := zfs.NewDatasetPath(ctx.RootDataset + "/foo bar")
+	if err != nil {
+		panic(err)
+	}
+
+	fs := ds.ToString()
+	snap := sendArgVersion(fs, "@1 with space")
+
+	destroyed, err := endpoint.MoveReplicationCursor(ctx, fs, &snap, jobid)
+	if err != nil {
+		panic(err)
+	}
+	assert.Empty(ctx, destroyed)
+
+	snapProps, err := zfs.ZFSGetCreateTXGAndGuid(snap.FullPath(fs))
+	if err != nil {
+		panic(err)
+	}
+
+	bm, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs, jobid)
+	if err != nil {
+		panic(err)
+	}
+	if bm.CreateTXG != snapProps.CreateTXG {
+		panic(fmt.Sprintf("createtxgs do not match: %v != %v", bm.CreateTXG, snapProps.CreateTXG))
+	}
+	if bm.Guid != snapProps.Guid {
+		panic(fmt.Sprintf("guids do not match: %v != %v", bm.Guid, snapProps.Guid))
+	}
+
+	// try moving
+	cursor1BookmarkName, err := endpoint.ReplicationCursorBookmarkName(fs, snap.GUID, jobid)
+	require.NoError(ctx, err)
+
+	snap2 := sendArgVersion(fs, "@2 with space")
+	destroyed, err = endpoint.MoveReplicationCursor(ctx, fs, &snap2, jobid)
+	require.NoError(ctx, err)
+	require.Equal(ctx, 1, len(destroyed))
+	require.Equal(ctx, zfs.Bookmark, destroyed[0].Type)
+	require.Equal(ctx, cursor1BookmarkName, destroyed[0].Name)
+}

platformtest/tests/resumableRecvAndTokenHandling.go

@@ -0,0 +1,64 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func ResumableRecvAndTokenHandling(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er"
+	`)
+
+	sendFS := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	recvFS := fmt.Sprintf("%s/recv er", ctx.RootDataset)
+
+	supported, err := zfs.ResumeRecvSupported(ctx, mustDatasetPath(sendFS))
+	check(err)
+
+	src := makeDummyDataSnapshots(ctx, sendFS)
+
+	s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgs{
+		FS:          sendFS,
+		To:          src.snapA,
+		Encrypted:   &zfs.NilBool{B: false},
+		ResumeToken: "",
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false, // doesnt' exist yet
+		SavePartialRecvState: true,
+	})
+
+	if !supported {
+		_, ok := s.recvErr.(*zfs.ErrRecvResumeNotSupported)
+		require.True(ctx, ok)
+
+		// we know that support on sendFS implies support on recvFS
+		// => asser that if we don't support resumed recv, the method returns ""
+		tok, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, mustDatasetPath(recvFS))
+		check(err)
+		require.Equal(ctx, "", tok)
+
+		return // nothing more to test for recv that doesn't support -s
+	}
+
+	getTokenRaw, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, mustDatasetPath(recvFS))
+	check(err)
+
+	require.NotEmpty(ctx, getTokenRaw)
+	decodedToken, err := zfs.ParseResumeToken(ctx, getTokenRaw)
+	check(err)
+
+	require.True(ctx, decodedToken.HasToGUID)
+	require.Equal(ctx, s.sendArgs.To.GUID, decodedToken.ToGUID)
+
+	recvErr := s.recvErr.(*zfs.RecvFailedWithResumeTokenErr)
+	require.Equal(ctx, recvErr.ResumeTokenRaw, getTokenRaw)
+	require.Equal(ctx, recvErr.ResumeTokenParsed, decodedToken)
+}

platformtest/tests/resumeTokenParsing.go

@@ -0,0 +1,105 @@
+package tests
+
+import (
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type resumeTokenTest struct {
+	Msg         string
+	Token       string
+	ExpectToken *zfs.ResumeToken
+	ExpectError error
+}
+
+func (rtt *resumeTokenTest) Test(t *platformtest.Context) {
+
+	resumeSendSupported, err := zfs.ResumeSendSupported()
+	if err != nil {
+		t.Errorf("cannot determine whether resume supported: %T %s", err, err)
+		t.FailNow()
+		return
+	}
+
+	res, err := zfs.ParseResumeToken(t, rtt.Token)
+
+	// if decoding is not supported, don't bother with the expectations
+	if !resumeSendSupported {
+		require.Error(t, err)
+		require.Equal(t, zfs.ResumeTokenDecodingNotSupported, err)
+		return
+	}
+
+	if rtt.ExpectError != nil {
+		require.EqualValues(t, rtt.ExpectError, err)
+		return
+	}
+	if rtt.ExpectToken != nil {
+		require.Nil(t, err)
+		require.EqualValues(t, rtt.ExpectToken, res)
+		return
+	}
+}
+
+func ResumeTokenParsing(ctx *platformtest.Context) {
+
+	// cases generated using resumeTokensGenerate.bash on ZoL 0.8.1
+	cases := []resumeTokenTest{
+		{
+			Msg:   "zreplplatformtest/dst/full",
+			Token: "1-b338b54f3-c0-789c636064000310a500c4ec50360710e72765a52697303030419460caa7a515a796806474e0f26c48f2499525a9c540ba42430fabfe92fcf4d2cc140686c88a76d578ae45530c90e439c1f27989b9a90c0c5545a905390539892569f945b940234bf48b8b921d12c1660200c61a1aba",
+			ExpectToken: &zfs.ResumeToken{
+				HasToGUID: true,
+				ToGUID:    0x94a20a5f25877859,
+				ToName:    "zreplplatformtest/src@a",
+			},
+		},
+		{
+			Msg:   "zreplplatformtest/dst/full_raw",
+			Token: "1-e3f40c323-f8-789c636064000310a500c4ec50360710e72765a52697303030419460caa7a515a796806474e0f26c48f2499525a9c540da454f0fabfe92fcf4d2cc140686c88a76d578ae45530c90e439c1f27989b9a90c0c5545a905390539892569f945b940234bf48b8b921d12c1e6713320dc9f9c9f5b50945a5c9c9f0d119380ba07265f94580e936200004ff12141",
+			ExpectToken: &zfs.ResumeToken{
+				HasToGUID:     true,
+				ToGUID:        0x94a20a5f25877859,
+				ToName:        "zreplplatformtest/src@a",
+				HasCompressOK: true, CompressOK: true,
+				HasRawOk: true, RawOK: true,
+			},
+		},
+
+		{
+			Msg:   "zreplplatformtest/dst/inc",
+			Token: "1-eadabb296-e8-789c636064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b518483ba8ea61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b09900d43e2191",
+			ExpectToken: &zfs.ResumeToken{
+				HasFromGUID: true, FromGUID: 0x94a20a5f25877859,
+				HasToGUID: true, ToGUID: 0xf784e1004f460f7a,
+				ToName: "zreplplatformtest/src@b",
+			},
+		},
+		{
+			Msg:   "zreplplatformtest/dst/inc_raw",
+			Token: "1-1164f8d409-120-789c636064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b51848f368eb61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b079dc0c087f26e7e71614a51617e76743c424a0ee81c9172596c3a41800dd2c2818",
+			ExpectToken: &zfs.ResumeToken{
+				HasFromGUID: true, FromGUID: 0x94a20a5f25877859,
+				HasToGUID: true, ToGUID: 0xf784e1004f460f7a,
+				ToName:        "zreplplatformtest/src@b",
+				HasCompressOK: true, CompressOK: true,
+				HasRawOk: true, RawOK: true,
+			},
+		},
+
+		// manual test csaes
+		{
+			Msg:         "corrupted",
+			Token:       "1-1164f8d409-120-badf00d064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b51848f368eb61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b079dc0c087f26e7e71614a51617e76743c424a0ee81c9172596c3a41800dd2c2818",
+			ExpectError: zfs.ResumeTokenCorruptError,
+		},
+	}
+
+	for _, test := range cases {
+		ctx.Logf("BEGIN SUBTEST: %s", test.Msg)
+		test.Test(ctx)
+		ctx.Logf("COMPLETE SUBTEST: %s", test.Msg)
+	}
+}

platformtest/tests/resumeTokensGenerate.bash

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+POOLNAME="zreplplatformtest"
+POOLIMG="$1"
+
+if zpool status "$POOLNAME"; then
+    exit 1
+fi
+
+if [ -e "$POOLIMG" ]; then
+    exit 1
+fi
+fallocate -l 500M "$POOLIMG"
+zpool create "$POOLNAME" "$POOLIMG"
+
+SRC_DATASET="$POOLNAME/src"
+DST_ROOT="$POOLNAME/dst"
+
+keylocation="$(mktemp /tmp/ZREPL_PLATFORMTEST_GENERATE_TOKENS_KEYFILE_XXX)"
+echo "foobar123" > "$keylocation"
+zfs create -o encryption=on -o keylocation="file:///$keylocation" -o keyformat=passphrase "$SRC_DATASET"
+rm "$keylocation"
+
+SRC_MOUNT="$(zfs get -H -o value mountpoint "$SRC_DATASET")"
+test -d "$SRC_MOUNT"
+
+A="$SRC_DATASET"@a
+B="$SRC_DATASET"@b
+dd if=/dev/urandom of="$SRC_MOUNT"/dummy_data bs=1M count=5
+zfs snapshot "$A"
+dd if=/dev/urandom of="$SRC_MOUNT"/dummy_data bs=1M count=5
+zfs snapshot "$B"
+
+zfs create "$DST_ROOT"
+
+cutoff="dd bs=1024 count=3K"
+
+set +e
+zfs send    "$A"         | $cutoff | zfs recv -s "$DST_ROOT"/full
+
+zfs send "$A" | zfs recv "$DST_ROOT"/inc
+zfs send    -i "$A" "$B" | $cutoff | zfs recv -s "$DST_ROOT"/inc
+
+zfs send -w "$A"         | $cutoff | zfs recv -s "$DST_ROOT"/full_raw
+
+zfs send -w "$A" | zfs recv "$DST_ROOT"/inc_raw
+zfs send -w -i "$A" "$B" | $cutoff | zfs recv -s "$DST_ROOT"/inc_raw
+set -e
+
+TOKENS="$(zfs list -H -o name,receive_resume_token -r "$DST_ROOT")"
+
+echo "$TOKENS" | awk -e '//{ if ($2 == "-") {  } else { system("zfs send -nvt " + $2);} }'
+
+zpool destroy "$POOLNAME"
+rm "$POOLIMG"

platformtest/tests/sendArgsValidation.go

@@ -0,0 +1,208 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	check(err)
+	expectNotSupportedErr := !supported
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er"
+	+   "send er@a snap"
+	`)
+
+	fs := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	props := mustGetProps(fs + "@a snap")
+
+	sendArgs := zfs.ZFSSendArgs{
+		FS: fs,
+		To: &zfs.ZFSSendArgVersion{
+			RelName: "@a snap",
+			GUID:    props.Guid,
+		},
+		Encrypted:   &zfs.NilBool{B: true},
+		ResumeToken: "",
+	}
+	stream, err := zfs.ZFSSend(ctx, sendArgs)
+	if err == nil {
+		defer stream.Close()
+	}
+
+	if expectNotSupportedErr {
+		require.Error(ctx, err)
+		require.Equal(ctx, zfs.ErrEncryptedSendNotSupported, err)
+		return
+	}
+	require.Error(ctx, err)
+	ctx.Logf("send err: %T %s", err, err)
+	validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+	require.True(ctx, ok)
+	require.True(ctx, validationErr.What == zfs.ZFSSendArgsEncryptedSendRequestedButFSUnencrypted)
+}
+
+func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+	supported, err = zfs.ResumeSendSupported()
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er" encrypted
+	`)
+
+	sendFS := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	unencRecvFS := fmt.Sprintf("%s/unenc recv", ctx.RootDataset)
+	encRecvFS := fmt.Sprintf("%s/enc recv", ctx.RootDataset)
+
+	src := makeDummyDataSnapshots(ctx, sendFS)
+
+	unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgs{
+		FS:        sendFS,
+		To:        src.snapA,
+		Encrypted: &zfs.NilBool{B: false}, // !
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgs{
+		FS:        sendFS,
+		To:        src.snapA,
+		Encrypted: &zfs.NilBool{B: true}, // !
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	// threat model: use of a crafted resume token that requests an unencrypted send
+	//               but send args require encrypted send
+	{
+		var maliciousSend zfs.ZFSSendArgs = encS.sendArgs
+		maliciousSend.ResumeToken = unencS.recvErrDecoded.ResumeTokenRaw
+
+		stream, err := zfs.ZFSSend(ctx, maliciousSend)
+		if err == nil {
+			defer stream.Close()
+		}
+		require.Nil(ctx, stream)
+		require.Error(ctx, err)
+		ctx.Logf("send err: %T %s", err, err)
+		validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+		require.True(ctx, ok)
+		require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+		ctx.Logf("%s", validationErr)
+
+		mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+		require.True(ctx, ok)
+		require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchEncryptionNotSet)
+	}
+
+	// threat model: use of a crafted resume token that requests an encryped send
+	//               but send args require unencrypted send
+	{
+		var maliciousSend zfs.ZFSSendArgs = unencS.sendArgs
+		maliciousSend.ResumeToken = encS.recvErrDecoded.ResumeTokenRaw
+
+		stream, err := zfs.ZFSSend(ctx, maliciousSend)
+		if err == nil {
+			defer stream.Close()
+		}
+		require.Nil(ctx, stream)
+		require.Error(ctx, err)
+		ctx.Logf("send err: %T %s", err, err)
+		validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+		require.True(ctx, ok)
+		require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+		ctx.Logf("%s", validationErr)
+
+		mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+		require.True(ctx, ok)
+		require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchEncryptionSet)
+	}
+
+}
+
+func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+	supported, err = zfs.ResumeSendSupported()
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er1"
+	+	"send er2"
+	`)
+
+	sendFS1 := fmt.Sprintf("%s/send er1", ctx.RootDataset)
+	sendFS2 := fmt.Sprintf("%s/send er2", ctx.RootDataset)
+	recvFS := fmt.Sprintf("%s/unenc recv", ctx.RootDataset)
+
+	src1 := makeDummyDataSnapshots(ctx, sendFS1)
+	src2 := makeDummyDataSnapshots(ctx, sendFS2)
+
+	rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgs{
+		FS:        sendFS1,
+		To:        src1.snapA,
+		Encrypted: &zfs.NilBool{B: false},
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	// threat model: forged resume token tries to steal a full send of snapA on fs2 by
+	//               presenting a resume token for full send of snapA on fs1
+	var maliciousSend zfs.ZFSSendArgs = zfs.ZFSSendArgs{
+		FS: sendFS2,
+		To: &zfs.ZFSSendArgVersion{
+			RelName: src2.snapA.RelName,
+			GUID:    src2.snapA.GUID,
+		},
+		Encrypted:   &zfs.NilBool{B: false},
+		ResumeToken: rs.recvErrDecoded.ResumeTokenRaw,
+	}
+
+	stream, err := zfs.ZFSSend(ctx, maliciousSend)
+	if err == nil {
+		defer stream.Close()
+	}
+	require.Nil(ctx, stream)
+	require.Error(ctx, err)
+	ctx.Logf("send err: %T %s", err, err)
+	validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+	require.True(ctx, ok)
+	require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+	ctx.Logf("%s", validationErr)
+
+	mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+	require.True(ctx, ok)
+	require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchFilesystem)
+}

platformtest/tests/tests.go

@@ -17,4 +17,18 @@ var Cases = []Case{
 	BatchDestroy,
 	UndestroyableSnapshotParsing,
 	GetNonexistent,
+	ReplicationCursor,
+	RollupReleaseIncluding,
+	RollupReleaseExcluding,
+	RollupReleaseMostRecentIsBookmarkWithoutSnapshot,
+	RollupReleaseMostRecentIsBookmarkAndSnapshotStillExists,
+	RollupReleaseMostRecentDoesntExist,
+	IdempotentHold,
+	IdempotentBookmark,
+	IdempotentDestroy,
+	ResumeTokenParsing,
+	ResumableRecvAndTokenHandling,
+	SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden,
+	SendArgsValidationResumeTokenEncryptionMismatchForbidden,
+	SendArgsValidationResumeTokenDifferentFilesystemForbidden,
 }

platformtest/tests/undestroyableSnapshotParsing.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 
 	"github.com/stretchr/testify/require"
+
 	"github.com/zrepl/zrepl/platformtest"
 	"github.com/zrepl/zrepl/zfs"
 )
@@ -18,10 +19,6 @@ func UndestroyableSnapshotParsing(t *platformtest.Context) {
 		+  "foo bar@7 8 9"
 		R  zfs hold zrepl_platformtest "${ROOTDS}/foo bar@4 5 6"
 	`)
-	defer platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
-	R zfs release zrepl_platformtest "${ROOTDS}/foo bar@4 5 6"
-	DESTROYROOT
-	`)
 
 	err := zfs.ZFSDestroy(fmt.Sprintf("%s/foo bar@1 2 3,4 5 6,7 8 9", t.RootDataset))
 	if err == nil {

replication/design.md

@@ -0,0 +1,342 @@
+The goal of this document is to describe what **logical steps** zrepl takes to replicate ZFS filesystems.
+Note that the actual code executing these steps is spread over the `endpoint.{Sender,Receiver}` RPC endpoint implementations. The code in `replication/{driver,logic}` (mostly `logic.Step.doReplication`) drives the replication and invokes the `endpoint` methods (locally or via RPC).
+Hence, when trying to map algorithm to implementation, use the code in package `replication` as the entrypoint and follow the RPC calls.
+
+
+* The [Single Replication Step](#zrepl-algo-single-step) algorithm is implemented as described
+  * step holds are implemented as described
+  * step bookmarks rely on bookmark cloning, which is WIP in OpenZFS (see respective TODO comments)
+  * the feature to hold receive-side `to` immediately after replication is used to move the `last-received-hold` forward
+    * this is a little more than what we allow in the algorithm (we only allow specifying a hold tag whereas moving the `last-received-hold` is a little more complex)
+  * the algorithm's sender-side callback is used move the `zrepl_CURSOR` bookmark to point to `to`
+* The `zrepl_CURSOR` bookmark and the `last-received-hold` ensure that future replication steps can always be done using incremental replication:
+  * Both reference / hold the last successfully received snapshot `to`.
+  * Thus, `to` or `#zrepl_CURSOR_G_${to.GUID}_J_${jobid}` can always be used for a future incremental replication step `to => future-to`:
+    * incremental send will be possible because `to` doesn't go away because we claim ownership of the `#zrepl_CURSOR` namespace
+    * incremental recv will be possible iff
+      * `to` will still be present on the recv-side because of `last-received-hold`
+      * the recv-side fs doesn't get newer snapshots than `to` in the meantime
+        * guaranteed because the zrepl model of the receiver assumes ownership of the filesystems it receives into
+          * if that assumption is broken, future replication attempts will fail with a conflict
+* The [Algorithm for Planning and Executing Replication of an Filesystems](#zrepl-algo-filesystem) is a design draft and not used
+* We also have [Notes on Planning and Executing Replication of Multiple Filesystems](#zrepl-algo-multiple-filesystems-notes)
+
+---
+
+<a id="zrepl-algo-single-step"></a>
+## Algorithm for a Single Replication Step
+
+The algorithm described below describes how a single replication step must be implemented.
+A *replication step* is a full send of a snapshot `to` for initial replication, or an incremental send (`from => to`) for incremental replication.
+
+The algorithm **ensures resumability** of the replication step in presence of
+
+* uncoordinated or unsynchronized destruction of the filesystem, snapshots, or bookmarks involved in the replication step
+* network failures at any time
+* other instances of this algorithm executing the same step in parallel (e.g. concurrent replication to different destinations)
+
+To accomplish this goal, the algorithm **assumes ownersip of parts of the ZFS hold tag namespace and the bookmark namespace**:
+* holds with prefix `zrepl_STEP` on any snapshot are reserved for zrepl
+* bookmarks with prefix `zrepl_STEP` are reserved for zrepl
+
+Resumability of a step cannot be guaranteed if these sub-namespaces are manipulated through software other than zrepl.
+
+Note that the algorithm **does not ensure** that a replication *plan*, which describes a *set* of replication steps, can be carried out successfully.
+If that is desirable, additional measures outside of this algorithm must be taken.
+
+---
+
+### Definitions:
+
+#### Step Completion & Invariants
+
+The replication step (full `to` send or `from => to` send) is *complete* iff the algorithm ran to completion without errors or a permanent non-network error is reported by sender or receiver.
+
+Specifically, the algorithm may be invoked with the same `from` and `to` arguments, and potentially a `resume_token`, after a temporary (like network-related) failure:
+**Unless permanent errors occur, repeated invocations of the algorithm with updated resume token will converge monotonically (but not strictly monotonically) toward completion.**
+
+Note that the mere existence of `to` on the receiving side does not constitue completion, since there may still be post-recv actions to be performed on sender and receiver.
+
+#### Job and Job ID
+This algorithm supports that *multiple* instance of it run in parallel on the *same* step (full `to` / `from => to` pair).
+An exemplary use case for this feature are concurrent replication jobs that replicate the same dataset to different receivers.
+
+**We require that all parallel invocations of this algorithm provide different and unique `jobid`s.**
+Violation of `jobid` uniqueness across parallel jobs may result in interference between instances of this algorithm, resulting in potential compromise of resumability.
+
+After a step is *completed*, `jobid` is guaranteed to not be encoded in on-disk state.
+Before a step is completed, there is no such guarantee.
+Changing the `jobid` before step completion may compromise resumability and may leak the underlying ZFS holds or step bookmarks (i.e. zrepl won't clean them up)
+Note the definition of *complete* above.
+
+#### Step Bookmark
+
+A step bookmark is our equivalent of ZFS holds, but for bookmarks.<br/>
+A step bookmark is a ZFS bookmark whose name matches the following regex:
+
+```
+STEP_BOOKMARK = #zrepl_STEP_bm_G_([0-9a-f]+)_J_(.+)
+```
+
+- capture group `1` must be the guid of `zfs get guid ${STEP_BOOKMARK}`, encoded hexadecimal (without leading `0x`) and fixed-length (i.e. padded with leading zeroes)
+- capture group `2` must be equal to the `jobid`
+
+### Algorithm
+
+INPUT:
+
+* `jobid`
+* `from`: snapshot or bookmark: may be nil for full send
+* `to`: snapshot, must never be nil
+* `resume_token` (may be nil)
+* (`from` and `to` must be on the same filesystem)
+
+#### Prepare-Phase
+
+Send-side: make sure `to` and `from` don't go away
+
+- hold `to` using `idempotent_hold(to, zrepl_STEP_J_${jobid})`
+- make sure `from` doesn't go away:
+  - if `from` is a snapshot: hold `from` using `idempotent_hold(from, zrepl_STEP_J_${jobid})`
+  - else `idempotent_step_bookmark(from)` (`from` is a bookmark)
+    - PERMANENT ERROR if this fails (e.g. because `from` is a bookmark whose snapshot no longer exists and we don't have bookmark copying yet (ZFS feature is in development) )
+      - Why? we must assume the given bookmark is externally managed (i.e. not in the )
+      - this means the bookmark is externally created bookmark and cannot be trusted to persist until the replication step succeeds
+      - Maybe provide an override-knob for this behavior
+
+Recv-side: no-op
+
+- `from` cannot go away once we received enough data for the step to be resumable:
+    ```text
+    # do a partial incremental recv @a=>@b (i.e. recv with -s, and abort the incremental recv in the middle)
+    # try destroying the incremental source on the receiving side
+    zfs destroy p1/recv@a
+    cannot destroy 'p1/recv@a': snapshot has dependent clones
+    use '-R' to destroy the following datasets:
+    p1/recv/%recv
+    # => doesn't work, because zfs recv is implemented as a `clone` internally, that's exactly what we want
+    ```
+
+- if recv-side `to` exists, goto cleaup-phase (no replication to do)
+  
+- `to` cannot be destroyed while being received, because it isn't visible as a snapshot yet (it isn't yet one after all)
+
+#### Replication Phase
+
+Attempt the replication step:
+start `zfs send` and pipe its output into `zfs recv` until an error occurs or `recv` returns without error.
+
+Let us now think about interferences that could occur during this phase, and ensure that none of them compromise the goals of this algorithm, i.e., monotonic convergence toward step completion using resumable send & recv.
+
+**Safety from External Pruning**
+We are safe from pruning during the replication step because we have guarantees that no external action will destroy send-side `from` and `to`, and recv-side `to` (for both snapshot and bookmark `from`s)<br/>
+
+**Safety In Presence of Network Failures During Replication**
+Network failures during replication can be recovered from using resumable send & recv:
+
+- Network failure before the receive-side could stored data:
+  - Send-side `from` and `to` are guaranteed to be still present due to zfs holds
+  - recv-side `from` may no longer exist because we don't `hold` it explicitly
+    - if that is the case, ERROR OUT, step can never complete
+  - If the step planning algorithm does not include the step, for example because a snapshot filter configuration was changed by the user inbetween which hides `from` or `to` from the second planning attempt: **tough luck, we're leaking all holds**
+- Network failure during the replication
+  - send-side `from` and `to` are still present due to zfs holds
+  - recv-side `from` is still present because the partial receive state prevents its destruction (see prepare-phase)
+  - if recv-side hasa resume token, the resume token will continue to work on the sender because `from`s and `to` are still present
+- Network failure at the end of the replication step stream transmission
+  - Variant A: failure from the sender's perspective, success from the receiver's perspective
+    - receive-side `to` doesn't have a hold and could be destroyed anytime
+    - receive-side `from` doesn't have a hold and could be destroyed anytime
+    - thus, when the step is invoked again, pattern match `(from_exists, to_exists)`
+      - `(true, true)`: prepare-phase will early-exit
+      - `(false,true)`: prepare-phase will error out bc. `from` does not exist
+      - `(true, false)`: entire step will be re-done
+        - FIXME monotonicity requirement does not hold
+      - `(false, false)`: prepare-phase will error out bc. `from` does not exist
+  - Variant B: success from the sender's perspective, failure from the receiver's perspective
+    - No idea how this would happen except for bugs in error reporting in the replication protocol
+    - Misclassification by the sender, most likely broken error handling in the sender or replication protocol
+    - => the sender will release holds and move the replication cursor while the receiver won't => tough luck
+
+If the RPC used for `zfs recv` returned without error, this phase is done.
+(Although the *replication step* (this algorithm) is not yet *complete*, see definition of complete).
+
+
+#### Cleanup-Phase
+
+At the end of this phase, all intermediate state we built up to support the resumable replication of the step is destroyed.
+However, consumers of this algorithm might want to take advantage of the fact that we currently still have holds / step bookmarks.
+
+##### Recv-side: Optionally hold `to` with a caller-defined tag
+
+Users of the algorithm might want to depend on `to` being available on the receiving side for a longer duration than the lifetime of the current step's algorithm invocation.
+For reasons explained in the next paragraph, we cannot guarantee that we have a `hold` when `recv` exists. If that were the case, we could take our time to provide a generalized callback to the user of the algorithm, and have them do whatever they want with `to` while we guarantee that `to` doesn't go away through the hold. But that functionality isn't available, so we only provide a fixed operation right after receive: **take a `hold` with a tag of the algorithm user's choice**. That's what's needed to guarantee that a replication plan, consisting of multiple consecutive steps, can be carried out without a receive-side prune job interfering by destroying `to`. Technically, this step is racy, i.e., it could happen that `to` is destroyed between `recv` and `hold`. But this is a) unlikely and b) not fatal because we can detect that hold fails because the 'dataset does not exist` and re-do the entire transmission since we still hold send-side `from` and `to`, i.e. we just lose a lot of work in rare cases.
+
+So why can't we have a `hold` at the time `recv` exits?
+It seems like [`zfs send --holds`](https://github.com/zfsonlinux/zfs/commit/9c5e88b1ded19cb4b19b9d767d5c71b34c189540) could be used to send the send-side's holds to the receive-side. But that flag always sends all holds, and `--holds` is implemented in user-space libzfs, i.e., doesn't happen in the same txg as the recv completion. Thus, the race window is in fact only smaller (unless we oversaw some synchronization in userland zfs).
+But if we're willing to entertain the idea a little further, we still hit the problem that `--holds` sends _all_ holds, whereas our goal is to _temporarily_ have >= 1 hold that we own until the callback is done, and then release all of the received holds so that no holds created by us exist after this algorithm completes.
+Since there could be concurrent `zfs hold` invocations on the sender and receiver while this algorithm runs, and because `--holds` doesn't provide info about which holds were sent, we cannot correctly destroy _exactly_ those holds that we received.
+
+##### Send-side: callback to consumer, then cleanup
+We can provide the algorithm user with a generic callback because we have holds / step bookmarks for `to` and `from`, respectively.
+
+Example use case for the sender-side callback is the replication cursor, see algorithm for filesystem replication below.
+
+After the callback is done: cleanup holds & step bookmark:
+
+- `idempotent_release(to, zrepl_STEP_J_${jobid})`
+- make sure `from` can now go away:
+  - if `from` is a snapshot: `idempotent_release(from, zrepl_STEP_J_${jobid})`
+  - else `idempotent_step_bookmark_destroy(from)`
+    - if `from` fulfills the properties of a step bookmark: destroy it
+    - otherwise: it's a bookmark not created by this algorithm that happened to be used for replication: leave it alone
+      - that can only happen if user used the override knob
+
+Note that "make sure `from` can now go away" is the inverse of "Send-side: make sure `to` and `from` don't go away". Those are relatively complex operations and should be implemented in the same file next to each other to ease maintainability.
+
+---
+
+### Notes & Pseudo-APIs used in the algorithm
+
+- `idempotent_hold(snapshot s, string tag)` like zfs hold, but doesn't error if hold already exists
+- `idempotent_release(snapshot s, string tag)` like zfs hold, but doesn't error if hold already exists
+- `idempotent_step_bookmark(snapshot_or_bookmark b)` creates a *step bookmark* of b
+  - determine step bookmark name N
+  - if `N` already exists, verify it's a correct step bookmark => DONE or ERROR
+  - if `b` is a snapshot, issue `zfs bookmark`
+  - if `b` is a bookmark:
+    - if bookmark cloning supported, use it to duplicate the bookmark
+    - else ERROR OUT, with an error that upper layers can identify as such, so that they are able to ignore the fact that we couldn't create a step bookmark 
+- `idempotent_destroy(bookmark #b_$GUID, of a snapshot s)` must atomically check that `$GUID == s.guid` before destroying s
+- `idempotent_bookmark(snapshot s, $GUID, name #b_$GUID)` must atomically check that `$GUID == s.guid` at the time of bookmark creation
+- `idempotent_destroy(snapshot s)` must atomically check that zrepl's `s.guid` matches the current `guid` of the snapshot (i.e. destroy by guid)
+
+
+<a id="zrepl-algo-filesystem"></a>
+## Algorithm for Planning and Executing Replication of a Filesystems (planned, not implemented yet)
+
+This algorithm describes how a filesystem or zvol is replicated in zrepl.
+The algorithm is invoked with a `jobid`, `sender`, `receiver`, a sender-side `filesystem path`.
+It builds a diff between the sender and receiver filesystem bookmarks+snapshots and determines whether a replication conflict exists or whether fast-forward replication using full or incremental sends is possible.
+In case of conflict, the algorithm errors out with a conflict description that can be used to manually or automatically resolve the conflict.
+Otherwise, the algorithm builds a list of replication steps that are then worked on sequentially by the "Algorithm for a Single Replication Step".
+
+The algorithm ensures that a plan can be executed exactly as planned by aquiring appropriate zfs holds.
+The algorithm can be configured to retry a plan when encountering non-permanent errors (e.g. network errors).
+However, permanent errors result in the plan being cancelled.
+
+Regardless of whether a plan can be executed to completion or is cancelled, the algorithm guarantees that leftover artifacts (i.e. holds) of its invocation are cleaned up.
+However, since network failure can occur at any point, there might be stale holds on sending or receiving side after a crash or other error.
+These will be cleaned up on a subsequent invocation of this algorithm with the same `jobid`.
+
+The algorithm is fit to be executed in parallel from the same sender to different receivers.
+To be clear: replicating in parallel from the same sender to different receivers is supported.
+But one instance of the algorithm assumes ownership of the `filesystem path` on the receiver.
+
+The algorithm reserves the following sub-namespaces:
+* zfs hold: `zrepl_FS`
+* bookmark names: `zrepl_FS`
+
+### Definitions
+
+#### ZFS Filesystem Path
+We refer to the name of a ZFS filesystem or volume as a *filesystem path*, sometimes just *path*.
+The name includes the pool name.
+
+#### Sender and Receiver
+Sender and Receiver are passed as RPC stubs to the algorithm.
+One of those RPC stubs will typically be local, i.e., call methods on a struct in the same address space.
+The other RPC stub may invoke actual calls over the network (unless local replication is used).
+The algorithm does not make any assumption about which object is local or an actual stub.
+This design decouples the replication logic (described in this algorithm) from the question which side (sender or receiver) initiates the replication.
+
+### Algorithm
+
+**Cleanup Previous Invocations With Same `jobid`** by scanning the filesystem's list of snapshots and step bookmarks, and destroying any which was created by this algorithm when it was invoked with `jobid`.<br/>
+TODO HOW
+
+**Build a fast-forward list of replication steps** `STEPS`.
+`STEPS` may contain an optional initial full send, and subsequent incremental send steps.
+`STEPS[0]` may also have a resume token.
+If fast-forward is not possible, produce a conflict description and ERROR OUT.<br/>
+TODOs:
+- make it configurable what snapshots are included in the list (i.e. every one we see, only most recent, at least one every X hours, ...)
+
+**Ensure that we will be able to carry out all steps** by aquiring holds or fsstep bookmarks on the sending side
+- `idempotent_hold([s.to for s in STEPS], zrepl_FS_J_${jobid})`
+- `if STEPS[0].from != nil: idempotent_FSSTEP_bookmark(STEPS[0].from, zrepl_FSSTEP_bm_G_${STEPS[0].from.guid}_J_${jobid})` 
+  
+**Determine which steps have not been completed (`uncompleted_steps`)** (we might be in an second invocation of this algorithm after a network failure and some steps might already be done):
+- `res_tok := receiver.ResumeToken(fs)`
+- `rmrfsv := receiver.MostRecentFilesystemVersion(fs)`
+- if `res_tok !=  nil`: ensure that `res_tok` has a correspondinng step in `STEPS`, otherwise ERROR OUT
+- if `rmrfsv != nil`: ensure that `res_tok` has a correspondinng step in `STEPS`, otherwise ERROR OUT
+- if `(res_token != nil && rmrfsv != nil)`: ensure that `res_tok` is the subsequent step to the one we found for `rmrfsv`
+- if both are nil, we are at the beginning, `uncompleted_steps = STEPS` and goto next block
+- `rstep := if res_tok != nil { res_tok } else { rmrfsv }`
+- `uncompleted_steps := STEPS[find_step_idx(STEPS, rstep).expect("must exist, checked above"):]`
+  - Note that we do not explicitly check for the completion of prior replication steps.
+    All we care about is what needs to be done from `rstep`.
+  - This is intentional and necessary because we cummutatively release all holds and step bookmarks made for steps that preceed a just-completed step (see next paragraph)
+
+**Execute uncompleted steps**<br/>
+Invoke the "Algorithm for a Single Replication Step" for each step in `uncompleted_steps`.
+Remember to pass the resume token if one exists.
+
+In the wind-down phase of each replication step `from => to`, while the per-step algorithm still holds send-side `from` and `to`, as well as recv-side `to`:
+
+- Sending side: **Idempotently move the replication cursor to `to`.**
+  - Why: replication cursor tracks the sender's last known replication state, outlives the current plan, but is required to guarantee that future invocations of this algorithm find an incremental path.
+  - Impl for 'atomic' move: have two cursors (fixes [#177](https://github.com/zrepl/zrepl/issues/177))
+      - Idempotently cursor-bookmark `to` using `idempotent_bookmark(to, to.guid, #zrepl_replication_cursor_G_${to.guid}_J_${jobid})`
+      - Idempotently destroy old cursor-bookmark of `from` `idempotent_destroy(#zrepl_replication_cursor_G_${from.guid}_J_${jobid}, from)`
+    - If `from` is a snapshot, release the hold on it using `idempotent_release(from,  zrepl_J_${jobid})`
+    - FIXME: resumability if we crash inbetween those operations (or scrap it and wait for channel programs to bring atomicity)
+
+- Receiving side: **`idempotent_hold(to, zrepl_FS_J_${jobid})` because its going to be the next step's `from`**
+  - As discussed in the section on the per-step algorithm, this is a feature provided to us by the per-step algorithm.
+
+- Receiving side + Sending side (order doesn't matter): Make sure all holds & step bookmarks made by this plan on already replicated snapshots are released / destroyed:
+  - `idempotent_release_prior_and_including(from, zrepl_FS_TODO)`
+  - `idempotent_step_bookmark_destroy_prior_and_including(from, zrepl_FS_TODO)`
+
+**Cleanup** receiving and sending side (order doesn't matter)
+
+  - `idempotent_release_prior_and_including(STEPS[-1].to, zrepl_FS_TODO)`
+  - `idempotent_step_bookmark_destroy_prior_and_including(STEPS[-1].from, zrepl_FS_TODO)`
+
+### Notes
+
+- `idempotent_FSSTEP_bookmark` is like `idempotent_STEP_bookmark`, but with prefix `zrepl_FSSTEP` instead of `zrepl_STEP`
+
+
+<a id="zrepl-algo-multiple-filesystems-notes"></a>
+
+## Notes on Planning and Executing Replication of Multiple Filesystems
+
+### The RPC Interfaces Uses Send-side Filesystem Names to Identify a Filesystem
+
+The model of the receiver includes the assumption that it will `Receive` all snapshot streams sent to it into filesystems with paths prefixed with `root_fs`.
+This is useful for separating filesystem path namespaces for multiple clients.
+The receiver hides this prefixing in its RPC interface, i.e., when responding to `ListFilesystems` rpcs, the prefix is removed before sending the response.
+This behavior is useful to achieve symmetry in this algorithm: we do not need to take the prefixing into consideration when computing diffs.
+For the receiver, it has the advantage that `Receive` rpc can enforce the namespacing and need not trust this algorithm to apply it correctly.
+
+The receiver is also allowed (and may need to) do implement other filtering / namespace transformations.
+For example, when only `foo/a/b` has been received, but not `foo` nor `foo/a`, the receiver must not include the latter two in its `ListFilesystems` response.
+The current zrepl receiver endpoint implementation uses the `zrepl:placeholder` property to mark filesystems `foo` and `foo/a` as placeholders, and filters out such filesystems in the `ListFilesystems` response.
+Another approach would be to have a flat list of received filesystems per sender, and have a separate table that associates send-side names and receive-side filesystem paths.
+
+Similarly, the sender is allowed to filter the output of its RPC interface to hide certain filesystems or snapshots.
+
+#### Consequences of the Above Design
+
+Namespace filtering and transformations of filesystem paths on sender and receiver are user-configurable.
+Both ends of the replication setup have their own config, and do not coordinate config changes.
+This results in a number of challenges:
+
+- A sender might change its filter to allow additional filesystems to be replicated.
+  For example, where `foo/a/b` was initially allowed, the new filter might allow `foo` and `foo/a` as well.
+  If the receiver uses the `zrepl:placeholder` approach as sketched out above, this means that the receiver will need to replace the placeholder filesystems `$root_fs/foo` and `$root_fs/foo/a` with the incoming full sends of `foo` and `foo/a`.
+
+- Send-side renames cannot be handled efficiently because send-side rename effectively changes the filesystem identity because we use its name.

replication/driver/replication_driver.go

@@ -371,9 +371,21 @@ func (a *attempt) do(ctx context.Context, prev *attempt) {
 }
 
 func (fs *fs) do(ctx context.Context, pq *stepQueue, prev *fs) {
-	psteps, err := fs.fs.PlanFS(ctx)
-	errTime := time.Now()
+
 	defer fs.l.Lock().Unlock()
+
+	// get planned steps from replication logic
+	var psteps []Step
+	var errTime time.Time
+	var err error
+	fs.l.DropWhile(func() {
+		// TODO hacky
+		// choose target time that is earlier than any snapshot, so fs planning is always prioritized
+		targetDate := time.Unix(0, 0)
+		defer pq.WaitReady(fs, targetDate)()
+		psteps, err = fs.fs.PlanFS(ctx) // no shadow
+		errTime = time.Now()            // no shadow
+	})
 	debug := debugPrefix("fs=%s", fs.fs.ReportInfo().Name)
 	fs.planning.done = true
 	if err != nil {

replication/logic/pdu/pdu.pb.go

@@ -23,6 +23,32 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
+type Tri int32
+
+const (
+	Tri_DontCare Tri = 0
+	Tri_False    Tri = 1
+	Tri_True     Tri = 2
+)
+
+var Tri_name = map[int32]string{
+	0: "DontCare",
+	1: "False",
+	2: "True",
+}
+var Tri_value = map[string]int32{
+	"DontCare": 0,
+	"False":    1,
+	"True":     2,
+}
+
+func (x Tri) String() string {
+	return proto.EnumName(Tri_name, int32(x))
+}
+func (Tri) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{0}
+}
+
 type FilesystemVersion_VersionType int32
 
 const (
@@ -43,7 +69,7 @@ func (x FilesystemVersion_VersionType) String() string {
 	return proto.EnumName(FilesystemVersion_VersionType_name, int32(x))
 }
 func (FilesystemVersion_VersionType) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{5, 0}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{5, 0}
 }
 
 type ListFilesystemReq struct {
@@ -56,7 +82,7 @@ func (m *ListFilesystemReq) Reset()         { *m = ListFilesystemReq{} }
 func (m *ListFilesystemReq) String() string { return proto.CompactTextString(m) }
 func (*ListFilesystemReq) ProtoMessage()    {}
 func (*ListFilesystemReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{0}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{0}
 }
 func (m *ListFilesystemReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ListFilesystemReq.Unmarshal(m, b)
@@ -87,7 +113,7 @@ func (m *ListFilesystemRes) Reset()         { *m = ListFilesystemRes{} }
 func (m *ListFilesystemRes) String() string { return proto.CompactTextString(m) }
 func (*ListFilesystemRes) ProtoMessage()    {}
 func (*ListFilesystemRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{1}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{1}
 }
 func (m *ListFilesystemRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ListFilesystemRes.Unmarshal(m, b)
@@ -118,6 +144,7 @@ type Filesystem struct {
 	Path                 string   `protobuf:"bytes,1,opt,name=Path,proto3" json:"Path,omitempty"`
 	ResumeToken          string   `protobuf:"bytes,2,opt,name=ResumeToken,proto3" json:"ResumeToken,omitempty"`
 	IsPlaceholder        bool     `protobuf:"varint,3,opt,name=IsPlaceholder,proto3" json:"IsPlaceholder,omitempty"`
+	IsEncrypted          bool     `protobuf:"varint,4,opt,name=IsEncrypted,proto3" json:"IsEncrypted,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -127,7 +154,7 @@ func (m *Filesystem) Reset()         { *m = Filesystem{} }
 func (m *Filesystem) String() string { return proto.CompactTextString(m) }
 func (*Filesystem) ProtoMessage()    {}
 func (*Filesystem) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{2}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{2}
 }
 func (m *Filesystem) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Filesystem.Unmarshal(m, b)
@@ -168,6 +195,13 @@ func (m *Filesystem) GetIsPlaceholder() bool {
 	return false
 }
 
+func (m *Filesystem) GetIsEncrypted() bool {
+	if m != nil {
+		return m.IsEncrypted
+	}
+	return false
+}
+
 type ListFilesystemVersionsReq struct {
 	Filesystem           string   `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
@@ -179,7 +213,7 @@ func (m *ListFilesystemVersionsReq) Reset()         { *m = ListFilesystemVersion
 func (m *ListFilesystemVersionsReq) String() string { return proto.CompactTextString(m) }
 func (*ListFilesystemVersionsReq) ProtoMessage()    {}
 func (*ListFilesystemVersionsReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{3}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{3}
 }
 func (m *ListFilesystemVersionsReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ListFilesystemVersionsReq.Unmarshal(m, b)
@@ -217,7 +251,7 @@ func (m *ListFilesystemVersionsRes) Reset()         { *m = ListFilesystemVersion
 func (m *ListFilesystemVersionsRes) String() string { return proto.CompactTextString(m) }
 func (*ListFilesystemVersionsRes) ProtoMessage()    {}
 func (*ListFilesystemVersionsRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{4}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{4}
 }
 func (m *ListFilesystemVersionsRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ListFilesystemVersionsRes.Unmarshal(m, b)
@@ -259,7 +293,7 @@ func (m *FilesystemVersion) Reset()         { *m = FilesystemVersion{} }
 func (m *FilesystemVersion) String() string { return proto.CompactTextString(m) }
 func (*FilesystemVersion) ProtoMessage()    {}
 func (*FilesystemVersion) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{5}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{5}
 }
 func (m *FilesystemVersion) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_FilesystemVersion.Unmarshal(m, b)
@@ -316,20 +350,18 @@ func (m *FilesystemVersion) GetCreation() string {
 
 type SendReq struct {
 	Filesystem string `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"`
-	From       string `protobuf:"bytes,2,opt,name=From,proto3" json:"From,omitempty"`
-	// May be empty / null to request a full transfer of From
-	To string `protobuf:"bytes,3,opt,name=To,proto3" json:"To,omitempty"`
-	// If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender.
-	// The sender MUST indicate in SendRes.UsedResumeToken
-	// If it does not work, the sender SHOULD clear the resume token on their side
-	// and use From and To instead
-	// If ResumeToken is not empty, the GUIDs of From and To
-	// MUST correspond to those encoded in the ResumeToken.
-	// Otherwise, the Sender MUST return an error.
+	// May be empty / null to request a full transfer of To
+	From *FilesystemVersion `protobuf:"bytes,2,opt,name=From,proto3" json:"From,omitempty"`
+	To   *FilesystemVersion `protobuf:"bytes,3,opt,name=To,proto3" json:"To,omitempty"`
+	// If ResumeToken is not empty, the resume token that CAN be used for 'zfs
+	// send' by the sender. The sender MUST indicate use of ResumeToken in the
+	// reply message SendRes.UsedResumeToken If it does not work, the sender
+	// SHOULD clear the resume token on their side and use From and To instead If
+	// ResumeToken is not empty, the GUIDs of From and To MUST correspond to those
+	// encoded in the ResumeToken. Otherwise, the Sender MUST return an error.
 	ResumeToken          string   `protobuf:"bytes,4,opt,name=ResumeToken,proto3" json:"ResumeToken,omitempty"`
-	Compress             bool     `protobuf:"varint,5,opt,name=Compress,proto3" json:"Compress,omitempty"`
-	Dedup                bool     `protobuf:"varint,6,opt,name=Dedup,proto3" json:"Dedup,omitempty"`
-	DryRun               bool     `protobuf:"varint,7,opt,name=DryRun,proto3" json:"DryRun,omitempty"`
+	Encrypted            Tri      `protobuf:"varint,5,opt,name=Encrypted,proto3,enum=Tri" json:"Encrypted,omitempty"`
+	DryRun               bool     `protobuf:"varint,6,opt,name=DryRun,proto3" json:"DryRun,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -339,7 +371,7 @@ func (m *SendReq) Reset()         { *m = SendReq{} }
 func (m *SendReq) String() string { return proto.CompactTextString(m) }
 func (*SendReq) ProtoMessage()    {}
 func (*SendReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{6}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{6}
 }
 func (m *SendReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_SendReq.Unmarshal(m, b)
@@ -366,18 +398,18 @@ func (m *SendReq) GetFilesystem() string {
 	return ""
 }
 
-func (m *SendReq) GetFrom() string {
+func (m *SendReq) GetFrom() *FilesystemVersion {
 	if m != nil {
 		return m.From
 	}
-	return ""
+	return nil
 }
 
-func (m *SendReq) GetTo() string {
+func (m *SendReq) GetTo() *FilesystemVersion {
 	if m != nil {
 		return m.To
 	}
-	return ""
+	return nil
 }
 
 func (m *SendReq) GetResumeToken() string {
@@ -387,18 +419,11 @@ func (m *SendReq) GetResumeToken() string {
 	return ""
 }
 
-func (m *SendReq) GetCompress() bool {
+func (m *SendReq) GetEncrypted() Tri {
 	if m != nil {
-		return m.Compress
+		return m.Encrypted
 	}
-	return false
-}
-
-func (m *SendReq) GetDedup() bool {
-	if m != nil {
-		return m.Dedup
-	}
-	return false
+	return Tri_DontCare
 }
 
 func (m *SendReq) GetDryRun() bool {
@@ -420,7 +445,7 @@ func (m *Property) Reset()         { *m = Property{} }
 func (m *Property) String() string { return proto.CompactTextString(m) }
 func (*Property) ProtoMessage()    {}
 func (*Property) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{7}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{7}
 }
 func (m *Property) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Property.Unmarshal(m, b)
@@ -456,6 +481,7 @@ func (m *Property) GetValue() string {
 
 type SendRes struct {
 	// Whether the resume token provided in the request has been used or not.
+	// If the SendReq.ResumeToken == "", this field has no meaning.
 	UsedResumeToken bool `protobuf:"varint,2,opt,name=UsedResumeToken,proto3" json:"UsedResumeToken,omitempty"`
 	// Expected stream size determined by dry run, not exact.
 	// 0 indicates that for the given SendReq, no size estimate could be made.
@@ -470,7 +496,7 @@ func (m *SendRes) Reset()         { *m = SendRes{} }
 func (m *SendRes) String() string { return proto.CompactTextString(m) }
 func (*SendRes) ProtoMessage()    {}
 func (*SendRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{8}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{8}
 }
 func (m *SendRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_SendRes.Unmarshal(m, b)
@@ -511,10 +537,80 @@ func (m *SendRes) GetProperties() []*Property {
 	return nil
 }
 
+type SendCompletedReq struct {
+	OriginalReq          *SendReq `protobuf:"bytes,2,opt,name=OriginalReq,proto3" json:"OriginalReq,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *SendCompletedReq) Reset()         { *m = SendCompletedReq{} }
+func (m *SendCompletedReq) String() string { return proto.CompactTextString(m) }
+func (*SendCompletedReq) ProtoMessage()    {}
+func (*SendCompletedReq) Descriptor() ([]byte, []int) {
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{9}
+}
+func (m *SendCompletedReq) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_SendCompletedReq.Unmarshal(m, b)
+}
+func (m *SendCompletedReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_SendCompletedReq.Marshal(b, m, deterministic)
+}
+func (dst *SendCompletedReq) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_SendCompletedReq.Merge(dst, src)
+}
+func (m *SendCompletedReq) XXX_Size() int {
+	return xxx_messageInfo_SendCompletedReq.Size(m)
+}
+func (m *SendCompletedReq) XXX_DiscardUnknown() {
+	xxx_messageInfo_SendCompletedReq.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_SendCompletedReq proto.InternalMessageInfo
+
+func (m *SendCompletedReq) GetOriginalReq() *SendReq {
+	if m != nil {
+		return m.OriginalReq
+	}
+	return nil
+}
+
+type SendCompletedRes struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *SendCompletedRes) Reset()         { *m = SendCompletedRes{} }
+func (m *SendCompletedRes) String() string { return proto.CompactTextString(m) }
+func (*SendCompletedRes) ProtoMessage()    {}
+func (*SendCompletedRes) Descriptor() ([]byte, []int) {
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{10}
+}
+func (m *SendCompletedRes) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_SendCompletedRes.Unmarshal(m, b)
+}
+func (m *SendCompletedRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_SendCompletedRes.Marshal(b, m, deterministic)
+}
+func (dst *SendCompletedRes) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_SendCompletedRes.Merge(dst, src)
+}
+func (m *SendCompletedRes) XXX_Size() int {
+	return xxx_messageInfo_SendCompletedRes.Size(m)
+}
+func (m *SendCompletedRes) XXX_DiscardUnknown() {
+	xxx_messageInfo_SendCompletedRes.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_SendCompletedRes proto.InternalMessageInfo
+
 type ReceiveReq struct {
 	Filesystem string             `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"`
-	// If true, the receiver should clear the resume token before perfoming the zfs recv of the stream in the request
-	ClearResumeToken     bool     `protobuf:"varint,2,opt,name=ClearResumeToken,proto3" json:"ClearResumeToken,omitempty"`
+	To         *FilesystemVersion `protobuf:"bytes,2,opt,name=To,proto3" json:"To,omitempty"`
+	// If true, the receiver should clear the resume token before perfoming the
+	// zfs recv of the stream in the request
+	ClearResumeToken     bool     `protobuf:"varint,3,opt,name=ClearResumeToken,proto3" json:"ClearResumeToken,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -524,7 +620,7 @@ func (m *ReceiveReq) Reset()         { *m = ReceiveReq{} }
 func (m *ReceiveReq) String() string { return proto.CompactTextString(m) }
 func (*ReceiveReq) ProtoMessage()    {}
 func (*ReceiveReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{9}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{11}
 }
 func (m *ReceiveReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ReceiveReq.Unmarshal(m, b)
@@ -551,6 +647,13 @@ func (m *ReceiveReq) GetFilesystem() string {
 	return ""
 }
 
+func (m *ReceiveReq) GetTo() *FilesystemVersion {
+	if m != nil {
+		return m.To
+	}
+	return nil
+}
+
 func (m *ReceiveReq) GetClearResumeToken() bool {
 	if m != nil {
 		return m.ClearResumeToken
@@ -568,7 +671,7 @@ func (m *ReceiveRes) Reset()         { *m = ReceiveRes{} }
 func (m *ReceiveRes) String() string { return proto.CompactTextString(m) }
 func (*ReceiveRes) ProtoMessage()    {}
 func (*ReceiveRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{10}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{12}
 }
 func (m *ReceiveRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ReceiveRes.Unmarshal(m, b)
@@ -601,7 +704,7 @@ func (m *DestroySnapshotsReq) Reset()         { *m = DestroySnapshotsReq{} }
 func (m *DestroySnapshotsReq) String() string { return proto.CompactTextString(m) }
 func (*DestroySnapshotsReq) ProtoMessage()    {}
 func (*DestroySnapshotsReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{11}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{13}
 }
 func (m *DestroySnapshotsReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_DestroySnapshotsReq.Unmarshal(m, b)
@@ -647,7 +750,7 @@ func (m *DestroySnapshotRes) Reset()         { *m = DestroySnapshotRes{} }
 func (m *DestroySnapshotRes) String() string { return proto.CompactTextString(m) }
 func (*DestroySnapshotRes) ProtoMessage()    {}
 func (*DestroySnapshotRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{12}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{14}
 }
 func (m *DestroySnapshotRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_DestroySnapshotRes.Unmarshal(m, b)
@@ -692,7 +795,7 @@ func (m *DestroySnapshotsRes) Reset()         { *m = DestroySnapshotsRes{} }
 func (m *DestroySnapshotsRes) String() string { return proto.CompactTextString(m) }
 func (*DestroySnapshotsRes) ProtoMessage()    {}
 func (*DestroySnapshotsRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{13}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{15}
 }
 func (m *DestroySnapshotsRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_DestroySnapshotsRes.Unmarshal(m, b)
@@ -721,10 +824,6 @@ func (m *DestroySnapshotsRes) GetResults() []*DestroySnapshotRes {
 
 type ReplicationCursorReq struct {
 	Filesystem           string   `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"`
-	// Types that are valid to be assigned to Op:
-	//	*ReplicationCursorReq_Get
-	//	*ReplicationCursorReq_Set
-	Op                   isReplicationCursorReq_Op `protobuf_oneof:"op"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
 	XXX_unrecognized     []byte   `json:"-"`
 	XXX_sizecache        int32    `json:"-"`
@@ -734,7 +833,7 @@ func (m *ReplicationCursorReq) Reset()         { *m = ReplicationCursorReq{} }
 func (m *ReplicationCursorReq) String() string { return proto.CompactTextString(m) }
 func (*ReplicationCursorReq) ProtoMessage()    {}
 func (*ReplicationCursorReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{14}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{16}
 }
 func (m *ReplicationCursorReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ReplicationCursorReq.Unmarshal(m, b)
@@ -761,185 +860,6 @@ func (m *ReplicationCursorReq) GetFilesystem() string {
 	return ""
 }
 
-type isReplicationCursorReq_Op interface {
-	isReplicationCursorReq_Op()
-}
-
-type ReplicationCursorReq_Get struct {
-	Get *ReplicationCursorReq_GetOp `protobuf:"bytes,2,opt,name=get,proto3,oneof"`
-}
-
-type ReplicationCursorReq_Set struct {
-	Set *ReplicationCursorReq_SetOp `protobuf:"bytes,3,opt,name=set,proto3,oneof"`
-}
-
-func (*ReplicationCursorReq_Get) isReplicationCursorReq_Op() {}
-
-func (*ReplicationCursorReq_Set) isReplicationCursorReq_Op() {}
-
-func (m *ReplicationCursorReq) GetOp() isReplicationCursorReq_Op {
-	if m != nil {
-		return m.Op
-	}
-	return nil
-}
-
-func (m *ReplicationCursorReq) GetGet() *ReplicationCursorReq_GetOp {
-	if x, ok := m.GetOp().(*ReplicationCursorReq_Get); ok {
-		return x.Get
-	}
-	return nil
-}
-
-func (m *ReplicationCursorReq) GetSet() *ReplicationCursorReq_SetOp {
-	if x, ok := m.GetOp().(*ReplicationCursorReq_Set); ok {
-		return x.Set
-	}
-	return nil
-}
-
-// XXX_OneofFuncs is for the internal use of the proto package.
-func (*ReplicationCursorReq) XXX_OneofFuncs() (func(msg proto.Message, b *proto.Buffer) error, func(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error), func(msg proto.Message) (n int), []interface{}) {
-	return _ReplicationCursorReq_OneofMarshaler, _ReplicationCursorReq_OneofUnmarshaler, _ReplicationCursorReq_OneofSizer, []interface{}{
-		(*ReplicationCursorReq_Get)(nil),
-		(*ReplicationCursorReq_Set)(nil),
-	}
-}
-
-func _ReplicationCursorReq_OneofMarshaler(msg proto.Message, b *proto.Buffer) error {
-	m := msg.(*ReplicationCursorReq)
-	// op
-	switch x := m.Op.(type) {
-	case *ReplicationCursorReq_Get:
-		b.EncodeVarint(2<<3 | proto.WireBytes)
-		if err := b.EncodeMessage(x.Get); err != nil {
-			return err
-		}
-	case *ReplicationCursorReq_Set:
-		b.EncodeVarint(3<<3 | proto.WireBytes)
-		if err := b.EncodeMessage(x.Set); err != nil {
-			return err
-		}
-	case nil:
-	default:
-		return fmt.Errorf("ReplicationCursorReq.Op has unexpected type %T", x)
-	}
-	return nil
-}
-
-func _ReplicationCursorReq_OneofUnmarshaler(msg proto.Message, tag, wire int, b *proto.Buffer) (bool, error) {
-	m := msg.(*ReplicationCursorReq)
-	switch tag {
-	case 2: // op.get
-		if wire != proto.WireBytes {
-			return true, proto.ErrInternalBadWireType
-		}
-		msg := new(ReplicationCursorReq_GetOp)
-		err := b.DecodeMessage(msg)
-		m.Op = &ReplicationCursorReq_Get{msg}
-		return true, err
-	case 3: // op.set
-		if wire != proto.WireBytes {
-			return true, proto.ErrInternalBadWireType
-		}
-		msg := new(ReplicationCursorReq_SetOp)
-		err := b.DecodeMessage(msg)
-		m.Op = &ReplicationCursorReq_Set{msg}
-		return true, err
-	default:
-		return false, nil
-	}
-}
-
-func _ReplicationCursorReq_OneofSizer(msg proto.Message) (n int) {
-	m := msg.(*ReplicationCursorReq)
-	// op
-	switch x := m.Op.(type) {
-	case *ReplicationCursorReq_Get:
-		s := proto.Size(x.Get)
-		n += 1 // tag and wire
-		n += proto.SizeVarint(uint64(s))
-		n += s
-	case *ReplicationCursorReq_Set:
-		s := proto.Size(x.Set)
-		n += 1 // tag and wire
-		n += proto.SizeVarint(uint64(s))
-		n += s
-	case nil:
-	default:
-		panic(fmt.Sprintf("proto: unexpected type %T in oneof", x))
-	}
-	return n
-}
-
-type ReplicationCursorReq_GetOp struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *ReplicationCursorReq_GetOp) Reset()         { *m = ReplicationCursorReq_GetOp{} }
-func (m *ReplicationCursorReq_GetOp) String() string { return proto.CompactTextString(m) }
-func (*ReplicationCursorReq_GetOp) ProtoMessage()    {}
-func (*ReplicationCursorReq_GetOp) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{14, 0}
-}
-func (m *ReplicationCursorReq_GetOp) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ReplicationCursorReq_GetOp.Unmarshal(m, b)
-}
-func (m *ReplicationCursorReq_GetOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ReplicationCursorReq_GetOp.Marshal(b, m, deterministic)
-}
-func (dst *ReplicationCursorReq_GetOp) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationCursorReq_GetOp.Merge(dst, src)
-}
-func (m *ReplicationCursorReq_GetOp) XXX_Size() int {
-	return xxx_messageInfo_ReplicationCursorReq_GetOp.Size(m)
-}
-func (m *ReplicationCursorReq_GetOp) XXX_DiscardUnknown() {
-	xxx_messageInfo_ReplicationCursorReq_GetOp.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_ReplicationCursorReq_GetOp proto.InternalMessageInfo
-
-type ReplicationCursorReq_SetOp struct {
-	Snapshot             string   `protobuf:"bytes,2,opt,name=Snapshot,proto3" json:"Snapshot,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *ReplicationCursorReq_SetOp) Reset()         { *m = ReplicationCursorReq_SetOp{} }
-func (m *ReplicationCursorReq_SetOp) String() string { return proto.CompactTextString(m) }
-func (*ReplicationCursorReq_SetOp) ProtoMessage()    {}
-func (*ReplicationCursorReq_SetOp) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{14, 1}
-}
-func (m *ReplicationCursorReq_SetOp) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_ReplicationCursorReq_SetOp.Unmarshal(m, b)
-}
-func (m *ReplicationCursorReq_SetOp) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_ReplicationCursorReq_SetOp.Marshal(b, m, deterministic)
-}
-func (dst *ReplicationCursorReq_SetOp) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ReplicationCursorReq_SetOp.Merge(dst, src)
-}
-func (m *ReplicationCursorReq_SetOp) XXX_Size() int {
-	return xxx_messageInfo_ReplicationCursorReq_SetOp.Size(m)
-}
-func (m *ReplicationCursorReq_SetOp) XXX_DiscardUnknown() {
-	xxx_messageInfo_ReplicationCursorReq_SetOp.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_ReplicationCursorReq_SetOp proto.InternalMessageInfo
-
-func (m *ReplicationCursorReq_SetOp) GetSnapshot() string {
-	if m != nil {
-		return m.Snapshot
-	}
-	return ""
-}
-
 type ReplicationCursorRes struct {
 	// Types that are valid to be assigned to Result:
 	//	*ReplicationCursorRes_Guid
@@ -954,7 +874,7 @@ func (m *ReplicationCursorRes) Reset()         { *m = ReplicationCursorRes{} }
 func (m *ReplicationCursorRes) String() string { return proto.CompactTextString(m) }
 func (*ReplicationCursorRes) ProtoMessage()    {}
 func (*ReplicationCursorRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{15}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{17}
 }
 func (m *ReplicationCursorRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ReplicationCursorRes.Unmarshal(m, b)
@@ -1090,7 +1010,7 @@ func (m *PingReq) Reset()         { *m = PingReq{} }
 func (m *PingReq) String() string { return proto.CompactTextString(m) }
 func (*PingReq) ProtoMessage()    {}
 func (*PingReq) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{16}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{18}
 }
 func (m *PingReq) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_PingReq.Unmarshal(m, b)
@@ -1129,7 +1049,7 @@ func (m *PingRes) Reset()         { *m = PingRes{} }
 func (m *PingRes) String() string { return proto.CompactTextString(m) }
 func (*PingRes) ProtoMessage()    {}
 func (*PingRes) Descriptor() ([]byte, []int) {
-	return fileDescriptor_pdu_83b7e2a28d820622, []int{17}
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{19}
 }
 func (m *PingRes) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_PingRes.Unmarshal(m, b)
@@ -1156,6 +1076,82 @@ func (m *PingRes) GetEcho() string {
 	return ""
 }
 
+type HintMostRecentCommonAncestorReq struct {
+	Filesystem           string             `protobuf:"bytes,1,opt,name=Filesystem,proto3" json:"Filesystem,omitempty"`
+	SenderVersion        *FilesystemVersion `protobuf:"bytes,2,opt,name=SenderVersion,proto3" json:"SenderVersion,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}           `json:"-"`
+	XXX_unrecognized     []byte             `json:"-"`
+	XXX_sizecache        int32              `json:"-"`
+}
+
+func (m *HintMostRecentCommonAncestorReq) Reset()         { *m = HintMostRecentCommonAncestorReq{} }
+func (m *HintMostRecentCommonAncestorReq) String() string { return proto.CompactTextString(m) }
+func (*HintMostRecentCommonAncestorReq) ProtoMessage()    {}
+func (*HintMostRecentCommonAncestorReq) Descriptor() ([]byte, []int) {
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{20}
+}
+func (m *HintMostRecentCommonAncestorReq) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HintMostRecentCommonAncestorReq.Unmarshal(m, b)
+}
+func (m *HintMostRecentCommonAncestorReq) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HintMostRecentCommonAncestorReq.Marshal(b, m, deterministic)
+}
+func (dst *HintMostRecentCommonAncestorReq) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HintMostRecentCommonAncestorReq.Merge(dst, src)
+}
+func (m *HintMostRecentCommonAncestorReq) XXX_Size() int {
+	return xxx_messageInfo_HintMostRecentCommonAncestorReq.Size(m)
+}
+func (m *HintMostRecentCommonAncestorReq) XXX_DiscardUnknown() {
+	xxx_messageInfo_HintMostRecentCommonAncestorReq.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HintMostRecentCommonAncestorReq proto.InternalMessageInfo
+
+func (m *HintMostRecentCommonAncestorReq) GetFilesystem() string {
+	if m != nil {
+		return m.Filesystem
+	}
+	return ""
+}
+
+func (m *HintMostRecentCommonAncestorReq) GetSenderVersion() *FilesystemVersion {
+	if m != nil {
+		return m.SenderVersion
+	}
+	return nil
+}
+
+type HintMostRecentCommonAncestorRes struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HintMostRecentCommonAncestorRes) Reset()         { *m = HintMostRecentCommonAncestorRes{} }
+func (m *HintMostRecentCommonAncestorRes) String() string { return proto.CompactTextString(m) }
+func (*HintMostRecentCommonAncestorRes) ProtoMessage()    {}
+func (*HintMostRecentCommonAncestorRes) Descriptor() ([]byte, []int) {
+	return fileDescriptor_pdu_0f43b713cd3bf056, []int{21}
+}
+func (m *HintMostRecentCommonAncestorRes) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HintMostRecentCommonAncestorRes.Unmarshal(m, b)
+}
+func (m *HintMostRecentCommonAncestorRes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HintMostRecentCommonAncestorRes.Marshal(b, m, deterministic)
+}
+func (dst *HintMostRecentCommonAncestorRes) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HintMostRecentCommonAncestorRes.Merge(dst, src)
+}
+func (m *HintMostRecentCommonAncestorRes) XXX_Size() int {
+	return xxx_messageInfo_HintMostRecentCommonAncestorRes.Size(m)
+}
+func (m *HintMostRecentCommonAncestorRes) XXX_DiscardUnknown() {
+	xxx_messageInfo_HintMostRecentCommonAncestorRes.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HintMostRecentCommonAncestorRes proto.InternalMessageInfo
+
 func init() {
 	proto.RegisterType((*ListFilesystemReq)(nil), "ListFilesystemReq")
 	proto.RegisterType((*ListFilesystemRes)(nil), "ListFilesystemRes")
@@ -1166,17 +1162,20 @@ func init() {
 	proto.RegisterType((*SendReq)(nil), "SendReq")
 	proto.RegisterType((*Property)(nil), "Property")
 	proto.RegisterType((*SendRes)(nil), "SendRes")
+	proto.RegisterType((*SendCompletedReq)(nil), "SendCompletedReq")
+	proto.RegisterType((*SendCompletedRes)(nil), "SendCompletedRes")
 	proto.RegisterType((*ReceiveReq)(nil), "ReceiveReq")
 	proto.RegisterType((*ReceiveRes)(nil), "ReceiveRes")
 	proto.RegisterType((*DestroySnapshotsReq)(nil), "DestroySnapshotsReq")
 	proto.RegisterType((*DestroySnapshotRes)(nil), "DestroySnapshotRes")
 	proto.RegisterType((*DestroySnapshotsRes)(nil), "DestroySnapshotsRes")
 	proto.RegisterType((*ReplicationCursorReq)(nil), "ReplicationCursorReq")
-	proto.RegisterType((*ReplicationCursorReq_GetOp)(nil), "ReplicationCursorReq.GetOp")
-	proto.RegisterType((*ReplicationCursorReq_SetOp)(nil), "ReplicationCursorReq.SetOp")
 	proto.RegisterType((*ReplicationCursorRes)(nil), "ReplicationCursorRes")
 	proto.RegisterType((*PingReq)(nil), "PingReq")
 	proto.RegisterType((*PingRes)(nil), "PingRes")
+	proto.RegisterType((*HintMostRecentCommonAncestorReq)(nil), "HintMostRecentCommonAncestorReq")
+	proto.RegisterType((*HintMostRecentCommonAncestorRes)(nil), "HintMostRecentCommonAncestorRes")
+	proto.RegisterEnum("Tri", Tri_name, Tri_value)
 	proto.RegisterEnum("FilesystemVersion_VersionType", FilesystemVersion_VersionType_name, FilesystemVersion_VersionType_value)
 }
 
@@ -1197,6 +1196,8 @@ type ReplicationClient interface {
 	ListFilesystemVersions(ctx context.Context, in *ListFilesystemVersionsReq, opts ...grpc.CallOption) (*ListFilesystemVersionsRes, error)
 	DestroySnapshots(ctx context.Context, in *DestroySnapshotsReq, opts ...grpc.CallOption) (*DestroySnapshotsRes, error)
 	ReplicationCursor(ctx context.Context, in *ReplicationCursorReq, opts ...grpc.CallOption) (*ReplicationCursorRes, error)
+	SendCompleted(ctx context.Context, in *SendCompletedReq, opts ...grpc.CallOption) (*SendCompletedRes, error)
+	HintMostRecentCommonAncestor(ctx context.Context, in *HintMostRecentCommonAncestorReq, opts ...grpc.CallOption) (*HintMostRecentCommonAncestorRes, error)
 }
 
 type replicationClient struct {
@@ -1252,6 +1253,24 @@ func (c *replicationClient) ReplicationCursor(ctx context.Context, in *Replicati
 	return out, nil
 }
 
+func (c *replicationClient) SendCompleted(ctx context.Context, in *SendCompletedReq, opts ...grpc.CallOption) (*SendCompletedRes, error) {
+	out := new(SendCompletedRes)
+	err := c.cc.Invoke(ctx, "/Replication/SendCompleted", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *replicationClient) HintMostRecentCommonAncestor(ctx context.Context, in *HintMostRecentCommonAncestorReq, opts ...grpc.CallOption) (*HintMostRecentCommonAncestorRes, error) {
+	out := new(HintMostRecentCommonAncestorRes)
+	err := c.cc.Invoke(ctx, "/Replication/HintMostRecentCommonAncestor", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // ReplicationServer is the server API for Replication service.
 type ReplicationServer interface {
 	Ping(context.Context, *PingReq) (*PingRes, error)
@@ -1259,6 +1278,8 @@ type ReplicationServer interface {
 	ListFilesystemVersions(context.Context, *ListFilesystemVersionsReq) (*ListFilesystemVersionsRes, error)
 	DestroySnapshots(context.Context, *DestroySnapshotsReq) (*DestroySnapshotsRes, error)
 	ReplicationCursor(context.Context, *ReplicationCursorReq) (*ReplicationCursorRes, error)
+	SendCompleted(context.Context, *SendCompletedReq) (*SendCompletedRes, error)
+	HintMostRecentCommonAncestor(context.Context, *HintMostRecentCommonAncestorReq) (*HintMostRecentCommonAncestorRes, error)
 }
 
 func RegisterReplicationServer(s *grpc.Server, srv ReplicationServer) {
@@ -1355,6 +1376,42 @@ func _Replication_ReplicationCursor_Handler(srv interface{}, ctx context.Context
 	return interceptor(ctx, in, info, handler)
 }
 
+func _Replication_SendCompleted_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(SendCompletedReq)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ReplicationServer).SendCompleted(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/Replication/SendCompleted",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ReplicationServer).SendCompleted(ctx, req.(*SendCompletedReq))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Replication_HintMostRecentCommonAncestor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(HintMostRecentCommonAncestorReq)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(ReplicationServer).HintMostRecentCommonAncestor(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/Replication/HintMostRecentCommonAncestor",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(ReplicationServer).HintMostRecentCommonAncestor(ctx, req.(*HintMostRecentCommonAncestorReq))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 var _Replication_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "Replication",
 	HandlerType: (*ReplicationServer)(nil),
@@ -1379,63 +1436,77 @@ var _Replication_serviceDesc = grpc.ServiceDesc{
 			MethodName: "ReplicationCursor",
 			Handler:    _Replication_ReplicationCursor_Handler,
 		},
+		{
+			MethodName: "SendCompleted",
+			Handler:    _Replication_SendCompleted_Handler,
+		},
+		{
+			MethodName: "HintMostRecentCommonAncestor",
+			Handler:    _Replication_HintMostRecentCommonAncestor_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "pdu.proto",
 }
 
-func init() { proto.RegisterFile("pdu.proto", fileDescriptor_pdu_83b7e2a28d820622) }
+func init() { proto.RegisterFile("pdu.proto", fileDescriptor_pdu_0f43b713cd3bf056) }
 
-var fileDescriptor_pdu_83b7e2a28d820622 = []byte{
-	// 785 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x55, 0xd1, 0x8e, 0xe3, 0x34,
-	0x14, 0x9d, 0xb4, 0x69, 0x9b, 0xde, 0x0e, 0xbb, 0x1d, 0x4f, 0x59, 0x85, 0x00, 0xab, 0xca, 0xcb,
-	0x43, 0x17, 0x89, 0x80, 0x0a, 0x2f, 0x08, 0x09, 0x89, 0x4e, 0x67, 0x67, 0x10, 0xb0, 0x54, 0x6e,
-	0x59, 0xad, 0xf6, 0x2d, 0x34, 0x57, 0x6d, 0x34, 0x6d, 0x9d, 0xb5, 0x13, 0xb4, 0xe5, 0x91, 0xbf,
-	0x9a, 0x7f, 0xe0, 0x91, 0x0f, 0x42, 0x76, 0xe3, 0x34, 0x6d, 0x52, 0xa9, 0x4f, 0xf1, 0x39, 0xf7,
-	0xda, 0x3e, 0xf7, 0xd8, 0xd7, 0x81, 0x76, 0x1c, 0xa6, 0x7e, 0x2c, 0x78, 0xc2, 0xe9, 0x35, 0x5c,
-	0xfd, 0x1a, 0xc9, 0xe4, 0x55, 0xb4, 0x42, 0xb9, 0x95, 0x09, 0xae, 0x19, 0xbe, 0xa7, 0xa3, 0x32,
-	0x29, 0xc9, 0x57, 0xd0, 0xd9, 0x13, 0xd2, 0xb5, 0xfa, 0xf5, 0x41, 0x67, 0xd8, 0xf1, 0x0b, 0x49,
-	0xc5, 0x38, 0x5d, 0x02, 0xec, 0x21, 0x21, 0x60, 0x4f, 0x82, 0x64, 0xe9, 0x5a, 0x7d, 0x6b, 0xd0,
-	0x66, 0x7a, 0x4c, 0xfa, 0xd0, 0x61, 0x28, 0xd3, 0x35, 0xce, 0xf8, 0x03, 0x6e, 0xdc, 0x9a, 0x0e,
-	0x15, 0x29, 0xf2, 0x05, 0x7c, 0xf4, 0xb3, 0x9c, 0xac, 0x82, 0x39, 0x2e, 0xf9, 0x2a, 0x44, 0xe1,
-	0xd6, 0xfb, 0xd6, 0xc0, 0x61, 0x87, 0x24, 0xfd, 0x01, 0x3e, 0x39, 0x54, 0xfb, 0x06, 0x85, 0x8c,
-	0xf8, 0x46, 0x32, 0x7c, 0x4f, 0x9e, 0x17, 0x65, 0x64, 0xdb, 0x17, 0x18, 0xfa, 0xcb, 0xe9, 0xc9,
-	0x92, 0xf8, 0xe0, 0x18, 0x98, 0xd5, 0x4b, 0xfc, 0x52, 0x26, 0xcb, 0x73, 0xe8, 0x7f, 0x16, 0x5c,
-	0x95, 0xe2, 0x64, 0x08, 0xf6, 0x6c, 0x1b, 0xa3, 0xde, 0xfc, 0xc9, 0xf0, 0x79, 0x79, 0x05, 0x3f,
-	0xfb, 0xaa, 0x2c, 0xa6, 0x73, 0x95, 0x5f, 0xaf, 0x83, 0x35, 0x66, 0xa6, 0xe8, 0xb1, 0xe2, 0xee,
-	0xd2, 0x28, 0xd4, 0x26, 0xd8, 0x4c, 0x8f, 0xc9, 0x67, 0xd0, 0xbe, 0x11, 0x18, 0x24, 0x38, 0x7b,
-	0x7b, 0xe7, 0xda, 0x3a, 0xb0, 0x27, 0x88, 0x07, 0x8e, 0x06, 0x11, 0xdf, 0xb8, 0x0d, 0xbd, 0x52,
-	0x8e, 0xe9, 0x4b, 0xe8, 0x14, 0xb6, 0x25, 0x97, 0xe0, 0x4c, 0x37, 0x41, 0x2c, 0x97, 0x3c, 0xe9,
-	0x5e, 0x28, 0x34, 0xe2, 0xfc, 0x61, 0x1d, 0x88, 0x87, 0xae, 0x45, 0x1f, 0x2d, 0x68, 0x4d, 0x71,
-	0x13, 0x9e, 0xe1, 0xa7, 0x12, 0xf9, 0x4a, 0xf0, 0xb5, 0x11, 0xae, 0xc6, 0xe4, 0x09, 0xd4, 0x66,
-	0x5c, 0xcb, 0x6e, 0xb3, 0xda, 0x8c, 0x1f, 0x1f, 0xbc, 0x5d, 0x3e, 0x78, 0x25, 0x9c, 0xaf, 0x63,
-	0x81, 0x52, 0x6a, 0xe1, 0x0e, 0xcb, 0x31, 0xe9, 0x41, 0x63, 0x8c, 0x61, 0x1a, 0xbb, 0x4d, 0x1d,
-	0xd8, 0x01, 0xf2, 0x0c, 0x9a, 0x63, 0xb1, 0x65, 0xe9, 0xc6, 0x6d, 0x69, 0x3a, 0x43, 0xf4, 0x3b,
-	0x70, 0x26, 0x82, 0xc7, 0x28, 0x92, 0x6d, 0x6e, 0xaa, 0x55, 0x30, 0xb5, 0x07, 0x8d, 0x37, 0xc1,
-	0x2a, 0x35, 0x4e, 0xef, 0x00, 0xfd, 0x27, 0xaf, 0x58, 0x92, 0x01, 0x3c, 0xfd, 0x43, 0x62, 0x78,
-	0x7c, 0x55, 0x1d, 0x76, 0x4c, 0x13, 0x0a, 0x97, 0xb7, 0x1f, 0x62, 0x9c, 0x27, 0x18, 0x4e, 0xa3,
-	0xbf, 0x51, 0x57, 0x5c, 0x67, 0x07, 0x1c, 0x79, 0x09, 0x90, 0xe9, 0x89, 0x50, 0xba, 0xb6, 0xbe,
-	0x54, 0x6d, 0xdf, 0x48, 0x64, 0x85, 0x20, 0x7d, 0x0b, 0xc0, 0x70, 0x8e, 0xd1, 0x5f, 0x78, 0x8e,
-	0xf1, 0x5f, 0x42, 0xf7, 0x66, 0x85, 0x81, 0x28, 0xeb, 0x2c, 0xf1, 0xf4, 0xb2, 0xb0, 0xb2, 0xa4,
-	0x0b, 0xb8, 0x1e, 0xa3, 0x4c, 0x04, 0xdf, 0x9a, 0x1b, 0x70, 0x4e, 0xe7, 0x90, 0x6f, 0xa0, 0x9d,
-	0xe7, 0xbb, 0xb5, 0x93, 0xdd, 0xb1, 0x4f, 0xa2, 0xef, 0x80, 0x1c, 0x6d, 0x94, 0x35, 0x99, 0x81,
-	0x7a, 0x97, 0x13, 0x4d, 0x66, 0x72, 0xd4, 0x89, 0xdd, 0x0a, 0xc1, 0x85, 0x39, 0x31, 0x0d, 0xe8,
-	0xb8, 0xaa, 0x08, 0xf5, 0x68, 0xb5, 0x54, 0xe1, 0xab, 0xc4, 0x34, 0xf0, 0xb5, 0x5f, 0x96, 0xc0,
-	0x4c, 0x0e, 0xfd, 0xd7, 0x82, 0x1e, 0xc3, 0x78, 0x15, 0xcd, 0x75, 0x93, 0xdc, 0xa4, 0x42, 0x72,
-	0x71, 0x8e, 0x19, 0x5f, 0x43, 0x7d, 0x81, 0x89, 0x96, 0xd4, 0x19, 0x7e, 0xea, 0x57, 0xad, 0xe1,
-	0xdf, 0x61, 0xf2, 0x7b, 0x7c, 0x7f, 0xc1, 0x54, 0xa6, 0x9a, 0x20, 0x31, 0xd1, 0x57, 0xe4, 0xe4,
-	0x84, 0xa9, 0x99, 0x20, 0x31, 0xf1, 0x5a, 0xd0, 0xd0, 0x0b, 0x78, 0x2f, 0xa0, 0xa1, 0x03, 0xaa,
-	0x49, 0x72, 0xe3, 0x76, 0x5e, 0xe4, 0x78, 0x64, 0x43, 0x8d, 0xc7, 0x74, 0x56, 0x59, 0x8d, 0x6a,
-	0xa1, 0xdd, 0x4b, 0xa2, 0xea, 0xb0, 0xef, 0x2f, 0xf2, 0xb7, 0xc4, 0x79, 0xcd, 0x13, 0xfc, 0x10,
-	0xc9, 0xdd, 0x7a, 0xce, 0xfd, 0x05, 0xcb, 0x99, 0x91, 0x03, 0xcd, 0x9d, 0x4b, 0xf4, 0x05, 0xb4,
-	0x26, 0xd1, 0x66, 0xa1, 0x6c, 0x71, 0xa1, 0xf5, 0x1b, 0x4a, 0x19, 0x2c, 0x4c, 0x53, 0x19, 0x48,
-	0x3f, 0x37, 0x49, 0x52, 0xb5, 0xdd, 0xed, 0x7c, 0xc9, 0x4d, 0xdb, 0xa9, 0xf1, 0xf0, 0xb1, 0xa6,
-	0xde, 0x80, 0x5c, 0x1a, 0xf1, 0xc0, 0x56, 0xe9, 0xc4, 0xf1, 0xb3, 0xa5, 0x3d, 0x33, 0x92, 0xe4,
-	0x7b, 0x78, 0x7a, 0xf8, 0x44, 0x4b, 0x42, 0xfc, 0xd2, 0x4f, 0xcb, 0x2b, 0x73, 0x92, 0x4c, 0xe0,
-	0x59, 0xf5, 0xeb, 0x4e, 0x3c, 0xff, 0xe4, 0x3f, 0xc3, 0x3b, 0x1d, 0x93, 0xe4, 0x47, 0xe8, 0x1e,
-	0xdf, 0x33, 0xd2, 0xf3, 0x2b, 0xfa, 0xc7, 0xab, 0x62, 0x25, 0xf9, 0x09, 0xae, 0x4a, 0x47, 0x42,
-	0x3e, 0xae, 0x3c, 0x7f, 0xaf, 0x92, 0x96, 0xa3, 0xc6, 0xbb, 0x7a, 0x1c, 0xa6, 0x7f, 0x36, 0xf5,
-	0x0f, 0xfc, 0xdb, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0x37, 0x0e, 0xf2, 0xe4, 0xcd, 0x07, 0x00,
-	0x00,
+var fileDescriptor_pdu_0f43b713cd3bf056 = []byte{
+	// 892 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x56, 0xdf, 0x6f, 0xdb, 0x36,
+	0x10, 0x8e, 0x6c, 0x39, 0x91, 0xcf, 0xe9, 0xea, 0x5c, 0xb2, 0x42, 0x13, 0xba, 0xce, 0xe3, 0x86,
+	0xc1, 0x0d, 0x30, 0x61, 0xc8, 0x7e, 0x60, 0xc3, 0x80, 0x02, 0x8d, 0x93, 0x34, 0xc5, 0xd6, 0xce,
+	0x60, 0xbc, 0x62, 0xe8, 0x9b, 0x6a, 0x1f, 0x12, 0x21, 0xb2, 0xa8, 0x90, 0xf4, 0x50, 0x6f, 0x7b,
+	0xda, 0xe3, 0xfe, 0xbd, 0xe5, 0x0f, 0x2a, 0x44, 0x4b, 0xb6, 0x6c, 0xc9, 0x89, 0x9f, 0xcc, 0xfb,
+	0x78, 0x14, 0xef, 0xbe, 0xfb, 0xee, 0x68, 0x68, 0x26, 0xa3, 0x89, 0x9f, 0x48, 0xa1, 0x05, 0xdb,
+	0x87, 0xbd, 0x5f, 0x43, 0xa5, 0xcf, 0xc2, 0x88, 0xd4, 0x54, 0x69, 0x1a, 0x73, 0xba, 0x61, 0xc7,
+	0x65, 0x50, 0xe1, 0xd7, 0xd0, 0x5a, 0x00, 0xca, 0xb5, 0x3a, 0xf5, 0x6e, 0xeb, 0xa8, 0xe5, 0x17,
+	0x9c, 0x8a, 0xfb, 0xec, 0x3f, 0x0b, 0x60, 0x61, 0x23, 0x82, 0xdd, 0x0f, 0xf4, 0x95, 0x6b, 0x75,
+	0xac, 0x6e, 0x93, 0x9b, 0x35, 0x76, 0xa0, 0xc5, 0x49, 0x4d, 0xc6, 0x34, 0x10, 0xd7, 0x14, 0xbb,
+	0x35, 0xb3, 0x55, 0x84, 0xf0, 0x4b, 0x78, 0xf0, 0x52, 0xf5, 0xa3, 0x60, 0x48, 0x57, 0x22, 0x1a,
+	0x91, 0x74, 0xeb, 0x1d, 0xab, 0xeb, 0xf0, 0x65, 0x30, 0xfd, 0xce, 0x4b, 0x75, 0x1a, 0x0f, 0xe5,
+	0x34, 0xd1, 0x34, 0x72, 0x6d, 0xe3, 0x53, 0x84, 0xd8, 0xcf, 0xf0, 0xc9, 0x72, 0x42, 0x6f, 0x48,
+	0xaa, 0x50, 0xc4, 0x8a, 0xd3, 0x0d, 0x3e, 0x29, 0x06, 0x9a, 0x05, 0x58, 0x40, 0xd8, 0x2f, 0xeb,
+	0x0f, 0x2b, 0xf4, 0xc1, 0xc9, 0xcd, 0x8c, 0x12, 0xf4, 0x4b, 0x9e, 0x7c, 0xee, 0xc3, 0x6e, 0x2d,
+	0xd8, 0x2b, 0xed, 0xe3, 0x11, 0xd8, 0x83, 0x69, 0x42, 0xe6, 0xf2, 0x8f, 0x8e, 0x9e, 0x94, 0xbf,
+	0xe0, 0x67, 0xbf, 0xa9, 0x17, 0x37, 0xbe, 0x29, 0xa3, 0xaf, 0x83, 0x31, 0x65, 0xb4, 0x99, 0x75,
+	0x8a, 0xbd, 0x98, 0x84, 0x23, 0x43, 0x93, 0xcd, 0xcd, 0x1a, 0x1f, 0x43, 0xb3, 0x27, 0x29, 0xd0,
+	0x34, 0xf8, 0xe3, 0x85, 0xe1, 0xc6, 0xe6, 0x0b, 0x00, 0x3d, 0x70, 0x8c, 0x11, 0x8a, 0xd8, 0x6d,
+	0x98, 0x2f, 0xcd, 0x6d, 0xf6, 0x14, 0x5a, 0x85, 0x6b, 0x71, 0x17, 0x9c, 0x8b, 0x38, 0x48, 0xd4,
+	0x95, 0xd0, 0xed, 0xad, 0xd4, 0x3a, 0x16, 0xe2, 0x7a, 0x1c, 0xc8, 0xeb, 0xb6, 0xc5, 0xfe, 0xb7,
+	0x60, 0xe7, 0x82, 0xe2, 0xd1, 0x06, 0x7c, 0xe2, 0x57, 0x60, 0x9f, 0x49, 0x31, 0x36, 0x81, 0x57,
+	0xd3, 0x65, 0xf6, 0x91, 0x41, 0x6d, 0x20, 0x4c, 0x2a, 0xd5, 0x5e, 0xb5, 0x81, 0x58, 0x95, 0x90,
+	0x5d, 0x96, 0x10, 0x83, 0xe6, 0x42, 0x1a, 0x0d, 0xc3, 0xaf, 0xed, 0x0f, 0x64, 0xc8, 0x17, 0x30,
+	0x3e, 0x82, 0xed, 0x13, 0x39, 0xe5, 0x93, 0xd8, 0xdd, 0x36, 0xda, 0xc9, 0x2c, 0xf6, 0x1d, 0x38,
+	0x7d, 0x29, 0x12, 0x92, 0x7a, 0x3a, 0xa7, 0xdb, 0x2a, 0xd0, 0x7d, 0x00, 0x8d, 0x37, 0x41, 0x34,
+	0xc9, 0x6b, 0x30, 0x33, 0xd8, 0xbf, 0x73, 0x2e, 0x14, 0x76, 0xe1, 0xe1, 0xef, 0x8a, 0x46, 0xab,
+	0x32, 0x77, 0xf8, 0x2a, 0x8c, 0x0c, 0x76, 0x4f, 0xdf, 0x27, 0x34, 0xd4, 0x34, 0xba, 0x08, 0xff,
+	0x22, 0x93, 0x77, 0x9d, 0x2f, 0x61, 0xf8, 0x14, 0x20, 0x8b, 0x27, 0x24, 0xe5, 0xda, 0x46, 0x6e,
+	0x4d, 0x3f, 0x0f, 0x91, 0x17, 0x36, 0xd9, 0x33, 0x68, 0xa7, 0x31, 0xf4, 0xc4, 0x38, 0x89, 0x48,
+	0x93, 0x29, 0xcc, 0x21, 0xb4, 0x7e, 0x93, 0xe1, 0x65, 0x18, 0x07, 0x11, 0xa7, 0x9b, 0x8c, 0x7f,
+	0xc7, 0xcf, 0xea, 0xc6, 0x8b, 0x9b, 0x0c, 0x4b, 0xe7, 0x15, 0xfb, 0x07, 0x80, 0xd3, 0x90, 0xc2,
+	0x3f, 0x69, 0x93, 0x32, 0xcf, 0xca, 0x57, 0xbb, 0xb3, 0x7c, 0x87, 0xd0, 0xee, 0x45, 0x14, 0xc8,
+	0x22, 0x3f, 0xb3, 0x16, 0x2f, 0xe1, 0x6c, 0xb7, 0x70, 0xbb, 0x62, 0x97, 0xb0, 0x7f, 0x42, 0x4a,
+	0x4b, 0x31, 0xcd, 0x35, 0xb9, 0x49, 0x2f, 0xe3, 0x37, 0xd0, 0x9c, 0xfb, 0xbb, 0xb5, 0xb5, 0xfd,
+	0xba, 0x70, 0x62, 0x6f, 0x01, 0x57, 0x2e, 0xca, 0xda, 0x3e, 0x37, 0xcd, 0x2d, 0x6b, 0xda, 0x3e,
+	0xf7, 0x49, 0x95, 0x72, 0x2a, 0xa5, 0x90, 0xb9, 0x52, 0x8c, 0xc1, 0x4e, 0xaa, 0x92, 0x48, 0x27,
+	0xed, 0x4e, 0x9a, 0x78, 0xa4, 0xf3, 0x91, 0xb2, 0xef, 0x97, 0x43, 0xe0, 0xb9, 0x0f, 0xfb, 0x01,
+	0x0e, 0x38, 0x25, 0x51, 0x38, 0x34, 0x5d, 0xdb, 0x9b, 0x48, 0x25, 0xe4, 0x26, 0x73, 0x6d, 0x50,
+	0x79, 0x4e, 0xe1, 0x41, 0x36, 0x44, 0xd2, 0x13, 0xf6, 0xf9, 0xd6, 0x7c, 0x8c, 0x38, 0xaf, 0x85,
+	0xa6, 0xf7, 0xa1, 0xd2, 0x33, 0x09, 0x9f, 0x6f, 0xf1, 0x39, 0x72, 0xec, 0xc0, 0xf6, 0x2c, 0x1c,
+	0xf6, 0x05, 0xec, 0xf4, 0xc3, 0xf8, 0x32, 0x0d, 0xc0, 0x85, 0x9d, 0x57, 0xa4, 0x54, 0x70, 0x99,
+	0x77, 0x4d, 0x6e, 0xb2, 0x4f, 0x73, 0x27, 0x95, 0xf6, 0xd5, 0xe9, 0xf0, 0x4a, 0xe4, 0x7d, 0x95,
+	0xae, 0xd9, 0xdf, 0xf0, 0xd9, 0x79, 0x18, 0xeb, 0x57, 0x42, 0xe9, 0xb4, 0xe4, 0xb1, 0xee, 0x89,
+	0xf1, 0x58, 0xc4, 0xcf, 0xe3, 0x21, 0x29, 0xbd, 0x51, 0x72, 0xf8, 0x23, 0x3c, 0x48, 0xf5, 0x4b,
+	0x32, 0xab, 0xc5, 0x1d, 0x42, 0x5c, 0x76, 0x64, 0x9f, 0xdf, 0x77, 0xb9, 0x3a, 0xec, 0x42, 0x7d,
+	0x20, 0xc3, 0x74, 0x04, 0x9e, 0x88, 0x58, 0xf7, 0x02, 0x49, 0xed, 0x2d, 0x6c, 0x42, 0xe3, 0x2c,
+	0x88, 0x14, 0xb5, 0x2d, 0x74, 0xc0, 0x1e, 0xc8, 0x09, 0xb5, 0x6b, 0x47, 0xb7, 0xf5, 0x74, 0x40,
+	0xcd, 0x49, 0x46, 0x0f, 0xec, 0x34, 0x71, 0x74, 0xfc, 0x8c, 0x24, 0x2f, 0x5f, 0x29, 0xfc, 0x09,
+	0x1e, 0x2e, 0xbf, 0x33, 0x0a, 0xd1, 0x2f, 0x3d, 0xce, 0x5e, 0x19, 0x53, 0xd8, 0x87, 0x47, 0xd5,
+	0x4f, 0x14, 0x7a, 0xfe, 0xda, 0x87, 0xcf, 0x5b, 0xbf, 0xa7, 0xf0, 0x19, 0xb4, 0x57, 0xa5, 0x89,
+	0x07, 0x7e, 0x45, 0xcb, 0x79, 0x55, 0xa8, 0xc2, 0xe7, 0xb0, 0x57, 0x12, 0x17, 0x7e, 0xec, 0x57,
+	0x09, 0xd5, 0xab, 0x84, 0x15, 0x7e, 0x3f, 0x2b, 0xe1, 0x7c, 0x04, 0xe1, 0x9e, 0xbf, 0x3a, 0xd2,
+	0xbc, 0x12, 0xa4, 0xf0, 0x1d, 0x3c, 0xbe, 0xab, 0x7e, 0xd8, 0xf1, 0xef, 0xd1, 0x96, 0x77, 0x9f,
+	0x87, 0x3a, 0x6e, 0xbc, 0xad, 0x27, 0xa3, 0xc9, 0xbb, 0x6d, 0xf3, 0x1f, 0xea, 0xdb, 0x0f, 0x01,
+	0x00, 0x00, 0xff, 0xff, 0xad, 0x4e, 0x98, 0x29, 0x50, 0x09, 0x00, 0x00,
 }

replication/logic/pdu/pdu.proto

@@ -2,33 +2,31 @@ syntax = "proto3";
 option go_package = "pdu";
 
 service Replication {
-    rpc Ping (PingReq) returns (PingRes);
-    rpc ListFilesystems (ListFilesystemReq) returns (ListFilesystemRes);
-    rpc ListFilesystemVersions (ListFilesystemVersionsReq) returns (ListFilesystemVersionsRes);
-    rpc DestroySnapshots (DestroySnapshotsReq) returns (DestroySnapshotsRes);
-    rpc ReplicationCursor (ReplicationCursorReq) returns (ReplicationCursorRes);
+  rpc Ping(PingReq) returns (PingRes);
+  rpc ListFilesystems(ListFilesystemReq) returns (ListFilesystemRes);
+  rpc ListFilesystemVersions(ListFilesystemVersionsReq)
+      returns (ListFilesystemVersionsRes);
+  rpc DestroySnapshots(DestroySnapshotsReq) returns (DestroySnapshotsRes);
+  rpc ReplicationCursor(ReplicationCursorReq) returns (ReplicationCursorRes);
+  rpc SendCompleted(SendCompletedReq) returns (SendCompletedRes);
+  rpc HintMostRecentCommonAncestor(HintMostRecentCommonAncestorReq) returns (HintMostRecentCommonAncestorRes);
   // for Send and Recv, see package rpc
 }
 
 message ListFilesystemReq {}
 
-message ListFilesystemRes {
-    repeated Filesystem Filesystems = 1;
-}
+message ListFilesystemRes { repeated Filesystem Filesystems = 1; }
 
 message Filesystem {
   string Path = 1;
   string ResumeToken = 2;
   bool IsPlaceholder = 3;
+  bool IsEncrypted = 4;
 }
 
-message ListFilesystemVersionsReq {
-    string Filesystem = 1;
-}
+message ListFilesystemVersionsReq { string Filesystem = 1; }
 
-message ListFilesystemVersionsRes {
-    repeated FilesystemVersion Versions = 1;
-}
+message ListFilesystemVersionsRes { repeated FilesystemVersion Versions = 1; }
 
 message FilesystemVersion {
   enum VersionType {
@@ -42,25 +40,28 @@ message FilesystemVersion {
   string Creation = 5; // RFC 3339
 }
 
+enum Tri {
+  DontCare = 0;
+  False = 1;
+  True = 2;
+}
 
 message SendReq {
   string Filesystem = 1;
-    string From = 2;
-    // May be empty / null to request a full transfer of From
-    string To = 3;
+  // May be empty / null to request a full transfer of To
+  FilesystemVersion From = 2;
+  FilesystemVersion To = 3;
 
-    // If ResumeToken is not empty, the resume token that CAN be tried for 'zfs send' by the sender.
-    // The sender MUST indicate in SendRes.UsedResumeToken
-    // If it does not work, the sender SHOULD clear the resume token on their side
-    // and use From and To instead
-    // If ResumeToken is not empty, the GUIDs of From and To
-    // MUST correspond to those encoded in the ResumeToken.
-    // Otherwise, the Sender MUST return an error.
+  // If ResumeToken is not empty, the resume token that CAN be used for 'zfs
+  // send' by the sender. The sender MUST indicate use of ResumeToken in the
+  // reply message SendRes.UsedResumeToken If it does not work, the sender
+  // SHOULD clear the resume token on their side and use From and To instead If
+  // ResumeToken is not empty, the GUIDs of From and To MUST correspond to those
+  // encoded in the ResumeToken. Otherwise, the Sender MUST return an error.
   string ResumeToken = 4;
-    bool Compress = 5;
-    bool Dedup = 6;
+  Tri Encrypted = 5;
 
-    bool DryRun = 7;
+  bool DryRun = 6;
 }
 
 message Property {
@@ -70,6 +71,7 @@ message Property {
 
 message SendRes {
   // Whether the resume token provided in the request has been used or not.
+  // If the SendReq.ResumeToken == "", this field has no meaning.
   bool UsedResumeToken = 2;
 
   // Expected stream size determined by dry run, not exact.
@@ -79,11 +81,19 @@ message SendRes {
   repeated Property Properties = 4;
 }
 
-message ReceiveReq {
-    string Filesystem = 1; // FIXME should be snapshot name, we can enforce that on recv
+message SendCompletedReq {
+  SendReq OriginalReq = 2;
+}
 
-    // If true, the receiver should clear the resume token before perfoming the zfs recv of the stream in the request
-    bool ClearResumeToken = 2;
+message SendCompletedRes {}
+
+message ReceiveReq {
+  string Filesystem = 1;
+  FilesystemVersion To = 2;
+
+  // If true, the receiver should clear the resume token before perfoming the
+  // zfs recv of the stream in the request
+  bool ClearResumeToken = 3;
 }
 
 message ReceiveRes {}
@@ -99,21 +109,9 @@ message DestroySnapshotRes {
   string Error = 2;
 }
 
-message DestroySnapshotsRes {
-    repeated DestroySnapshotRes Results = 1;
-}
+message DestroySnapshotsRes { repeated DestroySnapshotRes Results = 1; }
 
-message ReplicationCursorReq {
-    string Filesystem = 1;
-    message GetOp {}
-    message SetOp {
-        string Snapshot = 2;
-    }
-    oneof op {
-        GetOp get = 2;
-        SetOp set = 3;
-    }
-}
+message ReplicationCursorReq { string Filesystem = 1; }
 
 message ReplicationCursorRes {
   oneof Result {
@@ -122,11 +120,15 @@ message ReplicationCursorRes {
   }
 }
 
-message PingReq {
-    string Message = 1;
-}
+message PingReq { string Message = 1; }
 
 message PingRes {
   // Echo must be PingReq.Message
   string Echo = 1;
 }
+
+message HintMostRecentCommonAncestorReq {
+    string Filesystem = 1;
+    FilesystemVersion SenderVersion = 2;
+}
+message HintMostRecentCommonAncestorRes {}

replication/logic/pdu/pdu_extras.go

@@ -7,6 +7,14 @@ import (
 	"github.com/zrepl/zrepl/zfs"
 )
 
+func (v *FilesystemVersion) GetRelName() string {
+	zv, err := v.ZFSFilesystemVersion()
+	if err != nil {
+		return ""
+	}
+	return zv.String()
+}
+
 func (v *FilesystemVersion) RelName() string {
 	zv, err := v.ZFSFilesystemVersion()
 	if err != nil {

replication/logic/replication_logic.go

@@ -30,6 +30,7 @@ type Endpoint interface {
 	ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
 	DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error)
 	WaitForConnectivity(ctx context.Context) error
+	HintMostRecentCommonAncestor(context.Context, *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error)
 }
 
 type Sender interface {
@@ -38,6 +39,7 @@ type Sender interface {
 	// any next call to the parent github.com/zrepl/zrepl/replication.Endpoint.
 	// If the send request is for dry run the io.ReadCloser will be nil
 	Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error)
+	SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error)
 	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
 }
 
@@ -48,9 +50,14 @@ type Receiver interface {
 	Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error)
 }
 
+type PlannerPolicy struct {
+	EncryptedSend tri // all sends must be encrypted (send -w, and encryption!=off)
+}
+
 type Planner struct {
 	sender   Sender
 	receiver Receiver
+	policy   PlannerPolicy
 
 	promSecsPerState    *prometheus.HistogramVec // labels: state
 	promBytesReplicated *prometheus.CounterVec   // labels: filesystem
@@ -109,9 +116,10 @@ func (p *Planner) WaitForConnectivity(ctx context.Context) error {
 type Filesystem struct {
 	sender   Sender
 	receiver Receiver
+	policy   PlannerPolicy
 
 	Path                 string             // compat
-	receiverFS          *pdu.Filesystem
+	receiverFS, senderFS *pdu.Filesystem    // receiverFS may be nil, senderFS never nil
 	promBytesReplicated  prometheus.Counter // compat
 
 	sizeEstimateRequestSem *semaphore.S
@@ -146,7 +154,9 @@ type Step struct {
 	receiver Receiver
 
 	parent      *Filesystem
-	from, to *pdu.FilesystemVersion // compat
+	from, to    *pdu.FilesystemVersion // from may be nil, indicating full send
+	encrypt     tri
+	resumeToken string // empty means no resume token shall be used
 
 	expectedSize int64 // 0 means no size estimate present / possible
 
@@ -186,23 +196,36 @@ func (s *Step) ReportInfo() *report.StepInfo {
 	}
 	s.byteCounterMtx.Unlock()
 
-	// FIXME stick to zfs convention of from and to
 	from := ""
 	if s.from != nil {
 		from = s.from.RelName()
 	}
+	var encrypted report.EncryptedEnum
+	switch s.encrypt {
+	case DontCare:
+		encrypted = report.EncryptedSenderDependent
+	case True:
+		encrypted = report.EncryptedTrue
+	case False:
+		encrypted = report.EncryptedFalse
+	default:
+		panic(fmt.Sprintf("unknown variant %s", s.encrypt))
+	}
 	return &report.StepInfo{
 		From:            from,
 		To:              s.to.RelName(),
+		Resumed:         s.resumeToken != "",
+		Encrypted:       encrypted,
 		BytesExpected:   s.expectedSize,
 		BytesReplicated: byteCounter,
 	}
 }
 
-func NewPlanner(secsPerState *prometheus.HistogramVec, bytesReplicated *prometheus.CounterVec, sender Sender, receiver Receiver) *Planner {
+func NewPlanner(secsPerState *prometheus.HistogramVec, bytesReplicated *prometheus.CounterVec, sender Sender, receiver Receiver, policy PlannerPolicy) *Planner {
 	return &Planner{
 		sender:              sender,
 		receiver:            receiver,
+		policy:              policy,
 		promSecsPerState:    secsPerState,
 		promBytesReplicated: bytesReplicated,
 	}
@@ -239,7 +262,6 @@ func (p *Planner) doPlanning(ctx context.Context) ([]*Filesystem, error) {
 		return nil, err
 	}
 	sfss := slfssres.GetFilesystems()
-	// no progress here since we could run in a live-lock on connectivity issues
 
 	rlfssres, err := p.receiver.ListFilesystems(ctx, &pdu.ListFilesystemReq{})
 	if err != nil {
@@ -265,7 +287,9 @@ func (p *Planner) doPlanning(ctx context.Context) ([]*Filesystem, error) {
 		q = append(q, &Filesystem{
 			sender:                 p.sender,
 			receiver:               p.receiver,
+			policy:                 p.policy,
 			Path:                   fs.Path,
+			senderFS:               fs,
 			receiverFS:             receiverFS,
 			promBytesReplicated:    ctr,
 			sizeEstimateRequestSem: sizeEstimateRequestSem,
@@ -281,6 +305,10 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 
 	log.Debug("assessing filesystem")
 
+	if fs.policy.EncryptedSend == True && !fs.senderFS.GetIsEncrypted() {
+		return nil, fmt.Errorf("sender filesystem is not encrypted but policy mandates encrypted send")
+	}
+
 	sfsvsres, err := fs.sender.ListFilesystemVersions(ctx, &pdu.ListFilesystemVersionsReq{Filesystem: fs.Path})
 	if err != nil {
 		log.WithError(err).Error("cannot get remote filesystem versions")
@@ -306,6 +334,150 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 		rfsvs = []*pdu.FilesystemVersion{}
 	}
 
+	var resumeToken *zfs.ResumeToken
+	var resumeTokenRaw string
+	if fs.receiverFS != nil && fs.receiverFS.ResumeToken != "" {
+		resumeTokenRaw = fs.receiverFS.ResumeToken // shadow
+		log.WithField("receiverFS.ResumeToken", resumeTokenRaw).Debug("decode receiver fs resume token")
+		resumeToken, err = zfs.ParseResumeToken(ctx, resumeTokenRaw) // shadow
+		if err != nil {
+			// TODO in theory, we could do replication without resume token, but that would mean that
+			// we need to discard the resumable state on the receiver's side.
+			// Would be easy by setting UsedResumeToken=false in the RecvReq ...
+			// FIXME / CHECK semantics UsedResumeToken if SendReq.ResumeToken == ""
+			log.WithError(err).Error("cannot decode resume token, aborting")
+			return nil, err
+		}
+		log.WithField("token", resumeToken).Debug("decode resume token")
+	}
+
+	// give both sides a hint about how far the replication got
+	// This serves as a cummulative variant of SendCompleted and can be useful
+	// for example to release stale holds from an earlier (interrupted) replication.
+	// TODO FIXME: enqueue this as a replication step instead of doing it here during planning
+	//             then again, the step should run regardless of planning success
+	//             so maybe a separate phase before PLANNING, then?
+	path, conflict := IncrementalPath(rfsvs, sfsvs)
+	var sender_mrca *pdu.FilesystemVersion // from sfsvs
+	if conflict == nil && len(path) > 0 {
+		sender_mrca = path[0] // shadow
+	}
+	if sender_mrca != nil {
+		var wg sync.WaitGroup
+		doHint := func(ep Endpoint, name string) {
+			defer wg.Done()
+			log := log.WithField("to_side", name).
+				WithField("sender_mrca", sender_mrca.String())
+			log.Debug("hint most recent common ancestor")
+			hint := &pdu.HintMostRecentCommonAncestorReq{
+				Filesystem:    fs.Path,
+				SenderVersion: sender_mrca,
+			}
+			_, err := ep.HintMostRecentCommonAncestor(ctx, hint)
+			if err != nil {
+				log.WithError(err).Error("error hinting most recent common ancestor")
+			}
+		}
+		wg.Add(2)
+		go doHint(fs.sender, "sender")
+		go doHint(fs.receiver, "receiver")
+		wg.Wait()
+	} else {
+		log.Debug("cannot identify most recent common ancestor, skipping hint")
+	}
+
+	var steps []*Step
+	// build the list of replication steps
+	//
+	// prefer to resume any started replication instead of starting over with a normal IncrementalPath
+	//
+	// look for the step encoded in the resume token in the sender's version
+	// if we find that step:
+	//   1. use it as first step (including resume token)
+	//   2. compute subsequent steps by computing incremental path from the token.To version on
+	//      ...
+	//      that's actually equivalent to simply cutting off earlier versions from rfsvs and sfsvs
+	if resumeToken != nil {
+
+		sfsvs := SortVersionListByCreateTXGThenBookmarkLTSnapshot(sfsvs)
+
+		var fromVersion, toVersion *pdu.FilesystemVersion
+		var toVersionIdx int
+		for idx, sfsv := range sfsvs {
+			if resumeToken.HasFromGUID && sfsv.Guid == resumeToken.FromGUID {
+				if fromVersion != nil && fromVersion.Type == pdu.FilesystemVersion_Snapshot {
+					// prefer snapshots over bookmarks for size estimation
+				} else {
+					fromVersion = sfsv
+				}
+			}
+			if resumeToken.HasToGUID && sfsv.Guid == resumeToken.ToGUID && sfsv.Type == pdu.FilesystemVersion_Snapshot {
+				// `toversion` must always be a snapshot
+				toVersion, toVersionIdx = sfsv, idx
+			}
+		}
+
+		encryptionMatches := false
+		switch fs.policy.EncryptedSend {
+		case True:
+			encryptionMatches = resumeToken.RawOK && resumeToken.CompressOK
+		case False:
+			encryptionMatches = !resumeToken.RawOK && !resumeToken.CompressOK
+		case DontCare:
+			encryptionMatches = true
+		}
+
+		log.WithField("fromVersion", fromVersion).
+			WithField("toVersion", toVersion).
+			WithField("encryptionMatches", encryptionMatches).
+			Debug("result of resume-token-matching to sender's versions")
+
+		if !encryptionMatches {
+			return nil, fmt.Errorf("resume token `rawok`=%v and `compressok`=%v are incompatible with encryption policy=%v", resumeToken.RawOK, resumeToken.CompressOK, fs.policy.EncryptedSend)
+		} else if toVersion == nil {
+			return nil, fmt.Errorf("resume token `toguid` = %v not found on sender (`toname` = %q)", resumeToken.ToGUID, resumeToken.ToName)
+		} else if fromVersion == toVersion {
+			return nil, fmt.Errorf("resume token `fromguid` and `toguid` match same version on sener")
+		}
+		// fromVersion may be nil, toVersion is no nil, encryption matches
+		// good to go this one step!
+		resumeStep := &Step{
+			parent:   fs,
+			sender:   fs.sender,
+			receiver: fs.receiver,
+
+			from:    fromVersion,
+			to:      toVersion,
+			encrypt: fs.policy.EncryptedSend,
+
+			resumeToken: resumeTokenRaw,
+		}
+
+		// by definition, the resume token _must_ be the receiver's most recent version, if they have any
+		// don't bother checking, zfs recv will produce an error if above assumption is wrong
+		//
+		// thus, subsequent steps are just incrementals on the sender's remaining _snapshots_ (not bookmarks)
+
+		var remainingSFSVs []*pdu.FilesystemVersion
+		for _, sfsv := range sfsvs[toVersionIdx:] {
+			if sfsv.Type == pdu.FilesystemVersion_Snapshot {
+				remainingSFSVs = append(remainingSFSVs, sfsv)
+			}
+		}
+
+		steps = make([]*Step, 0, len(remainingSFSVs)) // shadow
+		steps = append(steps, resumeStep)
+		for i := 0; i < len(remainingSFSVs)-1; i++ {
+			steps = append(steps, &Step{
+				parent:   fs,
+				sender:   fs.sender,
+				receiver: fs.receiver,
+				from:     remainingSFSVs[i],
+				to:       remainingSFSVs[i+1],
+				encrypt:  fs.policy.EncryptedSend,
+			})
+		}
+	} else { // resumeToken == nil
 		path, conflict := IncrementalPath(rfsvs, sfsvs)
 		if conflict != nil {
 			var msg string
@@ -322,15 +494,16 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 			return nil, conflict
 		}
 
-	steps := make([]*Step, 0, len(path))
-	// FIXME unify struct declarations => initializer?
+		steps = make([]*Step, 0, len(path)) // shadow
 		if len(path) == 1 {
 			steps = append(steps, &Step{
 				parent:   fs,
 				sender:   fs.sender,
 				receiver: fs.receiver,
+
 				from:    nil,
 				to:      path[0],
+				encrypt: fs.policy.EncryptedSend,
 			})
 		} else {
 			for i := 0; i < len(path)-1; i++ {
@@ -338,11 +511,18 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 					parent:   fs,
 					sender:   fs.sender,
 					receiver: fs.receiver,
+
 					from:    path[i],
 					to:      path[i+1],
+					encrypt: fs.policy.EncryptedSend,
 				})
 			}
 		}
+	}
+
+	if len(steps) == 0 {
+		log.Info("planning determined that no replication steps are required")
+	}
 
 	log.Debug("compute send size estimate")
 	errs := make(chan error, len(steps))
@@ -389,29 +569,6 @@ func (fs *Filesystem) doPlanning(ctx context.Context) ([]*Step, error) {
 	return steps, nil
 }
 
-// type FilesystemsReplicationFailedError struct {
-// 	FilesystemsWithError []*fsrep.Replication
-// }
-
-// func (e FilesystemsReplicationFailedError) Error() string {
-// 	allSame := true
-// 	lastErr := e.FilesystemsWithError[0].Err().Error()
-// 	for _, fs := range e.FilesystemsWithError {
-// 		fsErr := fs.Err().Error()
-// 		allSame = allSame && lastErr == fsErr
-// 	}
-
-// 	fsstr := "multiple filesystems"
-// 	if len(e.FilesystemsWithError) == 1 {
-// 		fsstr = fmt.Sprintf("filesystem %s", e.FilesystemsWithError[0].FS())
-// 	}
-// 	errorStr := lastErr
-// 	if !allSame {
-// 		errorStr = "multiple different errors"
-// 	}
-// 	return fmt.Sprintf("%s could not be replicated: %s", fsstr, errorStr)
-// }
-
 func (s *Step) updateSizeEstimate(ctx context.Context) error {
 
 	log := getLogger(ctx)
@@ -430,20 +587,14 @@ func (s *Step) updateSizeEstimate(ctx context.Context) error {
 
 func (s *Step) buildSendRequest(dryRun bool) (sr *pdu.SendReq) {
 	fs := s.parent.Path
-	if s.from == nil {
 	sr = &pdu.SendReq{
 		Filesystem:  fs,
-			To:         s.to.RelName(),
+		From:        s.from, // may be nil
+		To:          s.to,
+		Encrypted:   s.encrypt.ToPDU(),
+		ResumeToken: s.resumeToken,
 		DryRun:      dryRun,
 	}
-	} else {
-		sr = &pdu.SendReq{
-			Filesystem: fs,
-			From:       s.from.RelName(),
-			To:         s.to.RelName(),
-			DryRun:     dryRun,
-		}
-	}
 	return sr
 }
 
@@ -478,6 +629,7 @@ func (s *Step) doReplication(ctx context.Context) error {
 
 	rr := &pdu.ReceiveReq{
 		Filesystem:       fs,
+		To:               sr.GetTo(),
 		ClearResumeToken: !sres.UsedResumeToken,
 	}
 	log.Debug("initiate receive request")
@@ -486,6 +638,7 @@ func (s *Step) doReplication(ctx context.Context) error {
 		log.
 			WithError(err).
 			WithField("errType", fmt.Sprintf("%T", err)).
+			WithField("rr", fmt.Sprintf("%v", rr)).
 			Error("receive request failed (might also be error on sender)")
 		// This failure could be due to
 		// 	- an unexpected exit of ZFS on the sending side
@@ -495,22 +648,12 @@ func (s *Step) doReplication(ctx context.Context) error {
 	}
 	log.Debug("receive finished")
 
-	log.Debug("advance replication cursor")
-	req := &pdu.ReplicationCursorReq{
-		Filesystem: fs,
-		Op: &pdu.ReplicationCursorReq_Set{
-			Set: &pdu.ReplicationCursorReq_SetOp{
-				Snapshot: s.to.GetName(),
-			},
-		},
-	}
-	_, err = s.sender.ReplicationCursor(ctx, req)
+	log.Debug("tell sender replication completed")
+	_, err = s.sender.SendCompleted(ctx, &pdu.SendCompletedReq{
+		OriginalReq: sr,
+	})
 	if err != nil {
-		log.WithError(err).Error("error advancing replication cursor")
-		// If this fails and replication planning restarts, the diff algorithm will find
-		// that cursor out of place. This is not a problem because then, it would just use another FS
-		// However, we FIXME have no means to just update the cursor in a
-		// second replication attempt right after this one where we don't have new snaps yet
+		log.WithError(err).Error("error telling sender that replication completed successfully")
 		return err
 	}
 

replication/logic/replication_logic_tri.go

@@ -0,0 +1,46 @@
+package logic
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/replication/logic/pdu"
+)
+
+type tri int
+
+const (
+	DontCare = 0x0
+	False    = 0x1
+	True     = 0x2
+)
+
+func (t tri) String() string {
+	switch t {
+	case DontCare:
+		return "dontcare"
+	case False:
+		return "false"
+	case True:
+		return "true"
+	}
+	panic(fmt.Sprintf("unknown variant %v", int(t)))
+}
+
+func (t tri) ToPDU() pdu.Tri {
+	switch t {
+	case DontCare:
+		return pdu.Tri_DontCare
+	case False:
+		return pdu.Tri_False
+	case True:
+		return pdu.Tri_True
+	}
+	panic(fmt.Sprintf("unknown variant %v", int(t)))
+}
+
+func TriFromBool(b bool) tri {
+	if b {
+		return True
+	}
+	return False
+}

replication/report/replication_report.go

@@ -85,25 +85,37 @@ type StepReport struct {
 	Info *StepInfo
 }
 
+type EncryptedEnum string
+
+const (
+	EncryptedTrue            EncryptedEnum = "yes"
+	EncryptedFalse           EncryptedEnum = "no"
+	EncryptedSenderDependent EncryptedEnum = "sender-dependent"
+)
+
 type StepInfo struct {
 	From, To        string
+	Resumed         bool
+	Encrypted       EncryptedEnum
 	BytesExpected   int64
 	BytesReplicated int64
 }
 
-func (a *AttemptReport) BytesSum() (expected, replicated int64) {
+func (a *AttemptReport) BytesSum() (expected, replicated int64, containsInvalidSizeEstimates bool) {
 	for _, fs := range a.Filesystems {
-		e, r := fs.BytesSum()
+		e, r, fsContainsInvalidEstimate := fs.BytesSum()
+		containsInvalidSizeEstimates = containsInvalidSizeEstimates || fsContainsInvalidEstimate
 		expected += e
 		replicated += r
 	}
-	return expected, replicated
+	return expected, replicated, containsInvalidSizeEstimates
 }
 
-func (f *FilesystemReport) BytesSum() (expected, replicated int64) {
+func (f *FilesystemReport) BytesSum() (expected, replicated int64, containsInvalidSizeEstimates bool) {
 	for _, step := range f.Steps {
 		expected += step.Info.BytesExpected
 		replicated += step.Info.BytesReplicated
+		containsInvalidSizeEstimates = containsInvalidSizeEstimates || step.Info.BytesExpected == 0
 	}
 	return
 }
@@ -148,5 +160,5 @@ func (f *FilesystemReport) NextStep() *StepReport {
 }
 
 func (f *StepReport) IsIncremental() bool {
-	return f.Info.From != "" // FIXME change to ZFS semantics (To != "")
+	return f.Info.From != ""
 }

rpc/dataconn/timeoutconn/timeoutconn.go

@@ -14,7 +14,6 @@ import (
 	"sync/atomic"
 	"syscall"
 	"time"
-	"unsafe"
 )
 
 type Wire interface {
@@ -50,7 +49,7 @@ type Wire interface {
 	// No data that could otherwise be Read is lost as a consequence of this call.
 	// The use case for this API is abortive connection shutdown.
 	// To provide any value over draining Wire using io.Read, an implementation
-	// will likely use out-of-bounds messaging mechanisms.
+	// will likely use out-of-band messaging mechanisms.
 	// TODO WaitForPeerClose() (supported bool, err error)
 }
 
@@ -155,25 +154,6 @@ type SyscallConner interface {
 
 var _ SyscallConner = (*net.TCPConn)(nil)
 
-func buildIovecs(buffers net.Buffers) (totalLen int64, vecs []syscall.Iovec) {
-	vecs = make([]syscall.Iovec, 0, len(buffers))
-	for i := range buffers {
-		totalLen += int64(len(buffers[i]))
-		if len(buffers[i]) == 0 {
-			continue
-		}
-
-		v := syscall.Iovec{
-			Base: &buffers[i][0],
-		}
-		// syscall.Iovec.Len has platform-dependent size, thus use SetLen
-		v.SetLen(len(buffers[i]))
-
-		vecs = append(vecs, v)
-	}
-	return totalLen, vecs
-}
-
 // Reads the given buffers full:
 // Think of io.ReadvFull, but for net.Buffers + using the readv syscall.
 //
@@ -184,25 +164,10 @@ func buildIovecs(buffers net.Buffers) (totalLen int64, vecs []syscall.Iovec) {
 // then + io.EOF is returned. This behavior is different to io.ReadFull
 // which returns io.ErrUnexpectedEOF.
 func (c Conn) ReadvFull(buffers net.Buffers) (n int64, err error) {
-	totalLen, iovecs := buildIovecs(buffers)
-	if debugReadvNoShortReadsAssertEnable {
-		defer debugReadvNoShortReadsAssert(totalLen, n, err)
-	}
-	scc, ok := c.Wire.(SyscallConner)
-	if !ok {
-		return c.readvFallback(buffers)
-	}
-	raw, err := scc.SyscallConn()
-	if err == SyscallConnNotSupported {
-		return c.readvFallback(buffers)
-	}
-	if err != nil {
-		return 0, err
-	}
-	n, err = c.readv(raw, iovecs)
-	return
+	return c.readv(buffers)
 }
 
+// invoked by c.readv if readv system call cannot be used
 func (c Conn) readvFallback(nbuffers net.Buffers) (n int64, err error) {
 	buffers := [][]byte(nbuffers)
 	for i := range buffers {
@@ -226,91 +191,3 @@ func (c Conn) readvFallback(nbuffers net.Buffers) (n int64, err error) {
 	}
 	return n, nil
 }
-
-func (c Conn) readv(rawConn syscall.RawConn, iovecs []syscall.Iovec) (n int64, err error) {
-	for len(iovecs) > 0 {
-		if err := c.renewReadDeadline(); err != nil {
-			return n, err
-		}
-		oneN, oneErr := c.doOneReadv(rawConn, &iovecs)
-		n += oneN
-		if netErr, ok := oneErr.(net.Error); ok && netErr.Timeout() && oneN > 0 { // TODO likely not working
-			continue
-		} else if oneErr == nil && oneN > 0 {
-			continue
-		} else {
-			return n, oneErr
-		}
-	}
-	return n, nil
-}
-
-func (c Conn) doOneReadv(rawConn syscall.RawConn, iovecs *[]syscall.Iovec) (n int64, err error) {
-	rawReadErr := rawConn.Read(func(fd uintptr) (done bool) {
-		// iovecs, n and err must not be shadowed!
-
-		// NOTE: unsafe.Pointer safety rules
-		// 		https://tip.golang.org/pkg/unsafe/#Pointer
-		//
-		//		(4) Conversion of a Pointer to a uintptr when calling syscall.Syscall.
-		// 		...
-		//		uintptr() conversions must appear within the syscall.Syscall argument list.
-		//      (even though we are not the escape analysis Likely not )
-		thisReadN, _, errno := syscall.Syscall(
-			syscall.SYS_READV,
-			fd,
-			uintptr(unsafe.Pointer(&(*iovecs)[0])),
-			uintptr(len(*iovecs)),
-		)
-		if thisReadN == ^uintptr(0) {
-			if errno == syscall.EAGAIN {
-				return false
-			}
-			err = syscall.Errno(errno)
-			return true
-		}
-		if int(thisReadN) < 0 {
-			panic("unexpected return value")
-		}
-		n += int64(thisReadN) // TODO check overflow
-
-		// shift iovecs forward
-		for left := int(thisReadN); left > 0; {
-			// conversion to uint does not change value, see TestIovecLenFieldIsMachineUint, and left > 0
-			thisIovecConsumedCompletely := uint((*iovecs)[0].Len) <= uint(left)
-			if thisIovecConsumedCompletely {
-				// Update left, cannot go below 0 due to
-				// a) definition of thisIovecConsumedCompletely
-				// b) left > 0 due to loop invariant
-				// Convertion .Len to int64 is thus also safe now, because it is < left < INT_MAX
-				left -= int((*iovecs)[0].Len)
-				*iovecs = (*iovecs)[1:]
-			} else {
-				// trim this iovec to remaining length
-
-				// NOTE: unsafe.Pointer safety rules
-				// 		https://tip.golang.org/pkg/unsafe/#Pointer
-				// 		(3) Conversion of a Pointer to a uintptr and back, with arithmetic.
-				// 		...
-				//		Note that both conversions must appear in the same expression,
-				//		with only the intervening arithmetic between them:
-				(*iovecs)[0].Base = (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer((*iovecs)[0].Base)) + uintptr(left)))
-				curVecNewLength := uint((*iovecs)[0].Len) - uint(left) // casts to uint do not change value
-				(*iovecs)[0].SetLen(int(curVecNewLength))              // int and uint have the same size, no change of value
-
-				break // inner
-			}
-		}
-		if thisReadN == 0 {
-			err = io.EOF
-			return true
-		}
-		return true
-	})
-
-	if rawReadErr != nil {
-		err = rawReadErr
-	}
-
-	return n, err
-}

rpc/dataconn/timeoutconn/timeoutconn_debug.go

@@ -1,19 +0,0 @@
-package timeoutconn
-
-import (
-	"fmt"
-	"io"
-)
-
-const debugReadvNoShortReadsAssertEnable = false
-
-func debugReadvNoShortReadsAssert(expectedLen, returnedLen int64, returnedErr error) {
-	readShort := expectedLen != returnedLen
-	if !readShort {
-		return
-	}
-	if returnedErr != io.EOF {
-		return
-	}
-	panic(fmt.Sprintf("ReadvFull short and error is not EOF%v\n", returnedErr))
-}

rpc/dataconn/timeoutconn/timeoutconn_readv_unsupported.go

@@ -0,0 +1,11 @@
+// +build illumos solaris
+
+package timeoutconn
+
+import "net"
+
+func (c Conn) readv(buffers net.Buffers) (n int64, err error) {
+	// Go does not expose the SYS_READV symbol for Solaris / Illumos - do they have it?
+	// Anyhow, use the fallback
+	return c.readvFallback(buffers)
+}

rpc/dataconn/timeoutconn/timoutconn_readv.go

@@ -0,0 +1,133 @@
+// +build !illumos
+// +build !solaris
+
+package timeoutconn
+
+import (
+	"io"
+	"net"
+	"syscall"
+	"unsafe"
+)
+
+func buildIovecs(buffers net.Buffers) (totalLen int64, vecs []syscall.Iovec) {
+	vecs = make([]syscall.Iovec, 0, len(buffers))
+	for i := range buffers {
+		totalLen += int64(len(buffers[i]))
+		if len(buffers[i]) == 0 {
+			continue
+		}
+
+		v := syscall.Iovec{
+			Base: &buffers[i][0],
+		}
+		// syscall.Iovec.Len has platform-dependent size, thus use SetLen
+		v.SetLen(len(buffers[i]))
+
+		vecs = append(vecs, v)
+	}
+	return totalLen, vecs
+}
+
+func (c Conn) readv(buffers net.Buffers) (n int64, err error) {
+
+	scc, ok := c.Wire.(SyscallConner)
+	if !ok {
+		return c.readvFallback(buffers)
+	}
+	rawConn, err := scc.SyscallConn()
+	if err == SyscallConnNotSupported {
+		return c.readvFallback(buffers)
+	}
+	if err != nil {
+		return 0, err
+	}
+
+	_, iovecs := buildIovecs(buffers)
+
+	for len(iovecs) > 0 {
+		if err := c.renewReadDeadline(); err != nil {
+			return n, err
+		}
+		oneN, oneErr := c.doOneReadv(rawConn, &iovecs)
+		n += oneN
+		if netErr, ok := oneErr.(net.Error); ok && netErr.Timeout() && oneN > 0 { // TODO likely not working
+			continue
+		} else if oneErr == nil && oneN > 0 {
+			continue
+		} else {
+			return n, oneErr
+		}
+	}
+	return n, nil
+}
+
+func (c Conn) doOneReadv(rawConn syscall.RawConn, iovecs *[]syscall.Iovec) (n int64, err error) {
+	rawReadErr := rawConn.Read(func(fd uintptr) (done bool) {
+		// iovecs, n and err must not be shadowed!
+
+		// NOTE: unsafe.Pointer safety rules
+		// 		https://tip.golang.org/pkg/unsafe/#Pointer
+		//
+		//		(4) Conversion of a Pointer to a uintptr when calling syscall.Syscall.
+		// 		...
+		//		uintptr() conversions must appear within the syscall.Syscall argument list.
+		//      (even though we are not the escape analysis Likely not )
+		thisReadN, _, errno := syscall.Syscall(
+			syscall.SYS_READV,
+			fd,
+			uintptr(unsafe.Pointer(&(*iovecs)[0])),
+			uintptr(len(*iovecs)),
+		)
+		if thisReadN == ^uintptr(0) {
+			if errno == syscall.EAGAIN {
+				return false
+			}
+			err = syscall.Errno(errno)
+			return true
+		}
+		if int(thisReadN) < 0 {
+			panic("unexpected return value")
+		}
+		n += int64(thisReadN) // TODO check overflow
+
+		// shift iovecs forward
+		for left := int(thisReadN); left > 0; {
+			// conversion to uint does not change value, see TestIovecLenFieldIsMachineUint, and left > 0
+			thisIovecConsumedCompletely := uint((*iovecs)[0].Len) <= uint(left)
+			if thisIovecConsumedCompletely {
+				// Update left, cannot go below 0 due to
+				// a) definition of thisIovecConsumedCompletely
+				// b) left > 0 due to loop invariant
+				// Convertion .Len to int64 is thus also safe now, because it is < left < INT_MAX
+				left -= int((*iovecs)[0].Len)
+				*iovecs = (*iovecs)[1:]
+			} else {
+				// trim this iovec to remaining length
+
+				// NOTE: unsafe.Pointer safety rules
+				// 		https://tip.golang.org/pkg/unsafe/#Pointer
+				// 		(3) Conversion of a Pointer to a uintptr and back, with arithmetic.
+				// 		...
+				//		Note that both conversions must appear in the same expression,
+				//		with only the intervening arithmetic between them:
+				(*iovecs)[0].Base = (*byte)(unsafe.Pointer(uintptr(unsafe.Pointer((*iovecs)[0].Base)) + uintptr(left)))
+				curVecNewLength := uint((*iovecs)[0].Len) - uint(left) // casts to uint do not change value
+				(*iovecs)[0].SetLen(int(curVecNewLength))              // int and uint have the same size, no change of value
+
+				break // inner
+			}
+		}
+		if thisReadN == 0 {
+			err = io.EOF
+			return true
+		}
+		return true
+	})
+
+	if rawReadErr != nil {
+		err = rawReadErr
+	}
+
+	return n, err
+}

rpc/grpcclientidentity/authlistener_grpc_adaptor.go

@@ -99,7 +99,9 @@ func (*transportCredentials) OverrideServerName(string) error {
 	panic("not implemented")
 }
 
-func NewInterceptors(logger Logger, clientIdentityKey interface{}) (unary grpc.UnaryServerInterceptor, stream grpc.StreamServerInterceptor) {
+type ContextInterceptor = func(ctx context.Context) context.Context
+
+func NewInterceptors(logger Logger, clientIdentityKey interface{}, ctxInterceptor ContextInterceptor) (unary grpc.UnaryServerInterceptor, stream grpc.StreamServerInterceptor) {
 	unary = func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) {
 		logger.WithField("fullMethod", info.FullMethod).Debug("request")
 		p, ok := peer.FromContext(ctx)
@@ -113,8 +115,13 @@ func NewInterceptors(logger Logger, clientIdentityKey interface{}) (unary grpc.U
 		}
 		logger.WithField("peer_client_identity", a.clientIdentity).Debug("peer client identity")
 		ctx = context.WithValue(ctx, clientIdentityKey, a.clientIdentity)
+		if ctxInterceptor != nil {
+			ctx = ctxInterceptor(ctx)
+		}
 		return handler(ctx, req)
 	}
-	stream = nil
+	stream = func(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		panic("unimplemented")
+	}
 	return
 }

rpc/grpcclientidentity/example/main.go

@@ -87,10 +87,13 @@ func server() {
 
 	log := logger.NewStderrDebugLogger()
 
-	srv, serve, err := grpchelper.NewServer(authListenerFactory, clientIdentityKey, log)
+	authListener, err := authListenerFactory()
 	if err != nil {
-		onErr(err, "new server")
+		onErr(err, "cannot listen")
 	}
+
+	srv, serve := grpchelper.NewServer(authListener, clientIdentityKey, log, nil)
+
 	svc := &greeter{"hello "}
 	pdu.RegisterGreeterServer(srv, svc)
 

rpc/grpcclientidentity/grpchelper/authlistener_grpc_adaptor_wrapper.go

@@ -50,25 +50,25 @@ func ClientConn(cn transport.Connecter, log Logger) *grpc.ClientConn {
 }
 
 // NewServer is a convenience interface around the TransportCredentials and Interceptors interface.
-func NewServer(authListenerFactory transport.AuthenticatedListenerFactory, clientIdentityKey interface{}, logger grpcclientidentity.Logger) (srv *grpc.Server, serve func() error, err error) {
+func NewServer(authListener transport.AuthenticatedListener, clientIdentityKey interface{}, logger grpcclientidentity.Logger, ctxInterceptor grpcclientidentity.ContextInterceptor) (srv *grpc.Server, serve func() error) {
 	ka := grpc.KeepaliveParams(keepalive.ServerParameters{
 		Time:    StartKeepalivesAfterInactivityDuration,
 		Timeout: KeepalivePeerTimeout,
 	})
+	ep := grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+		MinTime:             StartKeepalivesAfterInactivityDuration / 2, // avoid skew
+		PermitWithoutStream: true,
+	})
 	tcs := grpcclientidentity.NewTransportCredentials(logger)
-	unary, stream := grpcclientidentity.NewInterceptors(logger, clientIdentityKey)
-	srv = grpc.NewServer(grpc.Creds(tcs), grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream), ka)
+	unary, stream := grpcclientidentity.NewInterceptors(logger, clientIdentityKey, ctxInterceptor)
+	srv = grpc.NewServer(grpc.Creds(tcs), grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream), ka, ep)
 
 	serve = func() error {
-		l, err := authListenerFactory()
-		if err != nil {
-			return err
-		}
-		if err := srv.Serve(netadaptor.New(l, logger)); err != nil {
+		if err := srv.Serve(netadaptor.New(authListener, logger)); err != nil {
 			return err
 		}
 		return nil
 	}
 
-	return srv, serve, nil
+	return srv, serve
 }

rpc/rpc_client.go

@@ -116,6 +116,14 @@ func (c *Client) ReplicationCursor(ctx context.Context, in *pdu.ReplicationCurso
 	return c.controlClient.ReplicationCursor(ctx, in)
 }
 
+func (c *Client) SendCompleted(ctx context.Context, in *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	return c.controlClient.SendCompleted(ctx, in)
+}
+
+func (c *Client) HintMostRecentCommonAncestor(ctx context.Context, in *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	return c.controlClient.HintMostRecentCommonAncestor(ctx, in)
+}
+
 func (c *Client) WaitForConnectivity(ctx context.Context) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()

rpc/rpc_server.go

@@ -4,13 +4,10 @@ import (
 	"context"
 	"time"
 
-	"google.golang.org/grpc"
-
 	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/replication/logic/pdu"
 	"github.com/zrepl/zrepl/rpc/dataconn"
-	"github.com/zrepl/zrepl/rpc/grpcclientidentity"
-	"github.com/zrepl/zrepl/rpc/netadaptor"
+	"github.com/zrepl/zrepl/rpc/grpcclientidentity/grpchelper"
 	"github.com/zrepl/zrepl/rpc/versionhandshake"
 	"github.com/zrepl/zrepl/transport"
 	"github.com/zrepl/zrepl/util/envconst"
@@ -28,7 +25,6 @@ type serveFunc func(ctx context.Context, demuxedListener transport.Authenticated
 type Server struct {
 	logger             Logger
 	handler            Handler
-	controlServer      *grpc.Server
 	controlServerServe serveFunc
 	dataServer         *dataconn.Server
 	dataServerServe    serveFunc
@@ -40,11 +36,11 @@ type HandlerContextInterceptor func(ctx context.Context) context.Context
 func NewServer(handler Handler, loggers Loggers, ctxInterceptor HandlerContextInterceptor) *Server {
 
 	// setup control server
-	tcs := grpcclientidentity.NewTransportCredentials(loggers.Control) // TODO different subsystem for log
-	unary, stream := grpcclientidentity.NewInterceptors(loggers.Control, endpoint.ClientIdentityKey)
-	controlServer := grpc.NewServer(grpc.Creds(tcs), grpc.UnaryInterceptor(unary), grpc.StreamInterceptor(stream))
-	pdu.RegisterReplicationServer(controlServer, handler)
 	controlServerServe := func(ctx context.Context, controlListener transport.AuthenticatedListener, errOut chan<- error) {
+
+		controlServer, serve := grpchelper.NewServer(controlListener, endpoint.ClientIdentityKey, loggers.Control, ctxInterceptor)
+		pdu.RegisterReplicationServer(controlServer, handler)
+
 		// give time for graceful stop until deadline expires, then hard stop
 		go func() {
 			<-ctx.Done()
@@ -55,7 +51,7 @@ func NewServer(handler Handler, loggers Loggers, ctxInterceptor HandlerContextIn
 			controlServer.GracefulStop()
 		}()
 
-		errOut <- controlServer.Serve(netadaptor.New(controlListener, loggers.Control))
+		errOut <- serve()
 	}
 
 	// setup data server
@@ -76,7 +72,6 @@ func NewServer(handler Handler, loggers Loggers, ctxInterceptor HandlerContextIn
 	server := &Server{
 		logger:             loggers.General,
 		handler:            handler,
-		controlServer:      controlServer,
 		controlServerServe: controlServerServe,
 		dataServer:         dataServer,
 		dataServerServe:    dataServerServe,

rpc/versionhandshake/versionhandshake.go

@@ -152,7 +152,7 @@ func (m *HandshakeMessage) DecodeReader(r io.Reader, maxLen int) error {
 
 func DoHandshakeCurrentVersion(conn net.Conn, deadline time.Time) *HandshakeError {
 	// current protocol version is hardcoded here
-	return DoHandshakeVersion(conn, deadline, 1)
+	return DoHandshakeVersion(conn, deadline, 2)
 }
 
 const HandshakeMessageMaxLen = 16 * 4096

transport/tcp/serve_tcp.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/util/tcpsock"
 )
 
 type ipMapEntry struct {
@@ -44,16 +45,12 @@ func (m *ipMap) Get(ip net.IP) (string, error) {
 }
 
 func TCPListenerFactoryFromConfig(c *config.Global, in *config.TCPServe) (transport.AuthenticatedListenerFactory, error) {
-	addr, err := net.ResolveTCPAddr("tcp", in.Listen)
-	if err != nil {
-		return nil, errors.Wrap(err, "cannot parse listen address")
-	}
 	clientMap, err := ipMapFromConfig(in.Clients)
 	if err != nil {
 		return nil, errors.Wrap(err, "cannot parse client IP map")
 	}
 	lf := func() (transport.AuthenticatedListener, error) {
-		l, err := net.ListenTCP("tcp", addr)
+		l, err := tcpsock.Listen(in.Listen, in.ListenFreeBind)
 		if err != nil {
 			return nil, err
 		}

transport/tls/serve_tls.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
-	"net"
 	"time"
 
 	"github.com/pkg/errors"
@@ -12,6 +11,7 @@ import (
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/tlsconf"
 	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/util/tcpsock"
 )
 
 type TLSListenerFactory struct{}
@@ -32,7 +32,7 @@ func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (transp
 
 	serverCert, err := tls.LoadX509KeyPair(in.Cert, in.Key)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannot parse cer/key pair")
+		return nil, errors.Wrap(err, "cannot parse cert/key pair")
 	}
 
 	clientCNs := make(map[string]struct{}, len(in.ClientCNs))
@@ -45,12 +45,11 @@ func TLSListenerFactoryFromConfig(c *config.Global, in *config.TLSServe) (transp
 	}
 
 	lf := func() (transport.AuthenticatedListener, error) {
-		l, err := net.Listen("tcp", address)
+		l, err := tcpsock.Listen(address, in.ListenFreeBind)
 		if err != nil {
 			return nil, err
 		}
-		tcpL := l.(*net.TCPListener)
-		tl := tlsconf.NewClientAuthListener(tcpL, clientCA, serverCert, handshakeTimeout)
+		tl := tlsconf.NewClientAuthListener(l, clientCA, serverCert, handshakeTimeout)
 		return &tlsAuthListener{tl, clientCNs}, nil
 	}
 

util/circlog/circlog.go

@@ -27,6 +27,14 @@ type CircularLog struct {
 	mtx sync.Mutex
 }
 
+func MustNewCircularLog(max int) *CircularLog {
+	log, err := NewCircularLog(max)
+	if err != nil {
+		panic(err)
+	}
+	return log
+}
+
 func NewCircularLog(max int) (*CircularLog, error) {
 	if max <= 0 {
 		return nil, fmt.Errorf("max must be positive")

util/limitio/limitio.go

@@ -0,0 +1,36 @@
+package limitio
+
+import "io"
+
+type readCloser struct {
+	read  int64
+	limit int64
+	r     io.ReadCloser
+}
+
+func ReadCloser(rc io.ReadCloser, limit int64) io.ReadCloser {
+	return &readCloser{0, limit, rc}
+}
+
+var _ io.ReadCloser = (*readCloser)(nil)
+
+func (r *readCloser) Read(b []byte) (int, error) {
+
+	if len(b) == 0 {
+		return 0, nil
+	}
+
+	if r.read == r.limit {
+		return 0, io.EOF
+	}
+
+	if r.read+int64(len(b)) >= r.limit {
+		b = b[:int(r.limit-r.read)]
+	}
+
+	readN, err := r.r.Read(b)
+	r.read += int64(readN)
+	return readN, err
+}
+
+func (r *readCloser) Close() error { return r.r.Close() }

util/limitio/limitio_readcloser_test.go

@@ -0,0 +1,44 @@
+package limitio
+
+import (
+	"bytes"
+	"io"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+type mockRC struct {
+	r      io.Reader
+	closed bool
+}
+
+func newMockRC(r io.Reader) *mockRC { return &mockRC{r, false} }
+
+func (m mockRC) Read(b []byte) (int, error) {
+	if m.closed {
+		return 0, io.ErrClosedPipe
+	}
+	return m.r.Read(b)
+}
+
+func (m *mockRC) Close() error {
+	m.closed = true
+	return nil
+}
+
+func TestReadCloser(t *testing.T) {
+	foobarReader := bytes.NewReader([]byte("foobar2342"))
+	mock := newMockRC(foobarReader)
+	limited := ReadCloser(mock, 6)
+	var buf [20]byte
+	n, err := limited.Read(buf[:])
+	assert.NoError(t, err)
+	assert.Equal(t, 6, n)
+	assert.Equal(t, buf[:n], []byte("foobar"))
+	n, err = limited.Read(buf[:])
+	assert.Equal(t, 0, n)
+	assert.Equal(t, io.EOF, err)
+	limited.Close()
+	assert.True(t, mock.closed)
+}

util/tcpsock/tcpsock.go

@@ -0,0 +1,27 @@
+package tcpsock
+
+import (
+	"context"
+	"net"
+	"syscall"
+)
+
+func Listen(address string, tryFreeBind bool) (*net.TCPListener, error) {
+	control := func(network, address string, c syscall.RawConn) error {
+		if tryFreeBind {
+			if err := freeBind(network, address, c); err != nil {
+				return err
+			}
+		}
+		return nil
+	}
+	var listenConfig = net.ListenConfig{
+		Control: control,
+	}
+
+	l, err := listenConfig.Listen(context.Background(), "tcp", address)
+	if err != nil {
+		return nil, err
+	}
+	return l.(*net.TCPListener), nil
+}

util/tcpsock/tcpsock_freebind_freebsd.go

@@ -0,0 +1,25 @@
+// +build freebsd
+
+package tcpsock
+
+import (
+	"fmt"
+	"syscall"
+)
+
+func freeBind(network, address string, c syscall.RawConn) error {
+	var err, sockerr error
+	err = c.Control(func(fd uintptr) {
+		if network == "tcp6" {
+			sockerr = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_IPV6, syscall.IPV6_BINDANY, 1)
+		} else if network == "tcp4" {
+			sockerr = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_IP, syscall.IP_BINDANY, 1)
+		} else {
+			sockerr = fmt.Errorf("expecting 'tcp6' or 'tcp4', got %q", network)
+		}
+	})
+	if err != nil {
+		return err
+	}
+	return sockerr
+}