transport/ssh: bump go-netssh version to fix ssh client process leaks

fixes #322

.circleci/config.yml

@@ -6,8 +6,11 @@ workflows:
       - build-1.11
       - build-1.12
       - build-1.13
+      - build-1.14
       - build-latest
+      - test-build-in-docker
 jobs:
+
   # build-latest serves as the template
   # we use YAML anchors & aliases to exchange the docker image (and hence Go version used for the build)
   build-latest: &build-latest
@@ -52,10 +55,10 @@ jobs:
           paths:
             - "/usr/include/google/protobuf"
             
-      - run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev
+      - run: sudo apt update && sudo apt install python3 python3-pip libgirepository1.0-dev gawk
       - run: ./lazy.sh devsetup
 
-      - run: make
+      - run: make zrepl-bin
       - run: make vet
       - run: make test
       - run: make lint
@@ -108,3 +111,35 @@ jobs:
     <<: *build-latest
     docker:
     - image: circleci/golang:1.13
+
+  build-1.14:
+    <<: *build-latest
+    docker:
+    - image: circleci/golang:1.14
+
+  # this job tries to mimic the build-in-docker instructions
+  # given in docs/installation.rst
+  #
+  # However, CircleCI doesn't support volume mounts, so we have to copy
+  # the source into the build-container by modifying the Dockerfile
+  test-build-in-docker:
+    description: Check that build-in-docker works
+    docker:
+      - image: circleci/golang:latest
+    environment:
+    working_directory: /go/src/github.com/zrepl/zrepl
+    steps:
+    - checkout
+    - setup_remote_docker
+    - run:
+        name: (hacky) circleci doesn't allow volume mounts, so copy src to container
+        command: echo "ADD . /src" >> build.Dockerfile
+    - run:
+        name: (hacky) commit modified Dockerfile to avoid failing git clean check in Makefile
+        command: git -c user.name='circleci' -c user.email='circleci@localhost' commit -m 'CIRCLECI modified Dockerfile with zrepl src' --author 'autoauthor <circleci@localhost>' -- build.Dockerfile
+    - run:
+        name: build the build image (build deps)
+        command: docker build -t zrepl_build -f build.Dockerfile .
+    - run:
+        name: try compiling
+        command: docker run -it zrepl_build make release

Makefile

@@ -1,5 +1,6 @@
 .PHONY: generate build test vet cover release docs docs-clean clean format lint platformtest
-.DEFAULT_GOAL := build
+.PHONY: release bins-all release-noarch
+.DEFAULT_GOAL := zrepl-bin
 
 ARTIFACTDIR := artifacts
 
@@ -12,97 +13,71 @@ ifndef _ZREPL_VERSION
         $(error cannot infer variable ZREPL_VERSION using git and variable is not overriden by make invocation)
     endif
 endif
+
 GO := go
+GOOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOOS"')
+GOARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARCH"')
+GOARM ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOARM"')
+GOHOSTOS ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTOS"')
+GOHOSTARCH ?= $(shell bash -c 'source <($(GO) env) && echo "$$GOHOSTARCH"')
+GO_ENV_VARS := GO111MODULE=on
 GO_LDFLAGS := "-X github.com/zrepl/zrepl/version.zreplVersion=$(_ZREPL_VERSION)"
 GO_MOD_READONLY := -mod=readonly
-GO_BUILDFLAGS := $(GO_MOD_READONLY)
-GO_BUILD := GO111MODULE=on $(GO) build $(GO_BUILDFLAGS) -v -ldflags $(GO_LDFLAGS)
+GO_EXTRA_BUILDFLAGS :=
+GO_BUILDFLAGS := $(GO_MOD_READONLY) $(GO_EXTRA_BUILDFLAGS)
+GO_BUILD := $(GO_ENV_VARS) $(GO) build $(GO_BUILDFLAGS) -ldflags $(GO_LDFLAGS)
+GOLANGCI_LINT := golangci-lint
+ifneq ($(GOARM),)
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)v$(GOARM)
+else
+	ZREPL_TARGET_TUPLE := $(GOOS)-$(GOARCH)
+endif
 
-# keep in sync with vet target
-RELEASE_BINS := $(ARTIFACTDIR)/zrepl-freebsd-amd64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-amd64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-linux-arm64
-RELEASE_BINS += $(ARTIFACTDIR)/zrepl-darwin-amd64
+.PHONY: printvars
+printvars:
+	@echo GOOS=$(GOOS)
+	@echo GOARCH=$(GOARCH)
+	@echo GOARM=$(GOARM)
 
-RELEASE_NOARCH := $(ARTIFACTDIR)/zrepl-noarch.tar
-THIS_PLATFORM_RELEASE_BIN := $(shell bash -c 'source <($(GO) env) && echo "zrepl-$${GOOS}-$${GOARCH}"' )
 
-generate: #not part of the build, must do that manually
-	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
-	$(GO) generate $(GO_BUILDFLAGS) -x ./...
+##################### PRODUCING A RELEASE #############
+.PHONY: release wrapup-and-checksum check-git-clean sign clean
 
-format:
-	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
+release: clean
+	# no cross-platform support for target test
+	$(MAKE) test
+	$(MAKE) bins-all
+	$(MAKE) noarch
+	$(MAKE) wrapup-and-checksum
+	$(MAKE) check-git-clean
+ifeq (SIGN, 1)
+	$(make) sign
+endif
+	@echo "ZREPL RELEASE ARTIFACTS AVAILABLE IN artifacts/release"
 
-lint:
-	golangci-lint run ./...
-
-build:
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl"
-
-test:
-	$(GO) test $(GO_BUILDFLAGS) ./...
-	# TODO compile the tests for each supported platform
-	# but `go test -c ./...` is not supported
-
-vet:
-	$(GO) vet $(GO_BUILDFLAGS) ./...
-	# for each supported platform to cover conditional compilation
-	# (keep in sync with RELEASE_BINS)
-	GOOS=freebsd	GOARCH=amd64 	$(GO) vet $(GO_BUILDFLAGS) ./...
-	GOOS=linux		GOARCH=amd64 	$(GO) vet $(GO_BUILDFLAGS) ./...
-	GOOS=linux		GOARCH=arm64 	$(GO) vet $(GO_BUILDFLAGS) ./...
-	GOOS=darwin		GOARCH=amd64 	$(GO) vet $(GO_BUILDFLAGS) ./...
-
-ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
-ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
-$(ARTIFACTDIR)/zrepl_platformtest:
-	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl_platformtest" ./platformtest/harness
-platformtest: $(ARTIFACTDIR)/zrepl_platformtest
-	"$(ARTIFACTDIR)/zrepl_platformtest" -poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" -imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)"
-
-$(ARTIFACTDIR):
-	mkdir -p "$@"
-
-$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
-	mkdir -p "$@"
-
-$(ARTIFACTDIR)/bash_completion: $(RELEASE_BINS)
-	artifacts/$(THIS_PLATFORM_RELEASE_BIN) bashcomp "$@"
-
-.PHONY: $(ARTIFACTDIR)/go_version.txt
-$(ARTIFACTDIR)/go_version.txt:
-	$(GO) version > $@
-
-docs: $(ARTIFACTDIR)/docs
-	make -C docs \
-		html \
-		BUILDDIR=../artifacts/docs \
-
-docs-clean:
-	make -C docs \
-		clean \
-		BUILDDIR=../artifacts/docs
-
-.PHONY: $(RELEASE_BINS)
-# TODO: two wildcards possible
-$(RELEASE_BINS): $(ARTIFACTDIR)/zrepl-%: generate $(ARTIFACTDIR) vet test lint
-	STEM=$*; GOOS="$${STEM%%-*}"; GOARCH="$${STEM##*-}"; export GOOS GOARCH; \
-		$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$$GOOS-$$GOARCH"
-
-$(RELEASE_NOARCH): docs $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/go_version.txt
+# expects `release` target to have run before
+NOARCH_TARBALL := $(ARTIFACTDIR)/zrepl-noarch.tar
+wrapup-and-checksum:
+	rm -f $(NOARCH_TARBALL)
 	tar --mtime='1970-01-01' --sort=name \
 		--transform 's/$(ARTIFACTDIR)/zrepl-$(_ZREPL_VERSION)-noarch/' \
-		-acf $@ \
+		--transform 's#dist#zrepl-$(_ZREPL_VERSION)-noarch/dist#' \
+		--transform 's#config/samples#zrepl-$(_ZREPL_VERSION)-noarch/config#' \
+		-acf $(NOARCH_TARBALL) \
 		$(ARTIFACTDIR)/docs/html \
 		$(ARTIFACTDIR)/bash_completion \
-		$(ARTIFACTDIR)/go_version.txt
-
-release: $(RELEASE_BINS) $(RELEASE_NOARCH)
+		$(ARTIFACTDIR)/_zrepl.zsh_completion \
+		$(ARTIFACTDIR)/go_env.txt \
+		dist \
+		config/samples
 	rm -rf "$(ARTIFACTDIR)/release"
 	mkdir -p "$(ARTIFACTDIR)/release"
-	cp $^ "$(ARTIFACTDIR)/release"
+	cp -l $(ARTIFACTDIR)/zrepl-* \
+		$(ARTIFACTDIR)/platformtest-* \
+		"$(ARTIFACTDIR)/release"
 	cd "$(ARTIFACTDIR)/release" && sha512sum $$(ls | sort) > sha512sum.txt
+
+check-git-clean:
 	@# note that we use ZREPL_VERSION and not _ZREPL_VERSION because we want to detect the override
 	@if git describe --always --dirty 2>/dev/null | grep dirty >/dev/null; then \
         echo '[INFO] either git reports checkout is dirty or git is not installed or this is not a git checkout'; \
@@ -115,5 +90,105 @@ release: $(RELEASE_BINS) $(RELEASE_NOARCH)
 		fi; \
 	fi;
 
+sign:
+	gpg -u "89BC 5D89 C845 568B F578  B306 CDBD 8EC8 E27C A5FC" \
+		--armor \
+		--detach-sign $(ARTIFACTDIR)/release/sha512sum.txt
+
 clean: docs-clean
 	rm -rf "$(ARTIFACTDIR)"
+
+##################### BINARIES #####################
+.PHONY: bins-all lint test vet zrepl-bin platformtest-bin
+
+BINS_ALL_TARGETS := zrepl-bin platformtest-bin vet lint
+GO_SUPPORTS_ILLUMOS := $(shell $(GO) version | gawk -F '.' '/^go version /{split($$0, comps, " "); split(comps[3], v, "."); if (v[1] == "go1" && v[2] >= 13) { print "illumos"; } else { print "noillumos"; }}')
+bins-all:
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=freebsd   GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=arm     GOARM=7
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=linux     GOARCH=386
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=darwin    GOARCH=amd64
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=solaris   GOARCH=amd64
+ifeq ($(GO_SUPPORTS_ILLUMOS), illumos)
+	$(MAKE) $(BINS_ALL_TARGETS) GOOS=illumos   GOARCH=amd64
+else ifeq ($(GO_SUPPORTS_ILLUMOS), noillumos)
+	@echo "SKIPPING ILLUMOS BUILD BECAUSE GO VERSION DOESN'T SUPPORT IT"
+else
+	@echo "CANNOT DETERMINE WHETHER GO VERSION SUPPORTS GOOS=illumos"; exit 1
+endif
+
+lint:
+	$(GO_ENV_VARS) $(GOLANGCI_LINT) run ./...
+test:
+	$(GO_ENV_VARS) $(GO) test $(GO_BUILDFLAGS) ./...
+vet:
+	$(GO_ENV_VARS) $(GO) vet $(GO_BUILDFLAGS) ./...
+
+zrepl-bin:
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/zrepl-$(ZREPL_TARGET_TUPLE)"
+
+platformtest-bin:
+	$(GO_BUILD) -o "$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" ./platformtest/harness
+
+##################### DEV TARGETS #####################
+# not part of the build, must do that manually
+.PHONY: generate format platformtest
+
+generate:
+	protoc -I=replication/logic/pdu --go_out=plugins=grpc:replication/logic/pdu replication/logic/pdu/pdu.proto
+	$(GO_ENV_VARS) $(GO) generate $(GO_BUILDFLAGS) -x ./...
+
+format:
+	goimports -srcdir . -local 'github.com/zrepl/zrepl' -w $(shell find . -type f -name '*.go' -not -path "./vendor/*" -not -name '*.pb.go' -not -name '*_enumer.go')
+
+ZREPL_PLATFORMTEST_POOLNAME := zreplplatformtest
+ZREPL_PLATFORMTEST_IMAGEPATH := /tmp/zreplplatformtest.pool.img
+ZREPL_PLATFORMTEST_MOUNTPOINT := /tmp/zreplplatformtest.pool
+ZREPL_PLATFORMTEST_ZFS_LOG := /tmp/zreplplatformtest.zfs.log
+# ZREPL_PLATFORMTEST_STOP_AND_KEEP := -failure.stop-and-keep-pool
+ZREPL_PLATFORMTEST_ARGS := 
+platformtest: # do not track dependency on platformtest-bin to allow build of platformtest outside of test VM
+	rm -f "$(ZREPL_PLATFORMTEST_ZFS_LOG)"
+	platformtest/logmockzfs/logzfsenv "$(ZREPL_PLATFORMTEST_ZFS_LOG)" `which zfs` \
+	"$(ARTIFACTDIR)/platformtest-$(ZREPL_TARGET_TUPLE)" \
+		-poolname "$(ZREPL_PLATFORMTEST_POOLNAME)" \
+		-imagepath "$(ZREPL_PLATFORMTEST_IMAGEPATH)" \
+		-mountpoint "$(ZREPL_PLATFORMTEST_MOUNTPOINT)" \
+		$(ZREPL_PLATFORMTEST_STOP_AND_KEEP) \
+		$(ZREPL_PLATFORMTEST_ARGS)
+
+##################### NOARCH #####################
+.PHONY: noarch $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs docs-clean
+
+
+$(ARTIFACTDIR):
+	mkdir -p "$@"
+$(ARTIFACTDIR)/docs: $(ARTIFACTDIR)
+	mkdir -p "$@"
+
+noarch: $(ARTIFACTDIR)/bash_completion $(ARTIFACTDIR)/_zrepl.zsh_completion $(ARTIFACTDIR)/go_env.txt docs
+	# pass
+
+$(ARTIFACTDIR)/bash_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion bash "$@"
+
+$(ARTIFACTDIR)/_zrepl.zsh_completion:
+	$(MAKE) zrepl-bin GOOS=$(GOHOSTOS) GOARCH=$(GOHOSTARCH)
+	artifacts/zrepl-$(GOHOSTOS)-$(GOHOSTARCH) gencompletion zsh "$@"
+
+$(ARTIFACTDIR)/go_env.txt:
+	$(GO_ENV_VARS) $(GO) env > $@
+
+docs: $(ARTIFACTDIR)/docs
+	make -C docs \
+		html \
+		BUILDDIR=../artifacts/docs \
+
+docs-clean:
+	make -C docs \
+		clean \
+		BUILDDIR=../artifacts/docs

README.md

@@ -1,8 +1,9 @@
 [![GitHub license](https://img.shields.io/github/license/zrepl/zrepl.svg)](https://github.com/zrepl/zrepl/blob/master/LICENSE)
 [![Language: Go](https://img.shields.io/badge/language-Go-6ad7e5.svg)](https://golang.org/)
 [![User Docs](https://img.shields.io/badge/docs-web-blue.svg)](https://zrepl.github.io)
-[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
+[![Donate via Patreon](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Fshieldsio-patreon.herokuapp.com%2Fzrepl%2Fpledges&style=flat&color=yellow)](https://www.patreon.com/zrepl)
 [![Donate via Liberapay](https://img.shields.io/liberapay/receives/zrepl.svg?logo=liberapay)](https://liberapay.com/zrepl/donate)
+[![Donate via PayPal](https://img.shields.io/badge/donate-paypal-yellow.svg)](https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=R5QSXJVYHGX96)
 [![Twitter](https://img.shields.io/twitter/url/https/github.com/zrepl/zrepl.svg?style=social)](https://twitter.com/intent/tweet?text=Wow:&url=https%3A%2F%2Fgithub.com%2Fzrepl%2Fzrepl)
 
 # zrepl
@@ -23,6 +24,7 @@ zrepl is a one-stop ZFS backup & replication solution.
    If so, think of an expressive configuration example.
 2. Think of at least one use case that generalizes from your concrete application.
 3. Open an issue on GitHub with example conf & use case attached.
+4. **Optional**: [Post a bounty](https://www.bountysource.com/teams/zrepl) on the issue, or [contact Christian Schwarz](https://cschwarz.com) for contract work.
 
 The above does not apply if you already implemented everything.
 Check out the *Coding Workflow* section below for details.
@@ -46,11 +48,8 @@ zrepl is written in [Go](https://golang.org) and uses [Go modules](https://githu
 The documentation is written in [ReStructured Text](http://docutils.sourceforge.net/rst.html) using the [Sphinx](https://www.sphinx-doc.org) framework.
 
 To get started, run `./lazy.sh devsetup` to easily install build dependencies and read `docs/installation.rst -> Compiling from Source`.
-
-### Overall Architecture
-
-The application architecture is documented as part of the user docs in the *Implementation* section (`docs/content/impl`).
-Make sure to develop an understanding how zrepl is typically used by studying the user docs first.
+`lazy.sh` uses `python3-pip` to fetch the build dependencies for the docs - you might want to use a [venv](https://docs.python.org/3/library/venv.html).
+If you just want to install the Go dependencies, run `./lazy.sh godep`.
 
 ### Project Structure
 
@@ -62,14 +61,15 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   └── samples
 ├── daemon                  # the implementation of `zrepl daemon` subcommand
 │   ├── filters
+│   ├── hooks               # snapshot hooks
 │   ├── job                 # job implementations
 │   ├── logging             # logging outlets + formatters
 │   ├── nethelpers
 │   ├── prometheus
 │   ├── pruner              # pruner implementation
 │   ├── snapper             # snapshotter implementation
-├── docs                    # sphinx-based documentation
 ├── dist                    # supplemental material for users & package maintainers
+├── docs                    # sphinx-based documentation
 │   ├── **/*.rst            # documentation in reStructuredText
 │   ├── sphinxconf
 │   │   └── conf.py         # sphinx config (see commit 445a280 why its not in docs/)
@@ -78,6 +78,7 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   └── public_git          # checkout of zrepl.github.io managed by above shell script
 ├── endpoint                # implementation of replication endpoints (=> package replication)
 ├── logger                  # our own logger package
+├── platformtest            # test suite for our zfs abstractions (error classification, etc)
 ├── pruning                 # pruning rules (the logic, not the actual execution)
 │   └── retentiongrid
 ├── replication
@@ -92,14 +93,13 @@ Make sure to develop an understanding how zrepl is typically used by studying th
 │   ├── transportmux        # TCP connecter and listener used to split control & data traffic
 │   └── versionhandshake    # replication protocol version handshake perfomed on newly established connections
 ├── tlsconf                 # abstraction for Go TLS server + client config
-├── transport               # transports implementation
+├── transport               # transport implementations
 │   ├── fromconfig
 │   ├── local
 │   ├── ssh
 │   ├── tcp
 │   └── tls
 ├── util
-├── vendor                  # managed by dep
 ├── version                 # abstraction for versions (filled during build by Makefile)
 └── zfs                     # zfs(8) wrappers
 ```
@@ -134,3 +134,15 @@ There will not be a big refactoring (an attempt was made, but it's destroying to
 
 However, new contributions & patches should fix naming without further notice in the commit message.
 
+### RPC debugging
+
+Optionally, there are various RPC-related environment variables, that if set to something != `""` will produce additional debug output on stderr:
+
+https://github.com/zrepl/zrepl/blob/master/rpc/rpc_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/dataconn_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/stream/stream_debug.go#L11
+
+https://github.com/zrepl/zrepl/blob/master/rpc/dataconn/heartbeatconn/heartbeatconn_debug.go#L11
+

build.Dockerfile

@@ -2,7 +2,8 @@ FROM golang:latest
 
 RUN apt-get update && apt-get install -y \
     python3-pip \
-    unzip
+    unzip \
+    gawk
 
 ADD build.installprotoc.bash ./
 RUN bash build.installprotoc.bash
@@ -22,7 +23,7 @@ RUN mkdir -p /.cache && chmod -R 0777 /.cache
 WORKDIR /src
 
 # Install build tools (e.g. protobuf generator, stringer) into $GOPATH/bin
-ADD go.mod go.sum ./
+ADD build/ /tmp/build
 RUN /tmp/lazy.sh godep
 
 RUN chmod -R 0777 /go

build/build.go

@@ -1,23 +0,0 @@
-//+build windows
-
-// This package cannot actually be built, and since the Travis
-// tests run go test ./..., we have to avoid a build attempt.
-// Windows is not supported atm, so this works ¯\_(ツ)_/¯
-
-// This package is a pseudo-user of build-time dependencies for zrepl,
-// mostly for various code-generation tools.
-//
-// The imports are necessary to satisfy go dep
-package main
-
-import (
-	"fmt"
-
-	_ "github.com/alvaroloes/enumer"
-	_ "github.com/golang/protobuf/protoc-gen-go"
-	_ "golang.org/x/tools/cmd/stringer"
-)
-
-func main() {
-	fmt.Println("just a placeholder")
-}

build/go.mod

@@ -0,0 +1,51 @@
+module github.com/zrepl/zrepl/build
+
+go 1.12
+
+require (
+	github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
+	github.com/alvaroloes/enumer v1.1.1
+	github.com/fatih/color v1.7.0 // indirect
+	github.com/go-critic/go-critic v0.3.4 // indirect
+	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/golang/mock v1.2.0 // indirect
+	github.com/golang/protobuf v1.2.0
+	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
+	github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
+	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
+	github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
+	github.com/golangci/golangci-lint v1.17.1
+	github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
+	github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
+	github.com/golangci/misspell v0.3.4 // indirect
+	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/google/go-cmp v0.3.0 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/pkg/errors v0.8.1 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
+	github.com/spf13/afero v1.2.2 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/spf13/viper v1.3.2 // indirect
+	github.com/stretchr/testify v1.4.0 // indirect
+	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 // indirect
+	golang.org/x/sys v0.0.0-20191010194322-b09406accb47 // indirect
+	golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
+	mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
+	sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
+)
+
+// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
+replace (
+	github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
+	github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
+	github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
+	github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
+	github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
+	github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
+	github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
+	github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
+	github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
+	golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
+	mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
+)

build/go.sum

@@ -0,0 +1,280 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
+github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
+github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
+github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
+github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
+github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
+github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
+github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
+github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
+github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
+github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
+github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
+github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
+github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
+github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
+github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
+github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
+github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
+github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
+github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
+github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
+github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
+github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
+github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
+github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
+github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
+github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
+github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
+github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
+github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
+github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
+github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
+github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
+github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
+github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
+github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
+github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
+github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
+github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
+github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
+github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
+github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
+github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
+github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
+github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
+github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
+github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
+github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
+github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
+github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
+github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
+github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
+github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
+github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
+github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
+github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
+github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
+github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
+github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
+github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
+github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/gotool v0.0.0-20161130080628-0de1eaf82fa3/go.mod h1:jxZFDH7ILpTPQTk+E2s+z4CUas9lVNjIuKR4c5/zKgM=
+github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
+github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
+github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
+github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
+github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
+github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
+github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
+github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
+github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
+github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
+github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
+github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
+github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
+github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
+github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
+github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
+github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
+github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
+github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
+github.com/valyala/quicktemplate v1.1.1/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
+github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
+golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
+golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
+golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
+mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
+mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
+mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
+mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
+mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
+sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
+sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
+sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

build/tools.go

@@ -3,9 +3,9 @@
 package main
 
 import (
-	_ "golang.org/x/tools/cmd/stringer"
-	_ "github.com/golang/protobuf/protoc-gen-go"
 	_ "github.com/alvaroloes/enumer"
-	_ "golang.org/x/tools/cmd/goimports"
+	_ "github.com/golang/protobuf/protoc-gen-go"
 	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "golang.org/x/tools/cmd/goimports"
+	_ "golang.org/x/tools/cmd/stringer"
 )

cli/cli.go

@@ -19,29 +19,55 @@ var rootCmd = &cobra.Command{
 	Short: "One-stop ZFS replication solution",
 }
 
-var bashcompCmd = &cobra.Command{
-	Use:   "bashcomp path/to/out/file",
-	Short: "generate bash completions",
-	Run: func(cmd *cobra.Command, args []string) {
-		if len(args) != 1 {
-			fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
-			err := cmd.Usage()
-			if err != nil {
-				panic(err)
-			}
-			os.Exit(1)
-		}
-		if err := rootCmd.GenBashCompletionFile(args[0]); err != nil {
-			fmt.Fprintf(os.Stderr, "error generating bash completion: %s", err)
-			os.Exit(1)
-		}
+func init() {
+	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
+}
+
+var genCompletionCmd = &cobra.Command{
+	Use:   "gencompletion",
+	Short: "generate shell auto-completions",
+}
+
+type completionCmdInfo struct {
+	genFunc func(outpath string) error
+	help    string
+}
+
+var completionCmdMap = map[string]completionCmdInfo{
+	"zsh": {
+		rootCmd.GenZshCompletionFile,
+		"  save to file `_zrepl` in your zsh's $fpath",
+	},
+	"bash": {
+		rootCmd.GenBashCompletionFile,
+		"  save to a path and source that path in your .bashrc",
 	},
-	Hidden: true,
 }
 
 func init() {
-	rootCmd.PersistentFlags().StringVar(&rootArgs.configPath, "config", "", "config file path")
-	rootCmd.AddCommand(bashcompCmd)
+	for sh, info := range completionCmdMap {
+		sh, info := sh, info
+		genCompletionCmd.AddCommand(&cobra.Command{
+			Use:     fmt.Sprintf("%s path/to/out/file", sh),
+			Short:   fmt.Sprintf("generate %s completions", sh),
+			Example: info.help,
+			Run: func(cmd *cobra.Command, args []string) {
+				if len(args) != 1 {
+					fmt.Fprintf(os.Stderr, "specify exactly one positional agument\n")
+					err := cmd.Usage()
+					if err != nil {
+						panic(err)
+					}
+					os.Exit(1)
+				}
+				if err := info.genFunc(args[0]); err != nil {
+					fmt.Fprintf(os.Stderr, "error generating %s completion: %s", sh, err)
+					os.Exit(1)
+				}
+			},
+		})
+	}
+	rootCmd.AddCommand(genCompletionCmd)
 }
 
 type Subcommand struct {

client/migrate.go

@@ -4,9 +4,13 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/fatih/color"
+	"github.com/kr/pretty"
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
 
+	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/zfs"
 
 	"github.com/zrepl/zrepl/cli"
@@ -31,6 +35,13 @@ var migrations = []*cli.Subcommand{
 			f.BoolVar(&migratePlaceholder0_1Args.dryRun, "dry-run", false, "dry run")
 		},
 	},
+	&cli.Subcommand{
+		Use: "replication-cursor:v1-v2",
+		Run: doMigrateReplicationCursor,
+		SetupFlags: func(f *pflag.FlagSet) {
+			f.BoolVar(&migrateReplicationCursorArgs.dryRun, "dry-run", false, "dry run")
+		},
+	},
 }
 
 var migratePlaceholder0_1Args struct {
@@ -88,7 +99,7 @@ func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
 		}
 		for _, fs := range wi.fss {
 			fmt.Printf("\t%q ... ", fs.ToString())
-			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(fs, migratePlaceholder0_1Args.dryRun)
+			r, err := zfs.ZFSMigrateHashBasedPlaceholderToCurrent(ctx, fs, migratePlaceholder0_1Args.dryRun)
 			if err != nil {
 				fmt.Printf("error: %s\n", err)
 			} else if !r.NeedsModification {
@@ -102,3 +113,159 @@ func doMigratePlaceholder0_1(sc *cli.Subcommand, args []string) error {
 
 	return nil
 }
+
+var migrateReplicationCursorArgs struct {
+	dryRun bool
+}
+
+var bold = color.New(color.Bold)
+var succ = color.New(color.FgGreen)
+var fail = color.New(color.FgRed)
+
+var migrateReplicationCursorSkipSentinel = fmt.Errorf("skipping this filesystem")
+
+func doMigrateReplicationCursor(sc *cli.Subcommand, args []string) error {
+	if len(args) != 0 {
+		return fmt.Errorf("migration does not take arguments, got %v", args)
+	}
+
+	cfg := sc.Config()
+	jobs, err := job.JobsFromConfig(cfg)
+	if err != nil {
+		fmt.Printf("cannot parse config:\n%s\n\n", err)
+		fmt.Printf("NOTE: this migration was released together with a change in job name requirements.\n")
+		return fmt.Errorf("exiting migration after error")
+	}
+
+	ctx := context.Background()
+
+	v1cursorJobs := make([]job.Job, 0, len(cfg.Jobs))
+	for i, j := range cfg.Jobs {
+		if jobs[i].Name() != j.Name() {
+			panic("implementation error")
+		}
+		switch j.Ret.(type) {
+		case *config.PushJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		case *config.SourceJob:
+			v1cursorJobs = append(v1cursorJobs, jobs[i])
+		default:
+			fmt.Printf("ignoring job %q (%d/%d, type %T), not supposed to create v1 replication cursors\n", j.Name(), i, len(cfg.Jobs), j.Ret)
+			continue
+		}
+	}
+
+	// scan all filesystems for v1 replication cursors
+
+	fss, err := zfs.ZFSListMapping(ctx, zfs.NoFilter())
+	if err != nil {
+		return errors.Wrap(err, "list filesystems")
+	}
+
+	var hadError bool
+	for _, fs := range fss {
+
+		bold.Printf("INSPECT FILESYSTEM %q\n", fs.ToString())
+
+		err := doMigrateReplicationCursorFS(ctx, v1cursorJobs, fs)
+		if err == migrateReplicationCursorSkipSentinel {
+			bold.Printf("FILESYSTEM SKIPPED\n")
+		} else if err != nil {
+			hadError = true
+			fail.Printf("MIGRATION FAILED: %s\n", err)
+		} else {
+			succ.Printf("FILESYSTEM %q COMPLETE\n", fs.ToString())
+		}
+	}
+
+	if hadError {
+		fail.Printf("\n\none or more filesystems could not be migrated, please inspect output and or re-run migration")
+		return errors.Errorf("")
+	}
+	return nil
+}
+
+func doMigrateReplicationCursorFS(ctx context.Context, v1CursorJobs []job.Job, fs *zfs.DatasetPath) error {
+
+	var owningJob job.Job = nil
+	for _, job := range v1CursorJobs {
+		conf := job.SenderConfig()
+		if conf == nil {
+			continue
+		}
+		pass, err := conf.FSF.Filter(fs)
+		if err != nil {
+			return errors.Wrapf(err, "filesystem filter error in job %q for fs %q", job.Name(), fs.ToString())
+		}
+		if !pass {
+			continue
+		}
+		if owningJob != nil {
+			return errors.Errorf("jobs %q and %q both match %q\ncannot attribute replication cursor to either one", owningJob.Name(), job.Name(), fs)
+		}
+		owningJob = job
+	}
+	if owningJob == nil {
+		fmt.Printf("no job's Filesystems filter matches\n")
+		return migrateReplicationCursorSkipSentinel
+	}
+	fmt.Printf("identified owning job %q\n", owningJob.Name())
+
+	bookmarks, err := zfs.ZFSListFilesystemVersions(ctx, fs, zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "list filesystem versions of %q", fs.ToString())
+	}
+
+	var oldCursor *zfs.FilesystemVersion
+	for i, fsv := range bookmarks {
+		_, _, err := endpoint.ParseReplicationCursorBookmarkName(fsv.ToAbsPath(fs))
+		if err != endpoint.ErrV1ReplicationCursor {
+			continue
+		}
+
+		if oldCursor != nil {
+			fmt.Printf("unexpected v1 replication cursor candidate: %q", fsv.ToAbsPath(fs))
+			return errors.Wrap(err, "multiple filesystem versions identified as v1 replication cursors")
+		}
+
+		oldCursor = &bookmarks[i]
+
+	}
+
+	if oldCursor == nil {
+		bold.Printf("no v1 replication cursor found for filesystem %q\n", fs.ToString())
+		return migrateReplicationCursorSkipSentinel
+	}
+
+	fmt.Printf("found v1 replication cursor:\n%s\n", pretty.Sprint(oldCursor))
+
+	mostRecentNew, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs.ToString(), owningJob.SenderConfig().JobID)
+	if err != nil {
+		return errors.Wrapf(err, "get most recent v2 replication cursor")
+	}
+
+	if mostRecentNew == nil {
+		return errors.Errorf("no v2 replication cursor found for job %q on filesystem %q", owningJob.SenderConfig().JobID, fs.ToString())
+	}
+
+	fmt.Printf("most recent v2 replication cursor:\n%#v", oldCursor)
+
+	if !(mostRecentNew.CreateTXG >= oldCursor.CreateTXG) {
+		return errors.Errorf("v1 replication cursor createtxg is higher than v2 cursor's, skipping this filesystem")
+	}
+
+	fmt.Printf("determined that v2 cursor is bookmark of same or newer version than v1 cursor\n")
+	fmt.Printf("destroying v1 cursor %q\n", oldCursor.ToAbsPath(fs))
+
+	if migrateReplicationCursorArgs.dryRun {
+		succ.Printf("DRY RUN\n")
+	} else {
+		if err := zfs.ZFSDestroyFilesystemVersion(ctx, fs, oldCursor); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

client/status.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 	"time"
 
-	// tcell is the termbox-compatbile library for abstracting away escape sequences, etc.
+	// tcell is the termbox-compatible library for abstracting away escape sequences, etc.
 	// as of tcell#252, the number of default distributed terminals is relatively limited
 	// additional terminal definitions can be included via side-effect import
 	// See https://github.com/gdamore/tcell/blob/master/terminfo/base/base.go
@@ -81,7 +81,7 @@ type tui struct {
 	indent int
 
 	lock   sync.Mutex //For report and error
-	report map[string]job.Status
+	report map[string]*job.Status
 	err    error
 
 	jobFilter string
@@ -219,7 +219,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
 	defer termbox.Close()
 
 	update := func() {
-		m := make(map[string]job.Status)
+		var m daemon.Status
 
 		err2 := jsonRequestResponse(httpc, daemon.ControlJobEndpointStatus,
 			struct{}{},
@@ -228,7 +228,7 @@ func runStatus(s *cli.Subcommand, args []string) error {
 
 		t.lock.Lock()
 		t.err = err2
-		t.report = m
+		t.report = m.Jobs
 		t.lock.Unlock()
 		t.draw()
 	}
@@ -264,7 +264,7 @@ loop:
 
 }
 
-func (t *tui) getReplicationProgresHistory(jobName string) *bytesProgressHistory {
+func (t *tui) getReplicationProgressHistory(jobName string) *bytesProgressHistory {
 	p, ok := t.replicationProgress[jobName]
 	if !ok {
 		p = &bytesProgressHistory{}
@@ -329,7 +329,7 @@ func (t *tui) draw() {
 				t.printf("Replication:")
 				t.newline()
 				t.addIndent(1)
-				t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgresHistory(k))
+				t.renderReplicationReport(activeStatus.Replication, t.getReplicationProgressHistory(k))
 				t.addIndent(-1)
 
 				t.printf("Pruning Sender:")
@@ -461,12 +461,23 @@ func (t *tui) renderReplicationReport(rep *report.Report, history *bytesProgress
 	if latest.State != report.AttemptPlanning && latest.State != report.AttemptPlanningError {
 		// Draw global progress bar
 		// Progress: [---------------]
-		expected, replicated := latest.BytesSum()
+		expected, replicated, containsInvalidSizeEstimates := latest.BytesSum()
 		rate, changeCount := history.Update(replicated)
+		eta := time.Duration(0)
+		if rate > 0 {
+			eta = time.Duration((expected-replicated)/rate) * time.Second
+		}
 		t.write("Progress: ")
 		t.drawBar(50, replicated, expected, changeCount)
 		t.write(fmt.Sprintf(" %s / %s @ %s/s", ByteCountBinary(replicated), ByteCountBinary(expected), ByteCountBinary(rate)))
+		if eta != 0 {
+			t.write(fmt.Sprintf(" (%s remaining)", humanizeDuration(eta)))
+		}
 		t.newline()
+		if containsInvalidSizeEstimates {
+			t.write("NOTE: not all steps could be size-estimated, total estimate is likely imprecise!")
+			t.newline()
+		}
 
 		var maxFSLen int
 		for _, fs := range latest.Filesystems {
@@ -686,7 +697,7 @@ func rightPad(str string, length int, pad string) string {
 
 var arrowPositions = `>\|/`
 
-// changeCount = 0 indicates stall / no progresss
+// changeCount = 0 indicates stall / no progress
 func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
 	var completedLength int
 	if totalBytes > 0 {
@@ -707,11 +718,20 @@ func (t *tui) drawBar(length int, bytes, totalBytes int64, changeCount int) {
 
 func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, maxFS int) {
 
-	expected, replicated := rep.BytesSum()
-	status := fmt.Sprintf("%s (step %d/%d, %s/%s)",
+	expected, replicated, containsInvalidSizeEstimates := rep.BytesSum()
+	sizeEstimationImpreciseNotice := ""
+	if containsInvalidSizeEstimates {
+		sizeEstimationImpreciseNotice = " (some steps lack size estimation)"
+	}
+	if rep.CurrentStep < len(rep.Steps) && rep.Steps[rep.CurrentStep].Info.BytesExpected == 0 {
+		sizeEstimationImpreciseNotice = " (step lacks size estimation)"
+	}
+
+	status := fmt.Sprintf("%s (step %d/%d, %s/%s)%s",
 		strings.ToUpper(string(rep.State)),
 		rep.CurrentStep, len(rep.Steps),
 		ByteCountBinary(replicated), ByteCountBinary(expected),
+		sizeEstimationImpreciseNotice,
 	)
 
 	activeIndicator := " "
@@ -731,8 +751,17 @@ func (t *tui) printFilesystemStatus(rep *report.FilesystemReport, active bool, m
 			if nextStep.IsIncremental() {
 				next = fmt.Sprintf("next: %s => %s", nextStep.Info.From, nextStep.Info.To)
 			} else {
-				next = fmt.Sprintf("next: %s (full)", nextStep.Info.To)
+				next = fmt.Sprintf("next: full send %s", nextStep.Info.To)
 			}
+			attribs := []string{}
+
+			if nextStep.Info.Resumed {
+				attribs = append(attribs, "resumed")
+			}
+
+			attribs = append(attribs, fmt.Sprintf("encrypted=%s", nextStep.Info.Encrypted))
+
+			next += fmt.Sprintf(" (%s)", strings.Join(attribs, ", "))
 		} else {
 			next = "" // individual FSes may still be in planning state
 		}
@@ -755,3 +784,27 @@ func ByteCountBinary(b int64) string {
 	}
 	return fmt.Sprintf("%.1f %ciB", float64(b)/float64(div), "KMGTPE"[exp])
 }
+
+func humanizeDuration(duration time.Duration) string {
+	days := int64(duration.Hours() / 24)
+	hours := int64(math.Mod(duration.Hours(), 24))
+	minutes := int64(math.Mod(duration.Minutes(), 60))
+	seconds := int64(math.Mod(duration.Seconds(), 60))
+
+	var parts []string
+
+	force := false
+	chunks := []int64{days, hours, minutes, seconds}
+	for i, chunk := range chunks {
+		if force || chunk > 0 {
+			padding := 0
+			if force {
+				padding = 2
+			}
+			parts = append(parts, fmt.Sprintf("%*d%c", padding, chunk, "dhms"[i]))
+			force = true
+		}
+	}
+
+	return strings.Join(parts, " ")
+}

client/testcmd.go

@@ -1,7 +1,10 @@
 package client
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"os"
 
 	"github.com/pkg/errors"
 	"github.com/spf13/pflag"
@@ -15,7 +18,7 @@ import (
 var TestCmd = &cli.Subcommand{
 	Use: "test",
 	SetupSubcommands: func() []*cli.Subcommand {
-		return []*cli.Subcommand{testFilter, testPlaceholder}
+		return []*cli.Subcommand{testFilter, testPlaceholder, testDecodeResumeToken}
 	},
 }
 
@@ -46,6 +49,7 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 	}
 
 	conf := subcommand.Config()
+	ctx := context.Background()
 
 	var confFilter config.FilesystemsFilter
 	job, err := conf.Job(testFilterArgs.job)
@@ -57,6 +61,8 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 		confFilter = j.Filesystems
 	case *config.PushJob:
 		confFilter = j.Filesystems
+	case *config.SnapJob:
+		confFilter = j.Filesystems
 	default:
 		return fmt.Errorf("job type %T does not have filesystems filter", j)
 	}
@@ -70,7 +76,7 @@ func runTestFilterCmd(subcommand *cli.Subcommand, args []string) error {
 	if testFilterArgs.input != "" {
 		fsnames = []string{testFilterArgs.input}
 	} else {
-		out, err := zfs.ZFSList([]string{"name"})
+		out, err := zfs.ZFSList(ctx, []string{"name"})
 		if err != nil {
 			return fmt.Errorf("could not list ZFS filesystems: %s", err)
 		}
@@ -134,10 +140,11 @@ var testPlaceholder = &cli.Subcommand{
 func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
 
 	var checkDPs []*zfs.DatasetPath
+	ctx := context.Background()
 
 	// all actions first
 	if testPlaceholderArgs.all {
-		out, err := zfs.ZFSList([]string{"name"})
+		out, err := zfs.ZFSList(ctx, []string{"name"})
 		if err != nil {
 			return errors.Wrap(err, "could not list ZFS filesystems")
 		}
@@ -161,7 +168,7 @@ func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
 
 	fmt.Printf("IS_PLACEHOLDER\tDATASET\tzrepl:placeholder\n")
 	for _, dp := range checkDPs {
-		ph, err := zfs.ZFSGetFilesystemPlaceholderState(dp)
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, dp)
 		if err != nil {
 			return errors.Wrap(err, "cannot get placeholder state")
 		}
@@ -176,3 +183,32 @@ func runTestPlaceholder(subcommand *cli.Subcommand, args []string) error {
 	}
 	return nil
 }
+
+var testDecodeResumeTokenArgs struct {
+	token string
+}
+
+var testDecodeResumeToken = &cli.Subcommand{
+	Use:   "decoderesumetoken --token TOKEN",
+	Short: "decode resume token",
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVar(&testDecodeResumeTokenArgs.token, "token", "", "the resume token obtained from the receive_resume_token property")
+	},
+	Run: runTestDecodeResumeTokenCmd,
+}
+
+func runTestDecodeResumeTokenCmd(subcommand *cli.Subcommand, args []string) error {
+	if testDecodeResumeTokenArgs.token == "" {
+		return fmt.Errorf("token argument must be specified")
+	}
+	token, err := zfs.ParseResumeToken(context.Background(), testDecodeResumeTokenArgs.token)
+	if err != nil {
+		return err
+	}
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", " ")
+	if err := enc.Encode(&token); err != nil {
+		panic(err)
+	}
+	return nil
+}

client/zfsabstractions.go

@@ -0,0 +1,147 @@
+package client
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/daemon/filters"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var (
+	ZFSAbstractionsCmd = &cli.Subcommand{
+		Use:   "zfs-abstraction",
+		Short: "manage abstractions that zrepl builds on top of ZFS",
+		SetupSubcommands: func() []*cli.Subcommand {
+			return []*cli.Subcommand{
+				zabsCmdList,
+				zabsCmdReleaseAll,
+				zabsCmdReleaseStale,
+				zabsCmdCreate,
+			}
+		},
+	}
+)
+
+// a common set of CLI flags that map to the fields of an
+// endpoint.ListZFSHoldsAndBookmarksQuery
+type zabsFilterFlags struct {
+	Filesystems FilesystemsFilterFlag
+	Job         JobIDFlag
+	Types       AbstractionTypesFlag
+	Concurrency int64
+}
+
+// produce a query from the CLI flags
+func (f zabsFilterFlags) Query() (endpoint.ListZFSHoldsAndBookmarksQuery, error) {
+	q := endpoint.ListZFSHoldsAndBookmarksQuery{
+		FS:          f.Filesystems.FlagValue(),
+		What:        f.Types.FlagValue(),
+		JobID:       f.Job.FlagValue(),
+		Concurrency: f.Concurrency,
+	}
+	return q, q.Validate()
+}
+
+func (f *zabsFilterFlags) registerZabsFilterFlags(s *pflag.FlagSet, verb string) {
+	// Note: the default value is defined in the .FlagValue methods
+	s.Var(&f.Filesystems, "fs", fmt.Sprintf("only %s holds on the specified filesystem [default: all filesystems] [comma-separated list of <dataset-pattern>:<ok|!> pairs]", verb))
+	s.Var(&f.Job, "job", fmt.Sprintf("only %s holds created by the specified job [default: any job]", verb))
+
+	variants := make([]string, 0, len(endpoint.AbstractionTypesAll))
+	for v := range endpoint.AbstractionTypesAll {
+		variants = append(variants, string(v))
+	}
+	variants = sort.StringSlice(variants)
+	variantsJoined := strings.Join(variants, "|")
+	s.Var(&f.Types, "type", fmt.Sprintf("only %s holds of the specified type [default: all] [comma-separated list of %s]", verb, variantsJoined))
+
+	s.Int64VarP(&f.Concurrency, "concurrency", "p", 1, "number of concurrently queried filesystems")
+}
+
+type JobIDFlag struct{ J *endpoint.JobID }
+
+func (f *JobIDFlag) Set(s string) error {
+	if len(s) == 0 {
+		*f = JobIDFlag{J: nil}
+		return nil
+	}
+
+	jobID, err := endpoint.MakeJobID(s)
+	if err != nil {
+		return err
+	}
+	*f = JobIDFlag{J: &jobID}
+	return nil
+}
+func (f JobIDFlag) Type() string               { return "job-ID" }
+func (f JobIDFlag) String() string             { return fmt.Sprint(f.J) }
+func (f JobIDFlag) FlagValue() *endpoint.JobID { return f.J }
+
+type AbstractionTypesFlag map[endpoint.AbstractionType]bool
+
+func (f *AbstractionTypesFlag) Set(s string) error {
+	ats, err := endpoint.AbstractionTypeSetFromStrings(strings.Split(s, ","))
+	if err != nil {
+		return err
+	}
+	*f = AbstractionTypesFlag(ats)
+	return nil
+}
+func (f AbstractionTypesFlag) Type() string { return "abstraction-type" }
+func (f AbstractionTypesFlag) String() string {
+	return endpoint.AbstractionTypeSet(f).String()
+}
+func (f AbstractionTypesFlag) FlagValue() map[endpoint.AbstractionType]bool {
+	if len(f) > 0 {
+		return f
+	}
+	return endpoint.AbstractionTypesAll
+}
+
+type FilesystemsFilterFlag struct {
+	F endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter
+}
+
+func (flag *FilesystemsFilterFlag) Set(s string) error {
+	mappings := strings.Split(s, ",")
+	if len(mappings) == 1 && !strings.Contains(mappings[0], ":") {
+		flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &mappings[0],
+		}
+		return nil
+	}
+
+	f := filters.NewDatasetMapFilter(len(mappings), true)
+	for _, m := range mappings {
+		thisMappingErr := fmt.Errorf("expecting comma-separated list of <dataset-pattern>:<ok|!> pairs, got %q", m)
+		lhsrhs := strings.SplitN(m, ":", 2)
+		if len(lhsrhs) != 2 {
+			return thisMappingErr
+		}
+		err := f.Add(lhsrhs[0], lhsrhs[1])
+		if err != nil {
+			return fmt.Errorf("%s: %s", thisMappingErr, err)
+		}
+	}
+	flag.F = endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+		Filter: f,
+	}
+	return nil
+}
+func (flag FilesystemsFilterFlag) Type() string { return "filesystem filter spec" }
+func (flag FilesystemsFilterFlag) String() string {
+	return fmt.Sprintf("%v", flag.F)
+}
+func (flag FilesystemsFilterFlag) FlagValue() endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter {
+	var z FilesystemsFilterFlag
+	if flag == z {
+		return endpoint.ListZFSHoldsAndBookmarksQueryFilesystemFilter{Filter: zfs.NoFilter()}
+	}
+	return flag.F
+}

client/zfsabstractions_create.go

@@ -0,0 +1,14 @@
+package client
+
+import "github.com/zrepl/zrepl/cli"
+
+var zabsCmdCreate = &cli.Subcommand{
+	Use:             "create",
+	NoRequireConfig: true,
+	Short:           `create zrepl ZFS abstractions (mostly useful for debugging & development, users should not need to use this command)`,
+	SetupSubcommands: func() []*cli.Subcommand {
+		return []*cli.Subcommand{
+			zabsCmdCreateStepHold,
+		}
+	},
+}

client/zfsabstractions_create_step_hold.go

@@ -0,0 +1,60 @@
+package client
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var zabsCreateStepHoldFlags struct {
+	target string
+	jobid  JobIDFlag
+}
+
+var zabsCmdCreateStepHold = &cli.Subcommand{
+	Use:             "step",
+	Run:             doZabsCreateStep,
+	NoRequireConfig: true,
+	Short:           `create a step hold or bookmark`,
+	SetupFlags: func(f *pflag.FlagSet) {
+		f.StringVarP(&zabsCreateStepHoldFlags.target, "target", "t", "", "snapshot to be held / bookmark to be held")
+		f.VarP(&zabsCreateStepHoldFlags.jobid, "jobid", "j", "jobid for which the hold is installed")
+	},
+}
+
+func doZabsCreateStep(sc *cli.Subcommand, args []string) error {
+	if len(args) > 0 {
+		return errors.New("subcommand takes no arguments")
+	}
+
+	f := &zabsCreateStepHoldFlags
+
+	fs, _, _, err := zfs.DecomposeVersionString(f.target)
+	if err != nil {
+		return errors.Wrapf(err, "%q invalid target", f.target)
+	}
+
+	if f.jobid.FlagValue() == nil {
+		return errors.Errorf("jobid must be set")
+	}
+
+	ctx := context.Background()
+
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, f.target)
+	if err != nil {
+		return errors.Wrapf(err, "get info about target %q", f.target)
+	}
+
+	step, err := endpoint.HoldStep(ctx, fs, v, *f.jobid.FlagValue())
+	if err != nil {
+		return errors.Wrap(err, "create step hold")
+	}
+	fmt.Println(step.String())
+	return nil
+}

client/zfsabstractions_list.go

@@ -0,0 +1,100 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+	"sync"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/util/chainlock"
+)
+
+var zabsListFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+}
+
+var zabsCmdList = &cli.Subcommand{
+	Use:             "list",
+	Short:           `list zrepl ZFS abstractions`,
+	Run:             doZabsList,
+	NoRequireConfig: true,
+	SetupFlags: func(f *pflag.FlagSet) {
+		zabsListFlags.Filter.registerZabsFilterFlags(f, "list")
+		f.BoolVar(&zabsListFlags.Json, "json", false, "emit JSON")
+	},
+}
+
+func doZabsList(sc *cli.Subcommand, args []string) error {
+	var err error
+	ctx := context.Background()
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsListFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, errors, err := endpoint.ListAbstractionsStreamed(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	var line chainlock.L
+	var wg sync.WaitGroup
+	defer wg.Wait()
+	wg.Add(1)
+
+	// print results
+	go func() {
+		defer wg.Done()
+		enc := json.NewEncoder(os.Stdout)
+		for a := range abstractions {
+			func() {
+				defer line.Lock().Unlock()
+				if zabsListFlags.Json {
+					enc.SetIndent("", "  ")
+					if err := enc.Encode(abstractions); err != nil {
+						panic(err)
+					}
+					fmt.Println()
+				} else {
+					fmt.Println(a)
+				}
+			}()
+		}
+	}()
+
+	// print errors to stderr
+	errorColor := color.New(color.FgRed)
+	var errorsSlice []endpoint.ListAbstractionsError
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range errors {
+			func() {
+				defer line.Lock().Unlock()
+				errorsSlice = append(errorsSlice, err)
+				errorColor.Fprintf(os.Stderr, "%s\n", err)
+			}()
+		}
+	}()
+	wg.Wait()
+	if len(errorsSlice) > 0 {
+		errorColor.Add(color.Bold).Fprintf(os.Stderr, "there were errors in listing the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+
+}

client/zfsabstractions_release.go

@@ -0,0 +1,145 @@
+package client
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"os"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	"github.com/spf13/pflag"
+
+	"github.com/zrepl/zrepl/cli"
+	"github.com/zrepl/zrepl/endpoint"
+)
+
+// shared between release-all and release-step
+var zabsReleaseFlags struct {
+	Filter zabsFilterFlags
+	Json   bool
+	DryRun bool
+}
+
+func registerZabsReleaseFlags(s *pflag.FlagSet) {
+	zabsReleaseFlags.Filter.registerZabsFilterFlags(s, "release")
+	s.BoolVar(&zabsReleaseFlags.Json, "json", false, "emit json instead of pretty-printed")
+	s.BoolVar(&zabsReleaseFlags.DryRun, "dry-run", false, "do a dry-run")
+}
+
+var zabsCmdReleaseAll = &cli.Subcommand{
+	Use:             "release-all",
+	Run:             doZabsReleaseAll,
+	NoRequireConfig: true,
+	Short:           `(DANGEROUS) release ALL zrepl ZFS abstractions (mostly useful for uninstalling zrepl completely or for "de-zrepl-ing" a filesystem)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+var zabsCmdReleaseStale = &cli.Subcommand{
+	Use:             "release-stale",
+	Run:             doZabsReleaseStale,
+	NoRequireConfig: true,
+	Short:           `release stale zrepl ZFS abstractions (useful if zrepl has a bug and does not do it by itself)`,
+	SetupFlags:      registerZabsReleaseFlags,
+}
+
+func doZabsReleaseAll(sc *cli.Subcommand, args []string) error {
+	var err error
+	ctx := context.Background()
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	abstractions, listErrors, err := endpoint.ListAbstractions(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+	if len(listErrors) > 0 {
+		color.New(color.FgRed).Fprintf(os.Stderr, "there were errors in listing the abstractions:\n%s\n", listErrors)
+		// proceed anyways with rest of abstractions
+	}
+
+	return doZabsRelease_Common(ctx, abstractions)
+}
+
+func doZabsReleaseStale(sc *cli.Subcommand, args []string) error {
+
+	var err error
+	ctx := context.Background()
+
+	if len(args) > 0 {
+		return errors.New("this subcommand takes no positional arguments")
+	}
+
+	q, err := zabsReleaseFlags.Filter.Query()
+	if err != nil {
+		return errors.Wrap(err, "invalid filter specification on command line")
+	}
+
+	stalenessInfo, err := endpoint.ListStale(ctx, q)
+	if err != nil {
+		return err // context clear by invocation of command
+	}
+
+	return doZabsRelease_Common(ctx, stalenessInfo.Stale)
+}
+
+func doZabsRelease_Common(ctx context.Context, destroy []endpoint.Abstraction) error {
+
+	if zabsReleaseFlags.DryRun {
+		if zabsReleaseFlags.Json {
+			m, err := json.MarshalIndent(destroy, "", "  ")
+			if err != nil {
+				panic(err)
+			}
+			if _, err := os.Stdout.Write(m); err != nil {
+				panic(err)
+			}
+			fmt.Println()
+		} else {
+			for _, a := range destroy {
+				fmt.Printf("would destroy %s\n", a)
+			}
+		}
+		return nil
+	}
+
+	outcome := endpoint.BatchDestroy(ctx, destroy)
+	hadErr := false
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+	colorErr := color.New(color.FgRed)
+	printfSuccess := color.New(color.FgGreen).FprintfFunc()
+	printfSection := color.New(color.Bold).FprintfFunc()
+
+	for res := range outcome {
+		hadErr = hadErr || res.DestroyErr != nil
+		if zabsReleaseFlags.Json {
+			err := enc.Encode(res)
+			if err != nil {
+				colorErr.Fprintf(os.Stderr, "cannot marshal there were errors in destroying the abstractions")
+			}
+		} else {
+			printfSection(os.Stdout, "destroy %s ...", res.Abstraction)
+			if res.DestroyErr != nil {
+				colorErr.Fprintf(os.Stdout, " failed:\n%s\n", res.DestroyErr)
+			} else {
+				printfSuccess(os.Stdout, " OK\n")
+			}
+		}
+	}
+
+	if hadErr {
+		colorErr.Add(color.Bold).Fprintf(os.Stderr, "there were errors in destroying the abstractions")
+		return fmt.Errorf("")
+	} else {
+		return nil
+	}
+}

config/config.go

@@ -75,16 +75,42 @@ type SnapJob struct {
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
 }
 
+type SendOptions struct {
+	Encrypted bool `yaml:"encrypted"`
+}
+
+var _ yaml.Defaulter = (*SendOptions)(nil)
+
+func (l *SendOptions) SetDefault() {
+	*l = SendOptions{Encrypted: false}
+}
+
+type RecvOptions struct {
+	// Note: we cannot enforce encrypted recv as the ZFS cli doesn't provide a mechanism for it
+	// Encrypted bool `yaml:"may_encrypted"`
+
+	// Future:
+	// Reencrypt bool `yaml:"reencrypt"`
+}
+
+var _ yaml.Defaulter = (*RecvOptions)(nil)
+
+func (l *RecvOptions) SetDefault() {
+	*l = RecvOptions{}
+}
+
 type PushJob struct {
 	ActiveJob    `yaml:",inline"`
 	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,fromdefaults,optional"`
 }
 
 type PullJob struct {
 	ActiveJob `yaml:",inline"`
 	RootFS    string                   `yaml:"root_fs"`
 	Interval  PositiveDurationOrManual `yaml:"interval"`
+	Recv      *RecvOptions             `yaml:"recv,fromdefaults,optional"`
 }
 
 type PositiveDurationOrManual struct {
@@ -120,13 +146,15 @@ func (i *PositiveDurationOrManual) UnmarshalYAML(u func(interface{}, bool) error
 
 type SinkJob struct {
 	PassiveJob `yaml:",inline"`
-	RootFS     string `yaml:"root_fs"`
+	RootFS     string       `yaml:"root_fs"`
+	Recv       *RecvOptions `yaml:"recv,optional,fromdefaults"`
 }
 
 type SourceJob struct {
 	PassiveJob   `yaml:",inline"`
 	Snapshotting SnapshottingEnum  `yaml:"snapshotting"`
 	Filesystems  FilesystemsFilter `yaml:"filesystems"`
+	Send         *SendOptions      `yaml:"send,optional,fromdefaults"`
 }
 
 type FilesystemsFilter map[string]bool
@@ -245,14 +273,16 @@ type ServeCommon struct {
 }
 
 type TCPServe struct {
-	ServeCommon `yaml:",inline"`
-	Listen      string            `yaml:"listen,hostport"`
-	Clients     map[string]string `yaml:"clients"`
+	ServeCommon    `yaml:",inline"`
+	Listen         string            `yaml:"listen,hostport"`
+	ListenFreeBind bool              `yaml:"listen_freebind,default=false"`
+	Clients        map[string]string `yaml:"clients"`
 }
 
 type TLSServe struct {
 	ServeCommon      `yaml:",inline"`
 	Listen           string        `yaml:"listen,hostport"`
+	ListenFreeBind   bool          `yaml:"listen_freebind,default=false"`
 	Ca               string        `yaml:"ca"`
 	Cert             string        `yaml:"cert"`
 	Key              string        `yaml:"key"`
@@ -331,8 +361,9 @@ type MonitoringEnum struct {
 }
 
 type PrometheusMonitoring struct {
-	Type   string `yaml:"type"`
-	Listen string `yaml:"listen,hostport"`
+	Type           string `yaml:"type"`
+	Listen         string `yaml:"listen,hostport"`
+	ListenFreeBind bool   `yaml:"listen_freebind,default=false"`
 }
 
 type SyslogFacility syslog.Priority
@@ -589,7 +620,7 @@ func ParseConfigBytes(bytes []byte) (*Config, error) {
 
 var durationStringRegex *regexp.Regexp = regexp.MustCompile(`^\s*(\d+)\s*(s|m|h|d|w)\s*$`)
 
-func parsePostitiveDuration(e string) (d time.Duration, err error) {
+func parsePositiveDuration(e string) (d time.Duration, err error) {
 	comps := durationStringRegex.FindStringSubmatch(e)
 	if len(comps) != 3 {
 		err = fmt.Errorf("does not match regex: %s %#v", e, comps)

config/config_global_test.go

@@ -21,7 +21,7 @@ jobs:
     clients: {
       "10.0.0.1":"foo"
     }
-  root_fs: zoot/foo
+  root_fs: zroot/foo
 `
 	_, err := ParseConfigBytes([]byte(jobdef))
 	require.NoError(t, err)

config/config_jobid_test.go

@@ -0,0 +1 @@
+package config

config/config_send_test.go

@@ -0,0 +1,74 @@
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSendOptions(t *testing.T) {
+	tmpl := `
+jobs:
+- name: foo
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  %s
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	encrypted_false := `
+  send:
+    encrypted: false
+`
+
+	encrypted_true := `
+  send:
+    encrypted: true
+`
+
+	encrypted_unspecified := `
+  send: {}
+`
+
+	send_not_specified := `
+`
+
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+	var c *Config
+
+	t.Run("encrypted_false", func(t *testing.T) {
+		c = testValidConfig(t, fill(encrypted_false))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, false, encrypted)
+	})
+	t.Run("encrypted_true", func(t *testing.T) {
+		c = testValidConfig(t, fill(encrypted_true))
+		encrypted := c.Jobs[0].Ret.(*PushJob).Send.Encrypted
+		assert.Equal(t, true, encrypted)
+	})
+
+	t.Run("encrypted_unspecified", func(t *testing.T) {
+		c, err := testConfig(t, fill(encrypted_unspecified))
+		assert.Error(t, err)
+		assert.Nil(t, c)
+	})
+
+	t.Run("send_not_specified", func(t *testing.T) {
+		c, err := testConfig(t, fill(send_not_specified))
+		assert.NoError(t, err)
+		assert.NotNil(t, c)
+	})
+
+}

config/config_test.go

@@ -42,7 +42,7 @@ func TestSampleConfigsAreParsedWithoutErrors(t *testing.T) {
 
 }
 
-// template must be a template/text template with a single '{{ . }}' as placehodler for val
+// template must be a template/text template with a single '{{ . }}' as placeholder for val
 //nolint[:deadcode,unused]
 func testValidConfigTemplate(t *testing.T, tmpl string, val string) *Config {
 	tmp, err := template.New("master").Parse(tmpl)

config/retentiongrid.go

@@ -64,7 +64,7 @@ func parseRetentionGridIntervalString(e string) (intervals []RetentionInterval,
 		return nil, fmt.Errorf("contains factor <= 0")
 	}
 
-	duration, err := parsePostitiveDuration(comps[2])
+	duration, err := parsePositiveDuration(comps[2])
 	if err != nil {
 		return nil, err
 	}

config/samples/pull_ssh.yml

@@ -9,7 +9,7 @@ jobs:
     port: 22
     identity_file: /etc/zrepl/ssh/identity
     options: # optional, default [], `-o` arguments passed to ssh
-    - "Compression=on"
+    - "Compression=yes"
   root_fs: "pool2/backup_servers"
   interval: 10m
   pruning:

config/samples/push.yml

@@ -10,6 +10,8 @@ jobs:
       address: "backup-server.foo.bar:8888"
     snapshotting:
       type: manual
+    send:
+      encrypted: false
     pruning:
       keep_sender:
         - type: not_replicated

config/samples/source.yml

@@ -5,7 +5,11 @@ jobs:
     type: tcp
     listen: "0.0.0.0:8888"
     clients: {
-      "192.168.122.123" : "client1"
+      "192.168.122.123" :               "mysql01",
+      "192.168.122.42" :                "mx01",
+      "2001:0db8:85a3::8a2e:0370:7334": "gateway",
+      "10.23.42.0/24":       "cluster-*",
+      "fde4:8dba:82e1::/64": "san-*",
     }
   filesystems: {
     "<": true,

daemon/control.go

@@ -15,10 +15,12 @@ import (
 
 	"github.com/zrepl/zrepl/daemon/job"
 	"github.com/zrepl/zrepl/daemon/nethelpers"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/version"
 	"github.com/zrepl/zrepl/zfs"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
 type controlJob struct {
@@ -44,6 +46,8 @@ func (j *controlJob) Status() *job.Status { return &job.Status{Type: job.TypeInt
 
 func (j *controlJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
 
+func (j *controlJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 var promControl struct {
 	requestBegin    *prometheus.CounterVec
 	requestFinished *prometheus.HistogramVec
@@ -61,7 +65,7 @@ func (j *controlJob) RegisterMetrics(registerer prometheus.Registerer) {
 		Namespace: "zrepl",
 		Subsystem: "control",
 		Name:      "request_finished",
-		Help:      "time it took a request to finih",
+		Help:      "time it took a request to finish",
 		Buckets:   []float64{1e-6, 10e-6, 100e-6, 500e-6, 1e-3, 10e-3, 100e-3, 200e-3, 400e-3, 800e-3, 1, 10, 20},
 	}, []string{"endpoint"})
 	registerer.MustRegister(promControl.requestBegin)
@@ -114,7 +118,9 @@ func (j *controlJob) Run(ctx context.Context) {
 	mux.Handle(ControlJobEndpointStatus,
 		// don't log requests to status endpoint, too spammy
 		jsonResponder{log, func() (interface{}, error) {
-			s := j.jobs.status()
+			jobs := j.jobs.status()
+			globalZFS := zfscmd.GetReport()
+			s := Status{Jobs: jobs, Global: GlobalStatus{ZFSCmds: globalZFS}}
 			return s, nil
 		}})
 
@@ -247,7 +253,7 @@ func (j jsonRequestResponder) ServeHTTP(w http.ResponseWriter, r *http.Request)
 	var buf bytes.Buffer
 	encodeErr := json.NewEncoder(&buf).Encode(res)
 	if encodeErr != nil {
-		j.log.WithError(producerErr).Error("control handler json marhsal error")
+		j.log.WithError(producerErr).Error("control handler json marshal error")
 		w.WriteHeader(http.StatusInternalServerError)
 		_, err := io.WriteString(w, encodeErr.Error())
 		logIoErr(err)

daemon/daemon.go

@@ -20,6 +20,7 @@ import (
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/version"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
 func Run(conf *config.Config) error {
@@ -76,11 +77,14 @@ func Run(conf *config.Config) error {
 			return errors.Errorf("unknown monitoring job #%d (type %T)", i, v)
 		}
 		if err != nil {
-			return errors.Wrapf(err, "cannot build monitorin gjob #%d", i)
+			return errors.Wrapf(err, "cannot build monitoring job #%d", i)
 		}
 		jobs.start(ctx, job, true)
 	}
 
+	// register global (=non job-local) metrics
+	zfscmd.RegisterMetrics(prometheus.DefaultRegisterer)
+
 	log.Info("starting daemon")
 
 	// start regular jobs
@@ -128,6 +132,15 @@ func (s *jobs) wait() <-chan struct{} {
 	return ch
 }
 
+type Status struct {
+	Jobs   map[string]*job.Status
+	Global GlobalStatus
+}
+
+type GlobalStatus struct {
+	ZFSCmds *zfscmd.Report
+}
+
 func (s *jobs) status() map[string]*job.Status {
 	s.m.RLock()
 	defer s.m.RUnlock()
@@ -207,6 +220,7 @@ func (s *jobs) start(ctx context.Context, j job.Job, internal bool) {
 
 	s.jobs[jobName] = j
 	ctx = job.WithLogger(ctx, jobLog)
+	ctx = zfscmd.WithJobID(ctx, j.Name())
 	ctx, wakeup := wakeup.Context(ctx)
 	ctx, resetFunc := reset.Context(ctx)
 	s.wakeups[jobName] = wakeup

daemon/filters/fsmapfilter.go

@@ -161,7 +161,7 @@ func (m DatasetMapFilter) Filter(p *zfs.DatasetPath) (pass bool, err error) {
 }
 
 // Construct a new filter-only DatasetMapFilter from a mapping
-// The new filter allows excactly those paths that were not forbidden by the mapping.
+// The new filter allows exactly those paths that were not forbidden by the mapping.
 func (m DatasetMapFilter) InvertedFilter() (inv *DatasetMapFilter, err error) {
 
 	if m.filterMode {

daemon/filters/fsmapfilter_test.go

@@ -1,11 +1,11 @@
 package filters
 
 import (
+	"testing"
+
 	"github.com/zrepl/zrepl/zfs"
 )
 
-import "testing"
-
 func TestDatasetMapFilter(t *testing.T) {
 
 	type testCase struct {

daemon/filters/fsvfilter.go

@@ -1,41 +0,0 @@
-package filters
-
-import (
-	"strings"
-
-	"github.com/zrepl/zrepl/zfs"
-)
-
-type AnyFSVFilter struct{}
-
-func NewAnyFSVFilter() AnyFSVFilter {
-	return AnyFSVFilter{}
-}
-
-var _ zfs.FilesystemVersionFilter = AnyFSVFilter{}
-
-func (AnyFSVFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	return true, nil
-}
-
-type PrefixFilter struct {
-	prefix    string
-	fstype    zfs.VersionType
-	fstypeSet bool // optionals anyone?
-}
-
-var _ zfs.FilesystemVersionFilter = &PrefixFilter{}
-
-func NewPrefixFilter(prefix string) *PrefixFilter {
-	return &PrefixFilter{prefix: prefix}
-}
-
-func NewTypedPrefixFilter(prefix string, versionType zfs.VersionType) *PrefixFilter {
-	return &PrefixFilter{prefix, versionType, true}
-}
-
-func (f *PrefixFilter) Filter(t zfs.VersionType, name string) (accept bool, err error) {
-	fstypeMatches := (!f.fstypeSet || t == f.fstype)
-	prefixMatches := strings.HasPrefix(name, f.prefix)
-	return fstypeMatches && prefixMatches, nil
-}

daemon/job/active.go

@@ -2,6 +2,7 @@ package job
 
 import (
 	"context"
+	"fmt"
 	"sync"
 	"time"
 
@@ -28,7 +29,7 @@ import (
 
 type ActiveSide struct {
 	mode      activeMode
-	name      string
+	name      endpoint.JobID
 	connecter transport.Connecter
 
 	prunerFactory *pruner.PrunerFactory
@@ -82,17 +83,19 @@ type activeMode interface {
 	DisconnectEndpoints()
 	SenderReceiver() (logic.Sender, logic.Receiver)
 	Type() Type
+	PlannerPolicy() logic.PlannerPolicy
 	RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{})
 	SnapperReport() *snapper.Report
 	ResetConnectBackoff()
 }
 
 type modePush struct {
-	setupMtx sync.Mutex
-	sender   *endpoint.Sender
-	receiver *rpc.Client
-	fsfilter endpoint.FSFilter
-	snapper  *snapper.PeriodicOrManual
+	setupMtx      sync.Mutex
+	sender        *endpoint.Sender
+	receiver      *rpc.Client
+	senderConfig  *endpoint.SenderConfig
+	plannerPolicy *logic.PlannerPolicy
+	snapper       *snapper.PeriodicOrManual
 }
 
 func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
@@ -101,7 +104,7 @@ func (m *modePush) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Con
 	if m.receiver != nil || m.sender != nil {
 		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
 	}
-	m.sender = endpoint.NewSender(m.fsfilter)
+	m.sender = endpoint.NewSender(*m.senderConfig)
 	m.receiver = rpc.NewClient(connecter, loggers)
 }
 
@@ -121,6 +124,8 @@ func (m *modePush) SenderReceiver() (logic.Sender, logic.Receiver) {
 
 func (m *modePush) Type() Type { return TypePush }
 
+func (m *modePush) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
 func (m *modePush) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
 	m.snapper.Run(ctx, wakeUpCommon)
 }
@@ -137,13 +142,22 @@ func (m *modePush) ResetConnectBackoff() {
 	}
 }
 
-func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error) {
+func modePushFromConfig(g *config.Global, in *config.PushJob, jobID endpoint.JobID) (*modePush, error) {
 	m := &modePush{}
+
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+
+	m.senderConfig = &endpoint.SenderConfig{
+		FSF:     fsf,
+		Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
+		JobID:   jobID,
+	}
+	m.plannerPolicy = &logic.PlannerPolicy{
+		EncryptedSend: logic.TriFromBool(in.Send.Encrypted),
 	}
-	m.fsfilter = fsf
 
 	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
@@ -153,11 +167,13 @@ func modePushFromConfig(g *config.Global, in *config.PushJob) (*modePush, error)
 }
 
 type modePull struct {
-	setupMtx sync.Mutex
-	receiver *endpoint.Receiver
-	sender   *rpc.Client
-	rootFS   *zfs.DatasetPath
-	interval config.PositiveDurationOrManual
+	setupMtx       sync.Mutex
+	receiver       *endpoint.Receiver
+	receiverConfig endpoint.ReceiverConfig
+	sender         *rpc.Client
+	rootFS         *zfs.DatasetPath
+	plannerPolicy  *logic.PlannerPolicy
+	interval       config.PositiveDurationOrManual
 }
 
 func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Connecter) {
@@ -166,7 +182,7 @@ func (m *modePull) ConnectEndpoints(loggers rpc.Loggers, connecter transport.Con
 	if m.receiver != nil || m.sender != nil {
 		panic("inconsistent use of ConnectEndpoints and DisconnectEndpoints")
 	}
-	m.receiver = endpoint.NewReceiver(m.rootFS, false)
+	m.receiver = endpoint.NewReceiver(m.receiverConfig)
 	m.sender = rpc.NewClient(connecter, loggers)
 }
 
@@ -186,6 +202,8 @@ func (m *modePull) SenderReceiver() (logic.Sender, logic.Receiver) {
 
 func (*modePull) Type() Type { return TypePull }
 
+func (m *modePull) PlannerPolicy() logic.PlannerPolicy { return *m.plannerPolicy }
+
 func (m *modePull) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
 	if m.interval.Manual {
 		GetLogger(ctx).Info("manual pull configured, periodic pull disabled")
@@ -223,7 +241,7 @@ func (m *modePull) ResetConnectBackoff() {
 	}
 }
 
-func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err error) {
+func modePullFromConfig(g *config.Global, in *config.PullJob, jobID endpoint.JobID) (m *modePull, err error) {
 	m = &modePull{}
 	m.interval = in.Interval
 
@@ -235,26 +253,56 @@ func modePullFromConfig(g *config.Global, in *config.PullJob) (m *modePull, err
 		return nil, errors.New("RootFS must not be empty") // duplicates error check of receiver
 	}
 
+	m.plannerPolicy = &logic.PlannerPolicy{
+		EncryptedSend: logic.DontCare,
+	}
+
+	m.receiverConfig = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: m.rootFS,
+		AppendClientIdentity:       false, // !
+		UpdateLastReceivedHold:     true,
+	}
+	if err := m.receiverConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver config")
+	}
+
 	return m, nil
 }
 
-func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *ActiveSide, err error) {
+func activeSide(g *config.Global, in *config.ActiveJob, configJob interface{}) (j *ActiveSide, err error) {
+
+	j = &ActiveSide{}
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.PushJob:
+		j.mode, err = modePushFromConfig(g, v, j.name) // shadow
+	case *config.PullJob:
+		j.mode, err = modePullFromConfig(g, v, j.name) // shadow
+	default:
+		panic(fmt.Sprintf("implementation error: unknown job type %T", v))
+	}
+	if err != nil {
+		return nil, err // no wrapping required
+	}
 
-	j = &ActiveSide{mode: mode}
-	j.name = in.Name
 	j.promRepStateSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "replication",
 		Name:        "state_time",
 		Help:        "seconds spent during replication",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"state"})
 	j.promBytesReplicated = prometheus.NewCounterVec(prometheus.CounterOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "replication",
 		Name:        "bytes_replicated",
 		Help:        "number of bytes replicated from sender to receiver per filesystem",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"filesystem"})
 
 	j.connecter, err = fromconfig.ConnecterFromConfig(g, in.Connect)
@@ -267,7 +315,7 @@ func activeSide(g *config.Global, in *config.ActiveJob, mode activeMode) (j *Act
 		Subsystem:   "pruning",
 		Name:        "time",
 		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"prune_side"})
 	j.prunerFactory, err = pruner.NewPrunerFactory(in.Pruning, j.promPruneSecs)
 	if err != nil {
@@ -283,7 +331,7 @@ func (j *ActiveSide) RegisterMetrics(registerer prometheus.Registerer) {
 	registerer.MustRegister(j.promBytesReplicated)
 }
 
-func (j *ActiveSide) Name() string { return j.name }
+func (j *ActiveSide) Name() string { return j.name.String() }
 
 type ActiveSideStatus struct {
 	Replication                    *report.Report
@@ -318,6 +366,15 @@ func (j *ActiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
 	return pull.rootFS.Copy(), true
 }
 
+func (j *ActiveSide) SenderConfig() *endpoint.SenderConfig {
+	push, ok := j.mode.(*modePush)
+	if !ok {
+		_ = j.mode.(*modePull) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return push.senderConfig
+}
+
 func (j *ActiveSide) Run(ctx context.Context) {
 	log := GetLogger(ctx)
 	ctx = logging.WithSubsystemLoggers(ctx, log)
@@ -383,7 +440,7 @@ func (j *ActiveSide) do(ctx context.Context) {
 			*tasks = activeSideTasks{}
 			tasks.replicationCancel = repCancel
 			tasks.replicationReport, repWait = replication.Do(
-				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver),
+				ctx, logic.NewPlanner(j.promRepStateSecs, j.promBytesReplicated, sender, receiver, j.mode.PlannerPolicy()),
 			)
 			tasks.state = ActiveSideReplicating
 		})

daemon/job/build_jobs.go

@@ -48,20 +48,12 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
 	// FIXME prettify this
 	switch v := in.Ret.(type) {
 	case *config.SinkJob:
-		m, err := modeSinkFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.SourceJob:
-		m, err := modeSourceFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = passiveSideFromConfig(c, &v.PassiveJob, m)
+		j, err = passiveSideFromConfig(c, &v.PassiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
@@ -71,20 +63,12 @@ func buildJob(c *config.Global, in config.JobEnum) (j Job, err error) {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.PushJob:
-		m, err := modePushFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
+		j, err = activeSide(c, &v.ActiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
 	case *config.PullJob:
-		m, err := modePullFromConfig(c, v)
-		if err != nil {
-			return cannotBuildJob(err, v.Name)
-		}
-		j, err = activeSide(c, &v.ActiveJob, m)
+		j, err = activeSide(c, &v.ActiveJob, v)
 		if err != nil {
 			return cannotBuildJob(err, v.Name)
 		}
@@ -104,6 +88,11 @@ func validateReceivingSidesDoNotOverlap(receivingRootFSs []string) error {
 	sort.Slice(rfss, func(i, j int) bool {
 		return strings.Compare(rfss[i], rfss[j]) == -1
 	})
+	// add tailing slash because of hierarchy-simulation
+	// rootfs/ is not root of rootfs2/
+	for i := 0; i < len(rfss); i++ {
+		rfss[i] += "/"
+	}
 	// idea:
 	//   no path in rfss must be prefix of another
 	//

daemon/job/build_jobs_test.go

@@ -1,9 +1,13 @@
 package job
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/config"
 )
 
 func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
@@ -18,6 +22,7 @@ func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
 		{false, []string{"a"}},
 		{false, []string{"some/path"}},
 		{false, []string{"zroot/sink1", "zroot/sink2", "zroot/sink3"}},
+		{false, []string{"zroot/foo", "zroot/foobar"}},
 		{true, []string{"zroot/b", "zroot/b"}},
 		{true, []string{"zroot/foo", "zroot/foo/bar", "zroot/baz"}},
 		{false, []string{"a/x", "b/x"}},
@@ -40,3 +45,66 @@ func TestValidateReceivingSidesDoNotOverlap(t *testing.T) {
 		}
 	}
 }
+
+func TestJobIDErrorHandling(t *testing.T) {
+	tmpl := `
+jobs:
+- name: %s
+  type: push
+  connect:
+    type: local
+    listener_name: foo
+    client_identity: bar
+  filesystems: {"<": true}
+  snapshotting:
+    type: manual
+  pruning:
+    keep_sender:
+    - type: last_n
+      count: 10
+    keep_receiver:
+    - type: last_n
+      count: 10
+`
+	fill := func(s string) string { return fmt.Sprintf(tmpl, s) }
+
+	type Case struct {
+		jobName string
+		valid   bool
+	}
+	cases := []Case{
+		{"validjobname", true},
+		{"valid with spaces", true},
+		{"invalid\twith\ttabs", false},
+		{"invalid#withdelimiter", false},
+		{"invalid@withdelimiter", false},
+		{"withnewline\\nmiddle", false},
+		{"withnewline\\n", false},
+		{"withslash/", false},
+		{"withslash/inthemiddle", false},
+		{"/", false},
+	}
+
+	for i := range cases {
+		t.Run(cases[i].jobName, func(t *testing.T) {
+			c := cases[i]
+
+			conf, err := config.ParseConfigBytes([]byte(fill(c.jobName)))
+			require.NoError(t, err, "not expecting yaml-config to know about job ids")
+			require.NotNil(t, conf)
+			jobs, err := JobsFromConfig(conf)
+
+			if c.valid {
+				assert.NoError(t, err)
+				require.Len(t, jobs, 1)
+				assert.Equal(t, c.jobName, jobs[0].Name())
+			} else {
+				t.Logf("error: %s", err)
+				assert.Error(t, err)
+				assert.Nil(t, jobs)
+			}
+
+		})
+	}
+
+}

daemon/job/job.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/zfs"
 )
@@ -38,6 +39,7 @@ type Job interface {
 	// Jobs that return a subtree of the dataset hierarchy
 	// must return the root of that subtree as rfs and ok = true
 	OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
+	SenderConfig() *endpoint.SenderConfig
 }
 
 type Type string

daemon/job/passive.go

@@ -20,7 +20,7 @@ import (
 
 type PassiveSide struct {
 	mode   passiveMode
-	name   string
+	name   endpoint.JobID
 	listen transport.AuthenticatedListenerFactory
 }
 
@@ -32,43 +32,56 @@ type passiveMode interface {
 }
 
 type modeSink struct {
-	rootDataset *zfs.DatasetPath
+	receiverConfig endpoint.ReceiverConfig
 }
 
 func (m *modeSink) Type() Type { return TypeSink }
 
 func (m *modeSink) Handler() rpc.Handler {
-	return endpoint.NewReceiver(m.rootDataset, true)
+	return endpoint.NewReceiver(m.receiverConfig)
 }
 
 func (m *modeSink) RunPeriodic(_ context.Context)  {}
 func (m *modeSink) SnapperReport() *snapper.Report { return nil }
 
-func modeSinkFromConfig(g *config.Global, in *config.SinkJob) (m *modeSink, err error) {
+func modeSinkFromConfig(g *config.Global, in *config.SinkJob, jobID endpoint.JobID) (m *modeSink, err error) {
 	m = &modeSink{}
-	m.rootDataset, err = zfs.NewDatasetPath(in.RootFS)
+
+	rootDataset, err := zfs.NewDatasetPath(in.RootFS)
 	if err != nil {
 		return nil, errors.New("root dataset is not a valid zfs filesystem path")
 	}
-	if m.rootDataset.Length() <= 0 {
-		return nil, errors.New("root dataset must not be empty") // duplicates error check of receiver
+
+	m.receiverConfig = endpoint.ReceiverConfig{
+		JobID:                      jobID,
+		RootWithoutClientComponent: rootDataset,
+		AppendClientIdentity:       true, // !
+		UpdateLastReceivedHold:     true,
 	}
+	if err := m.receiverConfig.Validate(); err != nil {
+		return nil, errors.Wrap(err, "cannot build receiver config")
+	}
+
 	return m, nil
 }
 
 type modeSource struct {
-	fsfilter zfs.DatasetFilter
-	snapper  *snapper.PeriodicOrManual
+	senderConfig *endpoint.SenderConfig
+	snapper      *snapper.PeriodicOrManual
 }
 
-func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource, err error) {
+func modeSourceFromConfig(g *config.Global, in *config.SourceJob, jobID endpoint.JobID) (m *modeSource, err error) {
 	// FIXME exact dedup of modePush
 	m = &modeSource{}
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
+	}
+	m.senderConfig = &endpoint.SenderConfig{
+		FSF:     fsf,
+		Encrypt: &zfs.NilBool{B: in.Send.Encrypted},
+		JobID:   jobID,
 	}
-	m.fsfilter = fsf
 
 	if m.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
@@ -80,7 +93,7 @@ func modeSourceFromConfig(g *config.Global, in *config.SourceJob) (m *modeSource
 func (m *modeSource) Type() Type { return TypeSource }
 
 func (m *modeSource) Handler() rpc.Handler {
-	return endpoint.NewSender(m.fsfilter)
+	return endpoint.NewSender(*m.senderConfig)
 }
 
 func (m *modeSource) RunPeriodic(ctx context.Context) {
@@ -91,9 +104,25 @@ func (m *modeSource) SnapperReport() *snapper.Report {
 	return m.snapper.Report()
 }
 
-func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passiveMode) (s *PassiveSide, err error) {
+func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, configJob interface{}) (s *PassiveSide, err error) {
+
+	s = &PassiveSide{}
+
+	s.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
+
+	switch v := configJob.(type) {
+	case *config.SinkJob:
+		s.mode, err = modeSinkFromConfig(g, v, s.name) // shadow
+	case *config.SourceJob:
+		s.mode, err = modeSourceFromConfig(g, v, s.name) // shadow
+	}
+	if err != nil {
+		return nil, err // no wrapping necessary
+	}
 
-	s = &PassiveSide{mode: mode, name: in.Name}
 	if s.listen, err = fromconfig.ListenerFactoryFromConfig(g, in.Serve); err != nil {
 		return nil, errors.Wrap(err, "cannot build listener factory")
 	}
@@ -101,7 +130,7 @@ func passiveSideFromConfig(g *config.Global, in *config.PassiveJob, mode passive
 	return s, nil
 }
 
-func (j *PassiveSide) Name() string { return j.name }
+func (j *PassiveSide) Name() string { return j.name.String() }
 
 type PassiveStatus struct {
 	Snapper *snapper.Report
@@ -120,7 +149,16 @@ func (j *PassiveSide) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool)
 		_ = j.mode.(*modeSource) // make sure we didn't introduce a new job type
 		return nil, false
 	}
-	return sink.rootDataset.Copy(), true
+	return sink.receiverConfig.RootWithoutClientComponent.Copy(), true
+}
+
+func (j *PassiveSide) SenderConfig() *endpoint.SenderConfig {
+	source, ok := j.mode.(*modeSource)
+	if !ok {
+		_ = j.mode.(*modeSink) // make sure we didn't introduce a new job type
+		return nil
+	}
+	return source.senderConfig
 }
 
 func (*PassiveSide) RegisterMetrics(registerer prometheus.Registerer) {}

daemon/job/snapjob.go

@@ -2,7 +2,6 @@ package job
 
 import (
 	"context"
-	"fmt"
 	"sort"
 
 	"github.com/pkg/errors"
@@ -20,7 +19,7 @@ import (
 )
 
 type SnapJob struct {
-	name     string
+	name     endpoint.JobID
 	fsfilter zfs.DatasetFilter
 	snapper  *snapper.PeriodicOrManual
 
@@ -31,7 +30,7 @@ type SnapJob struct {
 	pruner *pruner.Pruner
 }
 
-func (j *SnapJob) Name() string { return j.name }
+func (j *SnapJob) Name() string { return j.name.String() }
 
 func (j *SnapJob) Type() Type { return TypeSnap }
 
@@ -39,20 +38,23 @@ func snapJobFromConfig(g *config.Global, in *config.SnapJob) (j *SnapJob, err er
 	j = &SnapJob{}
 	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
 	if err != nil {
-		return nil, errors.Wrap(err, "cannnot build filesystem filter")
+		return nil, errors.Wrap(err, "cannot build filesystem filter")
 	}
 	j.fsfilter = fsf
 
 	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
 		return nil, errors.Wrap(err, "cannot build snapper")
 	}
-	j.name = in.Name
+	j.name, err = endpoint.MakeJobID(in.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid job name")
+	}
 	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 		Namespace:   "zrepl",
 		Subsystem:   "pruning",
 		Name:        "time",
 		Help:        "seconds spent in pruner",
-		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
+		ConstLabels: prometheus.Labels{"zrepl_job": j.name.String()},
 	}, []string{"prune_side"})
 	j.prunerFactory, err = pruner.NewLocalPrunerFactory(in.Pruning, j.promPruneSecs)
 	if err != nil {
@@ -84,6 +86,8 @@ func (j *SnapJob) OwnedDatasetSubtreeRoot() (rfs *zfs.DatasetPath, ok bool) {
 	return nil, false
 }
 
+func (j *SnapJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 func (j *SnapJob) Run(ctx context.Context) {
 	log := GetLogger(ctx)
 	ctx = logging.WithSubsystemLoggers(ctx, log)
@@ -133,9 +137,6 @@ type alwaysUpToDateReplicationCursorHistory struct {
 var _ pruner.History = (*alwaysUpToDateReplicationCursorHistory)(nil)
 
 func (h alwaysUpToDateReplicationCursorHistory) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error) {
-	if req.GetGet() == nil {
-		return nil, fmt.Errorf("unsupported ReplicationCursor request: SnapJob only supports GETting a (faked) cursor")
-	}
 	fsvReq := &pdu.ListFilesystemVersionsReq{
 		Filesystem: req.GetFilesystem(),
 	}
@@ -162,7 +163,12 @@ func (h alwaysUpToDateReplicationCursorHistory) ListFilesystems(ctx context.Cont
 func (j *SnapJob) doPrune(ctx context.Context) {
 	log := GetLogger(ctx)
 	ctx = logging.WithSubsystemLoggers(ctx, log)
-	sender := endpoint.NewSender(j.fsfilter)
+	sender := endpoint.NewSender(endpoint.SenderConfig{
+		JobID: j.name,
+		FSF:   j.fsfilter,
+		// FIXME encryption setting is irrelevant for SnapJob because the endpoint is only used as pruner.Target
+		Encrypt: &zfs.NilBool{B: true},
+	})
 	j.pruner = j.prunerFactory.BuildLocalPruner(ctx, sender, alwaysUpToDateReplicationCursorHistory{sender})
 	log.Info("start pruning")
 	j.pruner.Prune()

daemon/logging/build_logging.go

@@ -22,6 +22,7 @@ import (
 	"github.com/zrepl/zrepl/rpc/transportmux"
 	"github.com/zrepl/zrepl/tlsconf"
 	"github.com/zrepl/zrepl/transport"
+	"github.com/zrepl/zrepl/zfs/zfscmd"
 )
 
 func OutletsFromConfig(in config.LoggingOutletEnumList) (*logger.Outlets, error) {
@@ -70,7 +71,7 @@ type Subsystem string
 
 const (
 	SubsysReplication  Subsystem = "repl"
-	SubsyEndpoint      Subsystem = "endpoint"
+	SubsysEndpoint     Subsystem = "endpoint"
 	SubsysPruning      Subsystem = "pruning"
 	SubsysSnapshot     Subsystem = "snapshot"
 	SubsysHooks        Subsystem = "hook"
@@ -79,17 +80,19 @@ const (
 	SubsysRPC          Subsystem = "rpc"
 	SubsysRPCControl   Subsystem = "rpc.ctrl"
 	SubsysRPCData      Subsystem = "rpc.data"
+	SubsysZFSCmd       Subsystem = "zfs.cmd"
 )
 
 func WithSubsystemLoggers(ctx context.Context, log logger.Logger) context.Context {
 	ctx = logic.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
 	ctx = driver.WithLogger(ctx, log.WithField(SubsysField, SubsysReplication))
-	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsyEndpoint))
+	ctx = endpoint.WithLogger(ctx, log.WithField(SubsysField, SubsysEndpoint))
 	ctx = pruner.WithLogger(ctx, log.WithField(SubsysField, SubsysPruning))
 	ctx = snapper.WithLogger(ctx, log.WithField(SubsysField, SubsysSnapshot))
 	ctx = hooks.WithLogger(ctx, log.WithField(SubsysField, SubsysHooks))
 	ctx = transport.WithLogger(ctx, log.WithField(SubsysField, SubsysTransport))
 	ctx = transportmux.WithLogger(ctx, log.WithField(SubsysField, SubsysTransportMux))
+	ctx = zfscmd.WithLogger(ctx, log.WithField(SubsysField, SubsysZFSCmd))
 	ctx = rpc.WithLoggers(ctx,
 		rpc.Loggers{
 			General: log.WithField(SubsysField, SubsysRPC),

daemon/logging/logging_formatters.go

@@ -211,7 +211,7 @@ func logfmtTryEncodeKeyval(enc *logfmt.Encoder, field, value interface{}) error
 	case logfmt.ErrUnsupportedValueType:
 		err := enc.EncodeKeyval(field, fmt.Sprintf("<%T>", value))
 		if err != nil {
-			return errors.Wrap(err, "cannot encode unsuuported value type Go type")
+			return errors.Wrap(err, "cannot encode unsupported value type Go type")
 		}
 		return nil
 	}

daemon/logging/logging_outlets.go

@@ -63,7 +63,7 @@ func NewTCPOutlet(formatter EntryFormatter, network, address string, tlsConfig *
 		return
 	}
 
-	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previos is in io.Copy()
+	entryChan := make(chan *bytes.Buffer, 1) // allow one message in flight while previous is in io.Copy()
 
 	o := &TCPOutlet{
 		formatter: formatter,

daemon/prometheus.go

@@ -10,20 +10,23 @@ import (
 
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/job"
+	"github.com/zrepl/zrepl/endpoint"
 	"github.com/zrepl/zrepl/logger"
 	"github.com/zrepl/zrepl/rpc/dataconn/frameconn"
+	"github.com/zrepl/zrepl/util/tcpsock"
 	"github.com/zrepl/zrepl/zfs"
 )
 
 type prometheusJob struct {
-	listen string
+	listen   string
+	freeBind bool
 }
 
 func newPrometheusJobFromConfig(in *config.PrometheusMonitoring) (*prometheusJob, error) {
 	if _, _, err := net.SplitHostPort(in.Listen); err != nil {
 		return nil, err
 	}
-	return &prometheusJob{in.Listen}, nil
+	return &prometheusJob{in.Listen, in.ListenFreeBind}, nil
 }
 
 var prom struct {
@@ -46,6 +49,8 @@ func (j *prometheusJob) Status() *job.Status { return &job.Status{Type: job.Type
 
 func (j *prometheusJob) OwnedDatasetSubtreeRoot() (p *zfs.DatasetPath, ok bool) { return nil, false }
 
+func (j *prometheusJob) SenderConfig() *endpoint.SenderConfig { return nil }
+
 func (j *prometheusJob) RegisterMetrics(registerer prometheus.Registerer) {}
 
 func (j *prometheusJob) Run(ctx context.Context) {
@@ -60,9 +65,10 @@ func (j *prometheusJob) Run(ctx context.Context) {
 
 	log := job.GetLogger(ctx)
 
-	l, err := net.Listen("tcp", j.listen)
+	l, err := tcpsock.Listen(j.listen, j.freeBind)
 	if err != nil {
 		log.WithError(err).Error("cannot listen")
+		return
 	}
 	go func() {
 		<-ctx.Done()

daemon/pruner/pruner.go

@@ -18,13 +18,13 @@ import (
 	"github.com/zrepl/zrepl/util/envconst"
 )
 
-// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
+// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
 type History interface {
 	ReplicationCursor(ctx context.Context, req *pdu.ReplicationCursorReq) (*pdu.ReplicationCursorRes, error)
 	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
 }
 
-// Try to keep it compatible with gitub.com/zrepl/zrepl/endpoint.Endpoint
+// Try to keep it compatible with github.com/zrepl/zrepl/endpoint.Endpoint
 type Target interface {
 	ListFilesystems(ctx context.Context, req *pdu.ListFilesystemReq) (*pdu.ListFilesystemRes, error)
 	ListFilesystemVersions(ctx context.Context, req *pdu.ListFilesystemVersionsReq) (*pdu.ListFilesystemVersionsRes, error)
@@ -412,9 +412,6 @@ tfss_loop:
 
 		rcReq := &pdu.ReplicationCursorReq{
 			Filesystem: tfs.Path,
-			Op: &pdu.ReplicationCursorReq_Get{
-				Get: &pdu.ReplicationCursorReq_GetOp{},
-			},
 		}
 		rc, err := receiver.ReplicationCursor(ctx, rcReq)
 		if err != nil {

daemon/snapper/snapper.go

@@ -13,6 +13,7 @@ import (
 	"github.com/zrepl/zrepl/daemon/filters"
 	"github.com/zrepl/zrepl/daemon/hooks"
 	"github.com/zrepl/zrepl/logger"
+	"github.com/zrepl/zrepl/util/envconst"
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -276,7 +277,7 @@ func snapshot(a args, u updater) state {
 
 		jobCallback := hooks.NewCallbackHookForFilesystem("snapshot", fs, func(_ context.Context) (err error) {
 			l.Debug("create snapshot")
-			err = zfs.ZFSSnapshot(fs, snapname, false) // TODO propagagte context to ZFSSnapshot
+			err = zfs.ZFSSnapshot(a.ctx, fs, snapname, false) // TODO propagate context to ZFSSnapshot
 			if err != nil {
 				l.WithError(err).Error("cannot create snapshot")
 			}
@@ -400,9 +401,19 @@ func listFSes(ctx context.Context, mf *filters.DatasetMapFilter) (fss []*zfs.Dat
 	return zfs.ZFSListMapping(ctx, mf)
 }
 
+var syncUpWarnNoSnapshotUntilSyncupMinDuration = envconst.Duration("ZREPL_SNAPPER_SYNCUP_WARN_MIN_DURATION", 1*time.Second)
+
+// see docs/snapshotting.rst
 func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval time.Duration) (syncPoint time.Time, err error) {
+
+	const (
+		prioHasVersions int = iota
+		prioNoVersions
+	)
+
 	type snapTime struct {
 		ds   *zfs.DatasetPath
+		prio int // lower is higher
 		time time.Time
 	}
 
@@ -411,55 +422,92 @@ func findSyncPoint(log Logger, fss []*zfs.DatasetPath, prefix string, interval t
 	}
 
 	snaptimes := make([]snapTime, 0, len(fss))
+	hardErrs := 0
 
 	now := time.Now()
 
-	log.Debug("examine filesystem state")
+	log.Debug("examine filesystem state to find sync point")
 	for _, d := range fss {
-
 		l := log.WithField("fs", d.ToString())
-
-		fsvs, err := zfs.ZFSListFilesystemVersions(d, filters.NewTypedPrefixFilter(prefix, zfs.Snapshot))
-		if err != nil {
-			l.WithError(err).Error("cannot list filesystem versions")
-			continue
-		}
-		if len(fsvs) <= 0 {
-			l.WithField("prefix", prefix).Debug("no filesystem versions with prefix")
-			continue
+		syncPoint, err := findSyncPointFSNextOptimalSnapshotTime(l, now, interval, prefix, d)
+		if err == findSyncPointFSNoFilesystemVersionsErr {
+			snaptimes = append(snaptimes, snapTime{
+				ds:   d,
+				prio: prioNoVersions,
+				time: now,
+			})
+		} else if err != nil {
+			hardErrs++
+			l.WithError(err).Error("cannot determine optimal sync point for this filesystem")
+		} else {
+			l.WithField("syncPoint", syncPoint).Debug("found optimal sync point for this filesystem")
+			snaptimes = append(snaptimes, snapTime{
+				ds:   d,
+				prio: prioHasVersions,
+				time: syncPoint,
+			})
 		}
+	}
 
-		// Sort versions by creation
-		sort.SliceStable(fsvs, func(i, j int) bool {
-			return fsvs[i].CreateTXG < fsvs[j].CreateTXG
-		})
-
-		latest := fsvs[len(fsvs)-1]
-		l.WithField("creation", latest.Creation).
-			Debug("found latest snapshot")
-
-		since := now.Sub(latest.Creation)
-		if since < 0 {
-			l.WithField("snapshot", latest.Name).
-				WithField("creation", latest.Creation).
-				Error("snapshot is from the future")
-			continue
-		}
-		next := now
-		if since < interval {
-			next = latest.Creation.Add(interval)
-		}
-		snaptimes = append(snaptimes, snapTime{d, next})
+	if hardErrs == len(fss) {
+		return time.Time{}, fmt.Errorf("hard errors in determining sync point for every matching filesystem")
 	}
 
 	if len(snaptimes) == 0 {
-		snaptimes = append(snaptimes, snapTime{nil, now})
+		panic("implementation error: loop must either inc hardErrs or add result to snaptimes")
 	}
 
+	// sort ascending by (prio,time)
+	// => those filesystems with versions win over those without any
 	sort.Slice(snaptimes, func(i, j int) bool {
-		return snaptimes[i].time.Before(snaptimes[j].time)
+		if snaptimes[i].prio == snaptimes[j].prio {
+			return snaptimes[i].time.Before(snaptimes[j].time)
+		}
+		return snaptimes[i].prio < snaptimes[j].prio
 	})
 
+	winnerSyncPoint := snaptimes[0].time
+	l := log.WithField("syncPoint", winnerSyncPoint.String())
+	l.Info("determined sync point")
+	if winnerSyncPoint.Sub(now) > syncUpWarnNoSnapshotUntilSyncupMinDuration {
+		for _, st := range snaptimes {
+			if st.prio == prioNoVersions {
+				l.WithField("fs", st.ds.ToString()).Warn("filesystem will not be snapshotted until sync point")
+			}
+		}
+	}
+
 	return snaptimes[0].time, nil
 
 }
+
+var findSyncPointFSNoFilesystemVersionsErr = fmt.Errorf("no filesystem versions")
+
+func findSyncPointFSNextOptimalSnapshotTime(l Logger, now time.Time, interval time.Duration, prefix string, d *zfs.DatasetPath) (time.Time, error) {
+
+	fsvs, err := zfs.ZFSListFilesystemVersions(context.TODO(), d, zfs.ListFilesystemVersionsOptions{
+		Types:           zfs.Snapshots,
+		ShortnamePrefix: prefix,
+	})
+	if err != nil {
+		return time.Time{}, errors.Wrap(err, "list filesystem versions")
+	}
+	if len(fsvs) <= 0 {
+		return time.Time{}, findSyncPointFSNoFilesystemVersionsErr
+	}
+
+	// Sort versions by creation
+	sort.SliceStable(fsvs, func(i, j int) bool {
+		return fsvs[i].CreateTXG < fsvs[j].CreateTXG
+	})
+
+	latest := fsvs[len(fsvs)-1]
+	l.WithField("creation", latest.Creation).Debug("found latest snapshot")
+
+	since := now.Sub(latest.Creation)
+	if since < 0 {
+		return time.Time{}, fmt.Errorf("snapshot %q is from the future: creation=%q now=%q", latest.ToAbsPath(d), latest.Creation, now)
+	}
+
+	return latest.Creation.Add(interval), nil
+}

dist/systemd/zrepl.service

@@ -4,8 +4,9 @@ Documentation=https://zrepl.github.io
 
 [Service]
 Type=simple
+ExecStartPre=/usr/local/bin/zrepl --config /etc/zrepl/zrepl.yml configcheck
 ExecStart=/usr/local/bin/zrepl --config /etc/zrepl/zrepl.yml daemon
-RuntimeDirectory=zrepl
+RuntimeDirectory=zrepl zrepl/stdinserver
 RuntimeDirectoryMode=0700
 
 ProtectSystem=strict
@@ -20,22 +21,13 @@ RestrictNamespaces=true
 RestrictRealtime=yes
 SystemCallArchitectures=native
 
-# BEGIN ProtectHome
-ProtectHome=read-only   # DEBIAN STRETCH
-# ProtectHome=tmpfs     # FEDORA 28 / 29
-# END ProtectHome
+ProtectHome=read-only
+# ProtectHome=tmpfs totally possible, not by default though because of Debian stretch
+
+# SystemCallFilter
+#   ~@privileged doesn't work with Ubuntu 18.04 ssh
+SystemCallFilter=~ @mount @cpu-emulation @keyring @module @obsolete @raw-io @debug @clock @resources
 
-# BEGIN SystemCallFilter
-## BEGIN DEBIAN STRETCH
-SystemCallFilter=~ @mount @cpu-emulation @keyring @module @obsolete @privileged @raw-io @debug @clock @resources
-## END DEBIAN STRETCH
-## BEGIN FEDORA 28/29 
-## Syscall blacklist (should be fairly stable)
-#SystemCallFilter=~ @mount @aio @cpu-emulation @keyring @memlock @module @obsolete @privileged @raw-io @reboot @setuid @swap @sync @timer @debug @clock @chown @resources
-## Syscall whitelist (not sure how stable)
-#SystemCallFilter=@default @file-system @process @basic-io @ipc @network-io @signal @io-event brk mprotect sched_getaffinity ioctl getrandom
-## END END FEDORA 28/29
-# END SystemCallFilter
 
 [Install]
 WantedBy=multi-user.target

docs/changelog.rst

@@ -27,6 +27,56 @@ We use the following annotations for classifying changes:
 * |bugfix| Change that fixes a bug, no regressions or incompatibilities expected.
 * |docs| Change to the documentation.
 
+0.3
+---
+
+This is a big one! Headlining features:
+
+* **Resumable Send & Recv Support**
+  No knobs required, automatically used where supported.
+* **Hold-Protected Send & Recv**
+  Automatic ZFS holds to ensure that we can always resume a replication step.
+* **Encrypted Send & Recv Support** for OpenZFS native encryption.
+  :ref:`Configurable <job-send-options>` at the job level, i.e., for all filesystems a job is responsible for.
+* **Receive-side hold on last received dataset**
+  The counterpart to the replication cursor bookmark on the send-side.
+  Ensures that incremental replication will always be possible between a sender and receiver.
+
+Actual changelog:
+
+* |break_config| **more restrictive job names than in prior zrepl versions**
+  Starting with this version, job names are going to be embedded into ZFS holds and bookmark names.
+
+* |break| |mig| replication cursor representation changed
+
+  * zrepl now manages the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` per job-filesystem tuple instead of a single replication cursor per filesystem.
+    In the future, this will permit multiple sending jobs to send from the same filesystems.
+  * ZFS does not allow bookmark renaming, thus we cannot migrate the old replication cursors.
+  * zrepl 0.3 will automatically create cursors in the new format for new replications, and warn if it still finds ones in the old format.
+  * Run ``zrepl migrate replication-cursor:v1-v2`` to safely destroy old-format cursors.
+    The migration will ensure that only those old-format cursors are destroyed that have been superseeded by new-format cursors.
+
+* |bugfix| missing logger context vars in control connection handlers
+* |bugfix| improved error messages on ``zfs send`` errors
+* |feature| New option ``listen_freebind`` (tcp, tls, prometheus listener)
+* |bugfix| |docs| snapshotting: clarify sync-up behavior and warn about filesystems
+  that will not be snapshotted until the sync-up phase is over
+* |docs| Document new replication features in the :ref:`config overview <overview-how-replication-works>` and :repomasterlink:`replication/design.md`.
+* |feature| documented subcommand to generate ``bash`` and ``zsh`` completions
+* **[MAINTAINER NOTICE]** New platform tests in this version, please make sure you run them for your distro!
+* **[MAINTAINER NOTICE]** Please add the shell completions to the zrepl packages.
+
+0.2.1
+-----
+
+* |feature| Illumos (and Solaris) compatibility and binary builds (thanks, `MNX.io <https://mnx.io>`_ )
+* |feature| 32bit binaries for Linux and FreeBSD (untested, though)
+* |bugfix| better error messages in ``ssh+stdinserver`` transport
+* |bugfix| systemd + ``ssh+stdinserver``: automatically create ``/var/run/zrepl/stdinserver``
+* |bugfix| crash if Prometheus listening socket cannot be opened
+
+* [MAINTAINER NOTICE] ``Makefile`` refactoring, see :commit:`080f2c0`
+
 0.2
 ---
 
@@ -56,7 +106,8 @@ We use the following annotations for classifying changes:
    |  zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
    |  You can support maintenance and feature development through one of the following services:
    |  |Donate via Patreon| |Donate via Liberapay| |Donate via PayPal|
-   |  For commerical support, please `contact Christian directly <https://cschwarz.com>`_.
+   |  Note that PayPal processing fees are relatively high for small donations.
+   |  For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
 
 
 0.1.1

docs/configuration.rst

@@ -11,6 +11,7 @@ Configuration
     configuration/jobs
     configuration/transports
     configuration/filter_syntax
+    configuration/sendrecvoptions
     configuration/snapshotting
     configuration/prune
     configuration/logging

docs/configuration/filter_syntax.rst

@@ -5,7 +5,7 @@
 Filter Syntax
 =============
 
-For :ref:`source jobs <job-source>` and :ref:`push jobs <job-push>`, a filesystem filter must be defined (field ``filesystems``).
+For :ref:`source<job-source>`, :ref:`push<job-push>` and :ref:`snap<job-snap>` jobs, a filesystem filter must be defined (field ``filesystems``).
 A filter takes a filesystem path (in the ZFS filesystem hierarchy) as parameter and returns ``true`` (pass) or ``false`` (block).
 
 A filter is specified as a **YAML dictionary** with patterns as keys and booleans as values.

docs/configuration/jobs.rst

@@ -24,6 +24,8 @@ Job Type ``push``
       - |connect-transport|
     * - ``filesystems``
       - |filter-spec| for filesystems to be snapshotted and pushed to the sink
+    * - ``send``
+      - |send-options| 
     * - ``snapshotting``
       - |snapshotting-spec|
     * - ``pruning``
@@ -101,6 +103,8 @@ Job Type ``source``
       - |serve-transport|
     * - ``filesystems``
       - |filter-spec| for filesystems to be snapshotted and exposed to connecting clients
+    * - ``send``
+      - |send-options| 
     * - ``snapshotting``
       - |snapshotting-spec|
 

docs/configuration/logging.rst

@@ -202,7 +202,7 @@ The latter is particularly useful in combination with log aggregation services.
 .. WARNING::
 
     zrepl drops log messages to the TCP outlet if the underlying connection is not fast enough.
-    Note that TCP buffering in the kernel must first run full becfore messages are dropped.
+    Note that TCP buffering in the kernel must first run full before messages are dropped.
 
     Make sure to always configure a ``stdout`` outlet as the special error outlet to be informed about problems
     with the TCP outlet (see :ref:`above <logging-error-outlet>` ).

docs/configuration/monitoring.rst

@@ -14,7 +14,8 @@ Prometheus & Grafana
 
 zrepl can expose `Prometheus metrics <https://prometheus.io/docs/instrumenting/exposition_formats/>`_ via HTTP.
 The ``listen`` attribute is a `net.Listen <https://golang.org/pkg/net/#Listen>`_  string for tcp, e.g. ``:9091`` or ``127.0.0.1:9091``.
-The Prometheues monitoring job appears in the ``zrepl control`` job list and may be specified **at most once**.
+The ``listen_freebind`` attribute is :ref:`explained here <listen-freebind-explanation>`.
+The Prometheus monitoring job appears in the ``zrepl control`` job list and may be specified **at most once**.
 
 zrepl also ships with an importable `Grafana <https://grafana.com>`_ dashboard that consumes the Prometheus metrics:
 see :repomasterlink:`dist/grafana`.
@@ -30,6 +31,7 @@ The dashboard also contains some advice on which metrics are important to monito
       monitoring:
         - type: prometheus
           listen: ':9091'
+          listen_freebind: true # optional, default false
 
 
 

docs/configuration/overview.rst

@@ -26,9 +26,9 @@ Config File Structure
      type: push
    - ...
 
-zrepl is confgured using a single YAML configuration file with two main sections: ``global`` and ``jobs``.
+zrepl is configured using a single YAML configuration file with two main sections: ``global`` and ``jobs``.
 The ``global`` section is filled with sensible defaults and is covered later in this chapter.
-The ``jobs`` section is a list of jobs which we are goind to explain now.
+The ``jobs`` section is a list of jobs which we are going to explain now.
 
 .. _job-overview:
 
@@ -41,7 +41,7 @@ Jobs are identified by their ``name``, both in log files and the ``zrepl status`
 
 Replication always happens between a pair of jobs: one is the **active side**, and one the **passive side**.
 The active side connects to the passive side using a :ref:`transport <transport>` and starts executing the replication logic.
-The passive side responds to requests from the active side after checking its persmissions.
+The passive side responds to requests from the active side after checking its permissions.
 
 The following table shows how different job types can be combined to achieve **both push and pull mode setups**.
 Note that snapshot-creation denoted by "(snap)" is orthogonal to whether a job is active or passive.
@@ -90,6 +90,8 @@ It uses the client identity for access control:
 .. TIP::
    The implementation of the ``sink`` job requires that the connecting client identities be a valid ZFS filesystem name components.
 
+.. _overview-how-replication-works:
+
 How Replication Works
 ---------------------
 
@@ -107,9 +109,8 @@ The following steps take place during replication and can be monitored using the
     * Per filesystem, compute a diff between sender and receiver snapshots
     * Build a list of replication steps
 
-      * If possible, use incremental sends (``zfs send -i``)
+      * If possible, use incremental and resumable sends (``zfs send -i``)
       * Otherwise, use full send of most recent snapshot on sender
-      * Give up on filesystems that cannot be replicated without data loss
 
   * Retry on errors that are likely temporary (i.e. network failures).
   * Give up on filesystems where a permanent error was received over RPC.
@@ -119,22 +120,28 @@ The following steps take place during replication and can be monitored using the
   * Perform replication steps in the following order:
     Among all filesystems with pending replication steps, pick the filesystem whose next replication step's snapshot is the oldest.
   * Create placeholder filesystems on the receiving side to mirror the dataset paths on the sender to ``root_fs/${client_identity}``.
-  * After a successful replication step, update the replication cursor bookmark (see below).
+  * Acquire send-side *step-holds* on the step's `from` and `to` snapshots.
+  * Perform the replication step.
+  * Move the **replication cursor** bookmark on the sending side (see below).
+  * Move the **last-received-hold** on the receiving side (see below).
+  * Release the send-side step-holds.
    
 The idea behind the execution order of replication steps is that if the sender snapshots all filesystems simultaneously at fixed intervals, the receiver will have all filesystems snapshotted at time ``T1`` before the first snapshot at ``T2 = T1 + $interval`` is replicated.
 
-.. _replication-cursor-bookmark:
+.. _replication-cursor-and-last-received-hold:
 
-The **replication cursor bookmark** ``#zrepl_replication_cursor`` is kept per filesystem on the sending side of a replication setup:
-It is a bookmark of the most recent successfully replicated snapshot to the receiving side.
-It is is used by the :ref:`not_replicated <prune-keep-not-replicated>` keep rule to identify all snapshots that have not yet been replicated to the receiving side.
-Regardless of whether that keep rule is used, the bookmark ensures that replication can always continue incrementally.
-Note that there is only one cursor bookmark per filesystem, which prohibits multiple jobs to replicate the same filesystem (:ref:`see below<jobs-multiple-jobs>`).
+**Replication cursor** bookmark and **last-received-hold** are managed by zrepl to ensure that future replications can always be done incrementally:
+the replication cursor is a send-side bookmark of the most recent successfully replicated snapshot,
+and the last-received-hold is a hold of that snapshot on the receiving side.
+The replication cursor has the format ``#zrepl_CUSOR_G_<GUID>_J_<JOBNAME>``.
+The last-received-hold tag has the format ``#zrepl_last_received_J_<JOBNAME>``.
+Encoding the job name in the names ensures that multiple sending jobs can replicate the same filesystem to different receivers without interference.
+The ``zrepl holds list`` provides a listing of all bookmarks and holds managed by zrepl.
 
 .. _replication-placeholder-property:
 
 **Placeholder filesystems** on the receiving side are regular ZFS filesystems with the placeholder property ``zrepl:placeholder=on``.
-Placeholders allow the receiving side to mirror the sender's ZFS dataset hierachy without replicating every filesystem at every intermediary dataset path component.
+Placeholders allow the receiving side to mirror the sender's ZFS dataset hierarchy without replicating every filesystem at every intermediary dataset path component.
 Consider the following example: ``S/H/J`` shall be replicated to ``R/sink/job/S/H/J``, but neither ``S/H`` nor ``S`` shall be replicated.
 ZFS requires the existence of ``R/sink/job/S`` and ``R/sink/job/S/H`` in order to receive into ``R/sink/job/S/H/J``.
 Thus, zrepl creates the parent filesystems as placeholders on the receiving side.
@@ -144,9 +151,13 @@ The ``zrepl test placeholder`` command can be used to check whether a filesystem
 .. ATTENTION::
 
     Currently, zrepl does not replicate filesystem properties.
-    Whe receiving a filesystem, it is never mounted (`-u` flag)  and `mountpoint=none` is set.
+    When receiving a filesystem, it is never mounted (`-u` flag)  and `mountpoint=none` is set.
     This is temporary and being worked on :issue:`24`.
 
+.. NOTE::
+
+    More details can be found in the design document :repomasterlink:`replication/design.md`.
+
 
 .. _jobs-multiple-jobs:
 
@@ -170,8 +181,8 @@ No Overlapping
 
 Jobs run independently of each other.
 If two jobs match the same filesystem with their ``filesystems`` filter, they will operate on that filesystem independently and potentially in parallel.
-For example, if job A prunes snapshots that job B is planning to replicate, the replication will fail because B asssumed the snapshot to still be present.
-More subtle race conditions can occur with the :ref:`replication cursor bookmark <replication-cursor-bookmark>`, which currently only exists once per filesystem.
+For example, if job A prunes snapshots that job B is planning to replicate, the replication will fail because B assumed the snapshot to still be present.
+However, the next replication attempt will re-examine the situation from scratch and should work.
 
 N push jobs to 1 sink
 ~~~~~~~~~~~~~~~~~~~~~
@@ -187,5 +198,5 @@ Multiple pull jobs pulling from the same source have potential for race conditio
 each pull job prunes the source side independently, causing replication-prune and prune-prune races.
 
 There is currently no way for a pull job to filter which snapshots it should attempt to replicate.
-Thus, it is not possibe to just manually assert that the prune rules of all pull jobs are disjoint to avoid replication-prune and prune-prune races.
+Thus, it is not possible to just manually assert that the prune rules of all pull jobs are disjoint to avoid replication-prune and prune-prune races.
 

docs/configuration/prune.rst

@@ -4,7 +4,7 @@ Pruning Policies
 ================
 
 In zrepl, *pruning* means *destroying snapshots*.
-Pruning must happen on both sides of a replication or the systems would inevitable run out of disk space at some point.
+Pruning must happen on both sides of a replication or the systems would inevitably run out of disk space at some point.
 
 Typically, the requirements to temporal resolution and maximum retention time differ per side.
 For example, when using zrepl to back up a busy database server, you will want high temporal resolution (snapshots every 10 min) for the last 24h in case of administrative disasters, but cannot afford to store them for much longer because you might have high turnover volume in the database.
@@ -66,7 +66,7 @@ Policy ``not_replicated``
 
 ``not_replicated`` keeps all snapshots that have not been replicated to the receiving side.
 It only makes sense to specify this rule on a sender (source or push job).
-The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark <replication-cursor-bookmark>` on the sending side.
+The state required to evaluate this rule is stored in the :ref:`replication cursor bookmark <replication-cursor-and-last-received-hold>` on the sending side.
 
 .. _prune-keep-retention-grid:
 
@@ -154,7 +154,7 @@ Policy ``regex``
            negate: true
            regex: "^zrepl_.*"
 
-``regex`` keeps all snapshots whose names are matched by the regular expressionin ``regex``.
+``regex`` keeps all snapshots whose names are matched by the regular expression in ``regex``.
 Like all other regular expression fields in prune policies, zrepl uses Go's `regexp.Regexp <https://golang.org/pkg/regexp/#Compile>`_ Perl-compatible regular expressions (`Syntax <https://golang.org/pkg/regexp/syntax>`_).
 The optional `negate` boolean field inverts the semantics: Use it if you want to keep all snapshots that *do not* match the given regex.
 

docs/configuration/sendrecvoptions.rst

@@ -0,0 +1,45 @@
+.. include:: ../global.rst.inc
+
+
+Send & Recv Options
+===================
+
+.. _job-send-options:
+
+Send Options
+~~~~~~~~~~~~
+
+::
+   
+   jobs:
+   - type: push
+     filesystems: ...
+     send:
+       encrypted: true
+     ...
+
+:ref:`Source<job-source>` and :ref:`push<job-push>` jobs have an optional ``send`` configuration section.
+
+``encryption`` option
+---------------------
+
+The ``encryption`` variable controls whether the matched filesystems are sent as `OpenZFS native encryption <http://open-zfs.org/wiki/ZFS-Native_Encryption>`_ raw sends.
+More specifically, if ``encryption=true``, zrepl
+
+* checks for any of the filesystems matched by ``filesystems`` whether the ZFS ``encryption`` property indicates that the filesystem is actually encrypted with ZFS native encryption and
+* invokes the ``zfs send`` subcommand with the ``-w`` option (raw sends) and
+* expects the receiving side to support OpenZFS native encryption (recv will fail otherwise)
+
+Filesystems matched by ``filesystems`` that are not encrypted are not sent and will cause error log messages.
+
+If ``encryption=false``, zrepl expects that filesystems matching ``filesystems`` are not encrypted or have loaded encryption keys.
+
+.. _job-recv-options:
+
+Recv Options
+~~~~~~~~~~~~
+
+:ref:`Sink<job-sink>` and :ref:`pull<job-pull>` jobs have an optional ``recv`` configuration section.
+However, there are currently no variables to configure there.
+
+

docs/configuration/snapshotting.rst

@@ -9,8 +9,15 @@ The ``push``, ``source`` and ``snap`` jobs can automatically take periodic snaps
 The snapshot names are composed of a user-defined prefix followed by a UTC date formatted like ``20060102_150405_000``.
 We use UTC because it will avoid name conflicts when switching time zones or between summer and winter time.
 
+When a job is started, the snapshotter attempts to get the snapshotting rhythms of the matched ``filesystems`` in sync because snapshotting all filesystems at the same time results in a more consistent backup.
+To find that sync point, the most recent snapshot, made by the snapshotter, in any of the matched ``filesystems`` is used.
+A filesystem that does not have snapshots by the snapshotter has lower priority than filesystem that do, and thus might not be snapshotted (and replicated) until it is snapshotted at the next sync point.
+
 For ``push`` jobs, replication is automatically triggered after all filesystems have been snapshotted.
 
+Note that the ``zrepl signal wakeup JOB`` subcommand does not trigger snapshotting.
+
+
 ::
 
     jobs:
@@ -113,14 +120,11 @@ Most hook types take additional parameters, please refer to the respective subse
       ...
 
 
-The hook type ``command`` is the only currently supported hook type.
-Future versions of zrepl may support other hook types.
-The ``path`` to the hook executables must be absolute (e.g. ``/etc/zrepl/hooks/zrepl-notify.sh``).
+``command`` hooks take a ``path`` to an executable script or binary to be executed before and after the snapshot.
+``path`` must be absolute (e.g. ``/etc/zrepl/hooks/zrepl-notify.sh``).
 No arguments may be specified; create a wrapper script if zrepl must call an executable that requires arguments.
-zrepl will call the hook both before and after the snapshot, but with different values of the ``ZREPL_HOOKTYPE`` environment variable; see below.
 The process standard output is logged at level INFO. Standard error is logged at level WARN.
-
-zrepl sets a number of environment variables for the hook processes:
+The following environment variables are set:
 
 * ``ZREPL_HOOKTYPE``: either "pre_snapshot" or "post_snapshot"
 * ``ZREPL_FS``: the ZFS filesystem name being snapshotted

docs/configuration/transports.rst

@@ -50,12 +50,25 @@ Serve
       serve:
         type: tcp
         listen: ":8888"
+        listen_freebind: true # optional, default false
         clients: {
-          "192.168.122.123" : "mysql01"
-          "192.168.122.123" : "mx01"
+          "192.168.122.123" :               "mysql01",
+          "192.168.122.42" :                "mx01",
+          "2001:0db8:85a3::8a2e:0370:7334": "gateway",
+
+          # CIDR masks require a '*' in the client identity string
+          # that is expanded to the client's IP address
+
+          "10.23.42.0/24":       "cluster-*"
+          "fde4:8dba:82e1::/64": "san-*"
         }
       ...
 
+.. _listen-freebind-explanation:
+
+``listen_freebind`` controls whether the socket is allowed to bind to non-local or unconfigured IP addresses (Linux ``IP_FREEBIND`` , FreeBSD ``IP_BINDANY``).
+Enable this option if you want to ``listen`` on a specific IP address that might not yet be configured when the zrepl daemon starts.
+
 Connect
 ~~~~~~~
 
@@ -101,6 +114,7 @@ Serve
         serve:
           type: tls
           listen: ":8888"
+          listen_freebind: true # optional, default false
           ca:   /etc/zrepl/ca.crt
           cert: /etc/zrepl/prod.fullchain
           key:  /etc/zrepl/prod.key
@@ -110,6 +124,7 @@ Serve
 
 The ``ca`` field specified the certificate authority used to validate client certificates.
 The ``client_cns`` list specifies a list of accepted client common names (which are also the client identities for this transport).
+The ``listen_freebind`` field is :ref:`explained here <listen-freebind-explanation>`.
 
 Connect
 ~~~~~~~
@@ -287,9 +302,9 @@ Connect
         user: root
         port: 22
         identity_file: /etc/zrepl/ssh/identity
-        options: # optional, default [], `-o` arguments passed to ssh
-        - "Compression=on"
-        dial_timeout: # optional, default 10s, max time.Duration until initial handshake is completed
+        # options: # optional, default [], `-o` arguments passed to ssh
+        # - "Compression=yes"
+        # dial_timeout: 10s # optional, default 10s, max time.Duration until initial handshake is completed
 
 The connecting zrepl daemon
 

docs/global.rst.inc

@@ -21,6 +21,8 @@
 
 .. |serve-transport| replace:: :ref:`serve specification<transport>`
 .. |connect-transport| replace:: :ref:`connect specification<transport>`
+.. |send-options| replace:: :ref:`send options<job-send-options>`, e.g. for encrypted sends
+.. |recv-options| replace:: :ref:`recv options<job-recv-options>`
 .. |snapshotting-spec| replace:: :ref:`snapshotting specification <job-snapshotting-spec>`
 .. |pruning-spec| replace:: :ref:`pruning specification <prune>`
 .. |filter-spec| replace:: :ref:`filter specification<pattern-filter>`

docs/implementation.rst

@@ -1,61 +0,0 @@
-.. _implementation_toc:
-
-Implementation Overview
-=======================
-
-.. WARNING::
-
-   Incomplete and possibly outdated.
-   Check out the :ref:`talks about zrepl <pr-talks>` at various conferences for up-to-date material.
-   Alternatively, have a `look at the source code <http://github.com/zrepl/zrepl>`_ ;)
-
-The following design aspects may convince you that ``zrepl`` is superior to a hacked-together shell script solution.
-Also check out the :ref:`talks about zrepl <pr-talks>` at various conferences.
-
-Testability & Performance
--------------------------
-
-zrepl is written in Go, a real programming language with type safety,
-reasonable performance, testing infrastructure and an (opinionated) idea of
-software engineering.
-
-* key parts & algorithms of zrepl are covered by unit tests (work in progress)
-* zrepl is noticably faster than comparable shell scripts
-
-
-RPC protocol
-------------
-
-While it is tempting to just issue a few ``ssh remote 'zfs send ...' | zfs recv``, this has a number of drawbacks:
-
-* The snapshot streams need to be compatible.
-* Communication is still unidirectional. Thus, you will most likely
-
-  * either not take advantage of advanced replication features such as *compressed send & recv*
-  * or issue additional ``ssh`` commands in advance to figure out what features are supported on the other side.
-
-* Advanced logic in shell scripts is ugly to read, poorly testable and a pain to maintain.
-
-zrepl takes a different approach:
-
-* Define an RPC protocol.
-* Establish an encrypted, authenticated, bidirectional communication channel.
-* Run daemons on both sides of the setup and let them talk to each other.
-
-This has several obvious benefits:
-
-* No blank root shell access is given to the other side.
-* An *authenticated* peer *requests* filesystem lists, snapshot streams, etc.
-* The server decides which filesystems it exposes to which peers.
-* The :ref:`transport mechanism <transport>` is decoupled from the remaining logic, which allows us to painlessly offer multiple transport mechanisms.
-
-Protocol Implementation
-~~~~~~~~~~~~~~~~~~~~~~~
-
-zrepl uses a custom RPC protol because, at the time of writing, existing solutions like gRPC do not provide efficient means to transport large amounts of data, whose size is unknown at send time (= zfs send streams).
-The package used is `github.com/problame/go-streamrpc <https://github.com/problame/go-streamrpc/tree/master>`_.
-
-Logging & Transparency
-----------------------
-
-zrepl comes with :ref:`rich, structured and configurable logging <logging>`, allowing administators to understand what the software is actually doing.

docs/index.rst

@@ -52,14 +52,16 @@ Main Features
 * **Filesystem replication**
 
   * [x] Pull & Push mode
-  * [x] Multiple transport :ref:`transports <transport>`: TCP, TCP + TLS client auth, SSH
+  * [x] Multiple :ref:`transport modes <transport>`: TCP, TCP + TLS client auth, SSH
 
   * Advanced replication features
 
     * [x] Automatic retries for temporary network errors
-    * [ ] Resumable send & receive
+    * [x] Automatic resumable send & receive
+    * [x] Automatic ZFS holds during send & receive
+    * [x] Automatic bookmark \& hold management for guaranteed incremental send & recv 
+    * [x] Encrypted raw send & receive to untrusted receivers (OpenZFS native encryption)
     * [ ] Compressed send & receive
-    * [ ] Raw encrypted send & receive
 
 * **Automatic snapshot management**
 
@@ -83,6 +85,7 @@ Main Features
 * **Maintainable implementation in Go**
 
   * [x] Cross platform
+  * [x] Dynamic feature checking
   * [x] Type safe & testable code
 
 
@@ -126,7 +129,6 @@ Table of Contents
    installation
    configuration
    usage
-   implementation
    pr
    changelog
    GitHub Repository & Issue Tracker <https://github.com/zrepl/zrepl>

docs/installation.rst

@@ -1,92 +1,19 @@
-.. _binary releases: https://github.com/zrepl/zrepl/releases
 
-.. _installation:
+.. _installation_toc:
 
+************
 Installation
-============
+************
 
 .. TIP::
 
     Note: check out the :ref:`tutorial` if you want a first impression of zrepl.
 
-User Privileges
----------------
-
-It is possible to run zrepl as an unprivileged user in combination with
-`ZFS delegation <https://www.freebsd.org/doc/handbook/zfs-zfs-allow.html>`_.
-Also, there is the possibility to run it in a jail on FreeBSD by delegating a dataset to the jail.
-However, until we get around documenting those setups, you will have to run zrepl as root or experiment yourself :)
-
-Packages
---------
-
-zrepl source releases are signed & tagged by the author in the git repository.
-Your OS vendor may provide binary packages of zrepl through the package manager.
-Additionally, `binary releases`_ are provided on GitHub.
-The following list may be incomplete, feel free to submit a PR with an update:
-
-.. list-table::
-    :header-rows: 1
-
-    * - OS / Distro
-      - Install Command
-      - Link
-    * - FreeBSD
-      - ``pkg install zrepl``
-      - `<https://www.freshports.org/sysutils/zrepl/>`_
-    * - MacOS
-      - ``brew install zrepl``
-      - Available on `homebrew <https://brew.sh>`_
-    * - Others
-      -
-      - Use `binary releases`_ or build from source.
-
-Compile From Source
-~~~~~~~~~~~~~~~~~~~
-
-Producing a release requires **Go 1.11** or newer and **Python 3** + **pip3** + ``docs/requirements.txt`` for the Sphinx documentation.
-A tutorial to install Go is available over at `golang.org <https://golang.org/doc/install>`_.
-Python and pip3 should probably be installed via your distro's package manager.
-
-Alternatively, you can use the Docker build process:
-it is used to produce the official zrepl `binary releases`_
-and serves as a reference for build dependencies and procedure:
-
-::
-
-    git clone https://github.com/zrepl/zrepl.git
-    cd zrepl
-    sudo docker build -t zrepl_build -f build.Dockerfile .
-    sudo docker run -it --rm \
-        -v "${PWD}:/src" \
-        --user "$(id -u):$(id -g)" \
-        zrepl_build make release
-
-Alternatively, you can install build dependencies on your local system and then build in your ``$GOPATH``:
-
-::
-
-    mkdir -p "${GOPATH}/src/github.com/zrepl/zrepl"
-    git clone https://github.com/zrepl/zrepl.git "${GOPATH}/src/github.com/zrepl/zrepl"
-    cd "${GOPATH}/src/github.com/zrepl/zrepl"
-    python3 -m venv3
-    source venv3/bin/activate
-    ./lazy.sh devsetup
-    make release
-
-The Python venv is used for the documentation build dependencies.
-If you just want to build the zrepl binary, leave it out and use `./lazy.sh godep` instead.
-Either way, all build results are located in the ``artifacts/`` directory.
-
-.. NOTE::
-
-    It is your job to install the apropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
-    Otherwise, the examples in the :ref:`tutorial` may need to be adjusted.
-
-What next?
-----------
-
-Read the :ref:`configuration chapter<configuration_toc>` and then continue with the :ref:`usage chapter<usage>`.
-
-**Reminder**: If you want a quick introduction, please read the :ref:`tutorial`.
+.. toctree::
 
+    installation/user-privileges
+    installation/packages
+    installation/apt-repos
+    installation/compile-from-source
+    installation/freebsd-jail-with-iocage
+    installation/what-next

docs/installation/apt-repos.rst

@@ -0,0 +1,29 @@
+
+.. _installation-apt-repos:
+
+Debian / Ubuntu APT repositories
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+We maintain APT repositories for Debian, Ubuntu and derivatives.
+The fingerprint of the signing key is ``E101 418F D3D6 FBCB 9D65  A62D 7086 99FC 5F2E BF16``.
+It is available at `<https://zrepl.cschwarz.com/apt/apt-key.asc>`_ .
+Please open an issue `in the packaging repository <https://github.com/zrepl/debian-binary-packaging>`_ if you encounter any issues with the repository.
+
+The following snippet configure the repository for your Debian or Ubuntu release:
+
+::
+
+    apt update && apt install curl gnupg lsb-release; \
+    ARCH="$(dpkg --print-architecture)"; \
+    CODENAME="$(lsb_release -i -s | tr '[:upper:]' '[:lower:]') $(lsb_release -c -s | tr '[:upper:]' '[:lower:]')"; \
+    echo "Using Distro and Codename: $CODENAME"; \
+    (curl https://zrepl.cschwarz.com/apt/apt-key.asc | apt-key add -) && \
+    (echo "deb [arch=$ARCH] https://zrepl.cschwarz.com/apt/$CODENAME main" > /etc/apt/sources.list.d/zrepl.list) && \
+    apt update
+
+
+.. NOTE::
+
+   Until zrepl reaches 1.0, all APT repositories will be updated to the latest zrepl release immediately.
+   This includes breaking changes between zrepl versions.
+   Use ``apt-mark hold zrepl`` to prevent upgrades of zrepl.

docs/installation/compile-from-source.rst

@@ -0,0 +1,45 @@
+.. _binary releases: https://github.com/zrepl/zrepl/releases
+
+.. _installation-compile-from-source:
+
+Compile From Source
+~~~~~~~~~~~~~~~~~~~
+
+Producing a release requires **Go 1.11** or newer and **Python 3** + **pip3** + ``docs/requirements.txt`` for the Sphinx documentation.
+A tutorial to install Go is available over at `golang.org <https://golang.org/doc/install>`_.
+Python and pip3 should probably be installed via your distro's package manager.
+
+Alternatively, you can use the Docker build process:
+it is used to produce the official zrepl `binary releases`_
+and serves as a reference for build dependencies and procedure:
+
+::
+
+    git clone https://github.com/zrepl/zrepl.git && \
+    cd zrepl && \
+    sudo docker build -t zrepl_build -f build.Dockerfile . && \
+    sudo docker run -it --rm \
+        -v "${PWD}:/src" \
+        --user "$(id -u):$(id -g)" \
+        zrepl_build make release
+
+Alternatively, you can install build dependencies on your local system and then build in your ``$GOPATH``:
+
+::
+
+    mkdir -p "${GOPATH}/src/github.com/zrepl/zrepl"
+    git clone https://github.com/zrepl/zrepl.git "${GOPATH}/src/github.com/zrepl/zrepl"
+    cd "${GOPATH}/src/github.com/zrepl/zrepl"
+    python3 -m venv3
+    source venv3/bin/activate
+    ./lazy.sh devsetup
+    make release
+
+The Python venv is used for the documentation build dependencies.
+If you just want to build the zrepl binary, leave it out and use `./lazy.sh godep` instead.
+Either way, all build results are located in the ``artifacts/`` directory.
+
+.. NOTE::
+
+    It is your job to install the appropriate binary in the zrepl users's ``$PATH``, e.g. ``/usr/local/bin/zrepl``.
+    Otherwise, the examples in the :ref:`tutorial` may need to be adjusted.

docs/installation/freebsd-jail-with-iocage.rst

@@ -0,0 +1,140 @@
+.. include:: ../global.rst.inc
+
+.. _installation-freebsd-jail-with-iocage:
+
+FreeBSD Jail With iocage
+========================
+
+
+This tutorial shows how zrepl can be installed on FreeBSD, or FreeNAS in a jail using iocage.
+While this tutorial focuses on using iocage, much of the setup would be similar
+using a different jail manager.
+
+.. NOTE::
+
+   From a security perspective, just keep in mind that ``zfs send``/``recv`` was never designed with
+   jails in mind, an attacker could probably crash the receive-side kernel or worse induce stateful
+   damage to the receive-side pool if they were able to get access to the jail.
+
+   The jail doesn't provide security benefits, but only management ones.
+
+Requirements
+------------
+
+A dataset that will be delegated to the jail needs to be created if one does not already exist.
+For the tutorial ``tank/zrepl`` will be used.
+
+.. code-block:: bash
+
+   zfs create -o mountpoint=none tank/zrepl
+
+The only software requirements on the host system are ``iocage``, which can be installed
+from ports or packages.
+
+.. code-block:: bash
+
+   pkg install py37-iocage
+
+.. NOTE::
+
+   By default ``iocage`` will "activate" on first use which will set up some defaults such as
+   which pool will be used. To activate ``iocage`` manually the ``iocage activate`` command can be used.
+
+Jail Creation
+-------------
+
+There are two options for jail creation using FreeBSD.
+
+1. Manually set up the jail from scratch
+2. Create the jail using the ``zrepl`` plugin. On FreeNAS this is possible from the user interface using the community index.
+
+Manual Jail
+###########
+
+Create a jail, using the same release as the host, called ``zrepl`` that will be automatically started at boot.
+The jail will have ``tank/zrepl`` delegated into it.
+
+.. code-block:: bash
+
+   iocage create --release "$(freebsd-version -k | cut -d '-' -f '1,2')" --name zrepl \
+          boot=on nat=1 \
+          jail_zfs=on \
+          jail_zfs_dataset=zrepl \
+          jail_zfs_mountpoint='none'
+
+Enter the jail:
+
+.. code-block:: bash
+
+   iocage console zrepl
+
+Install ``zrepl``
+
+.. code-block:: bash
+
+   pkg update && pkg upgrade
+   pkg install zrepl
+
+Create the log file ``/var/log/zrepl.log``
+
+.. code-block:: bash
+
+   touch /var/log/zrepl.log && service newsyslog restart
+
+Tell syslogd to redirect facility local0 to the ``zrepl.log`` file:
+
+.. code-block:: bash
+
+   service syslogd reload
+
+Enable the zrepl daemon to start automatically at boot:
+
+.. code-block:: bash
+
+   sysrc zrepl_enable="YES"
+
+
+Plugin
+######
+
+When using the plugin, ``zrepl`` will be installed for you in a jail using the following ``iocage`` properties.
+
+* ``nat=1``
+* ``jail_zfs=on``
+* ``jail_zfs_mountpoint=none``
+
+Additionally the delegated dataset should be specified upon creation, and optionally start on boot can be set.
+This can also be done from the FreeNAS webui.
+
+.. code-block:: bash
+
+   fetch https://raw.githubusercontent.com/ix-plugin-hub/iocage-plugin-index/master/zrepl.json -o /tmp/zrepl.json
+   iocage fetch -P /tmp/zrepl.json --name zrepl jail_zfs_dataset=zrepl boot=on
+
+Configuration
+-------------
+
+Now ``zrepl`` can be configured.
+
+Enter the jail.
+
+.. code-block:: bash
+
+   iocage console zrepl
+
+Modify the ``/usr/local/etc/zrepl/zrepl.yml`` configuration file.
+
+.. TIP::
+
+    Note: check out the :ref:`tutorial` for examples of a ``sink`` job.
+
+Now ``zrepl`` can be started.
+
+.. code-block:: bash
+
+   service zrepl start
+
+Summary
+-------
+
+Congratulations, you have a working jail!

docs/installation/packages.rst

@@ -0,0 +1,50 @@
+.. _installation-packages:
+
+.. _binary releases: https://github.com/zrepl/zrepl/releases
+
+Packages
+--------
+
+zrepl source releases are signed & tagged by the author in the git repository.
+Your OS vendor may provide binary packages of zrepl through the package manager.
+Additionally, `binary releases`_ are provided on GitHub.
+The following list may be incomplete, feel free to submit a PR with an update:
+
+.. list-table::
+    :header-rows: 1
+
+    * - OS / Distro
+      - Install Command
+      - Link
+    * - FreeBSD
+      - ``pkg install zrepl``
+      - `<https://www.freshports.org/sysutils/zrepl/>`_
+      
+        :ref:`installation-freebsd-jail-with-iocage`
+    * - FreeNAS
+      -
+      - :ref:`installation-freebsd-jail-with-iocage`
+    * - MacOS
+      - ``brew install zrepl``
+      - Available on `homebrew <https://brew.sh>`_
+    * - Arch Linux
+      - ``yay install zrepl``
+      - Available on `AUR <https://aur.archlinux.org/packages/zrepl>`_
+    * - Fedora
+      - ``dnf install zrepl``
+      - Available on `COPR <https://copr.fedorainfracloud.org/coprs/poettlerric/zrepl/>`_
+    * - CentOS/RHEL
+      - ``yum install zrepl``
+      - Available on `COPR <https://copr.fedorainfracloud.org/coprs/poettlerric/zrepl/>`_
+    * - Debian + Ubuntu
+      - ``apt install zrepl``
+      - APT repository config :ref:`see below <installation-apt-repos>`
+    * - OmniOS
+      - ``pkg install zrepl``
+      - Available since `r151030 <https://pkg.omniosce.org/r151030/extra/en/search.shtml?token=zrepl&action=Search>`_
+    * - Void Linux
+      - ``xbps-install zrepl``
+      - Available since `a88a2a4 <https://github.com/void-linux/void-packages/commit/a88a2a4d7bf56072dadf61ab56b8424e39155890>`_
+    * - Others
+      -
+      - Use `binary releases`_ or build from source.

docs/installation/user-privileges.rst

@@ -0,0 +1,12 @@
+.. _installation-user-privileges:
+
+User Privileges
+---------------
+
+It is possible to run zrepl as an unprivileged user in combination with
+`ZFS delegation <https://www.freebsd.org/doc/handbook/zfs-zfs-allow.html>`_.
+Also, there is the possibility to run it in a jail on FreeBSD by delegating a dataset to the jail.
+
+.. TIP::
+
+    Note: check out the :ref:`installation-freebsd-jail-with-iocage` for FreeBSD jail setup instructions.

docs/installation/what-next.rst

@@ -0,0 +1,8 @@
+.. _installation-what-next:
+
+What next?
+----------
+
+Read the :ref:`configuration chapter<configuration_toc>` and then continue with the :ref:`usage chapter<usage>`.
+
+**Reminder**: If you want a quick introduction, please read the :ref:`tutorial`.

docs/publish.sh

@@ -55,8 +55,7 @@ set -e
 sphinx-versioning build \
    --show-banner \
    --whitelist-branches '^master$' \
-   --whitelist-tags '(v)?\d+\.\d+\.\d+$' \
-   --whitelist-tags '(v)?0.1.0-rc(3|4)$' \
+   --whitelist-tags '(v)?\d+\.[1-9][0-9]*.\d+$' \
    docs ./public_git \
    -- -c sphinxconf # older conf.py throw errors because they used
                     # version = subprocess.show_output(["git", "describe"])

docs/supporters.rst

@@ -6,7 +6,7 @@
 
 zrepl is a spare-time project primarily developed by `Christian Schwarz <https://cschwarz.com>`_.
 You can support maintenance and feature development through one of the services listed above.
-For commerical support, please `contact Christian directly <https://cschwarz.com>`_.
+For SEPA wire transfer and **commercial support**, please `contact Christian directly <https://cschwarz.com>`_.
 
 **Thanks for your support!**
 
@@ -18,11 +18,31 @@ For commerical support, please `contact Christian directly <https://cschwarz.com
 Supporters
 ==========
 
-We would like to thank the following people / organizations for supporting zrepl through monetary and other means:
+We would like to thank the following people and organizations for supporting zrepl through monetary and other means:
 
-* `Ross Williams <https://github.com/overhacked>`_
-* Mike T.
-* `Justin Scholz <https://github.com/JMoVS>`_
-* `InsanePrawn <https://github.com/InsanePrawn>`_
-* `Ben Woods <https://www.freshports.org/sysutils/zrepl/>`_
-* `Anton Schirg <https://github.com/antonxy>`_
+.. |supporter-std| raw:: html
+
+   <div class="fa fa-star-o" style="width: 1em;"></div>
+
+.. |supporter-gold| raw:: html
+
+   <div class="fa fa-star" style="width: 1em;"></div>
+
+.. |supporter-code| raw:: html
+
+   <div class="fa fa-code" style="width: 1em;"></div>
+
+* |supporter-gold| `Mateusz Kwiatkowski (runhyve.app) <https://runhyve.app>`_
+* |supporter-std| `Gaelan D'costa <https://www.robot-disco.net>`_
+* |supporter-std| `Tenzin Lhakhang <https://confluence.tenzin.io>`_
+* |supporter-std| `Lapo Luchini <https://github.com/lapo-luchini>`_
+* |supporter-std| `F. Schmid <https://github.com/tdschmidl>`_
+* |supporter-gold| `MNX.io <https://mnx.io>`_
+* |supporter-std| `Marshall Clyburn <https://github.com/mdclyburn>`_
+* |supporter-code| `Ross Williams <https://github.com/overhacked>`_
+* |supporter-std| Mike T.
+* |supporter-code| `Justin Scholz <https://github.com/JMoVS>`_
+* |supporter-code| `InsanePrawn <https://github.com/InsanePrawn>`_
+* |supporter-code| `Ben Woods <https://www.freshports.org/sysutils/zrepl/>`_
+* |supporter-code| `Janis Streib <https://github.com/janisstreib>`_
+* |supporter-code| `Anton Schirg <https://github.com/antonxy>`_

docs/tutorial.rst

@@ -47,14 +47,14 @@ We can model this situation as two jobs:
 Install zrepl
 -------------
 
-Follow the :ref:`OS-specific installation instructions <installation>` and come back here.
+Follow the :ref:`OS-specific installation instructions <installation_toc>` and come back here.
 
 Generate TLS Certificates
 -------------------------
 
-We use the `TLS client authentication transport <transport-tcp+tlsclientauth>` to protect our data on the wire.
+We use the :ref:`TLS client authentication transport <transport-tcp+tlsclientauth>` to protect our data on the wire.
 To get things going quickly, we skip setting up a CA and generate two self-signed certificates as described :ref:`here <transport-tcp+tlsclientauth-2machineopenssl>`.
-Again, for convenience, We generate the key pairs on our local machine and distribute them using ssh:
+For convenience, we generate the key pairs on our local machine and distribute them using ssh:
 
 .. code-block:: bash
    :emphasize-lines: 6,13
@@ -96,7 +96,7 @@ We define a **push job** named ``prod_to_backups`` in ``/etc/zrepl/zrepl.yml`` o
         key:  /etc/zrepl/prod.key
         server_cn: "backups"
       filesystems: {
-        "zroot/var/db:": true,
+        "zroot/var/db": true,
         "zroot/usr/home<": true,
         "zroot/usr/home/paranoid": false
       }
@@ -119,7 +119,7 @@ We define a **push job** named ``prod_to_backups`` in ``/etc/zrepl/zrepl.yml`` o
 Configure server ``backups``
 ----------------------------
 
-We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`` on host ``prod`` : ::
+We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`` on host ``backups`` : ::
 
     jobs:
     - name: sink
@@ -138,7 +138,7 @@ We define a corresponding **sink job** named ``sink`` in ``/etc/zrepl/zrepl.yml`
 Apply Configuration Changes
 ---------------------------
 
-We use ``zrepl configcheck`` before to catch any configuration errors: no output indicates that everything is fine.
+We use ``zrepl configcheck`` to catch any configuration errors: no output indicates that everything is fine.
 If that is the case, restart the zrepl daemon on **both** ``prod`` and ``backups`` using ``service zrepl restart`` or ``systemctl restart zrepl``.
 
 
@@ -175,6 +175,3 @@ Congratulations, you have a working push backup. Where to go next?
 
 * Read more about :ref:`configuration format, options & job types <configuration_toc>`
 * Configure :ref:`logging <logging>` \& :ref:`monitoring <monitoring>`.
-* Learn about :ref:`implementation details <implementation_toc>` of zrepl.
-
-

docs/usage.rst

@@ -38,6 +38,8 @@ CLI Overview
     * - ``zrepl migrate``
       - | perform on-disk state / ZFS property migrations
         | (see :ref:`changelog <changelog>` for details)
+    * - ``zrepl zfs-abstractions``
+      - list and remove zrepl's abstractions on top of ZFS, e.g. holds and step bookmarks (see :ref:`overview <replication-cursor-and-last-received-hold>` )
 
 .. _usage-zrepl-daemon:
 
@@ -48,7 +50,7 @@ zrepl daemon
 All actual work zrepl does is performed by a daemon process.
 The daemon supports structured :ref:`logging <logging>` and provides :ref:`monitoring endpoints <monitoring>`.
 
-When installating from a package, the package maintainer should have provided an init script / systemd.service file.
+When installing from a package, the package maintainer should have provided an init script / systemd.service file.
 You should thus be able to start zrepl daemon using your init system.
 
 Alternatively, or for running zrepl in the foreground, simply execute ``zrepl daemon``.
@@ -73,6 +75,6 @@ The daemon exits as soon as all jobs have reported shut down.
 Systemd Unit File
 ~~~~~~~~~~~~~~~~~
 
-A systemd service defintion template is available in :repomasterlink:`dist/systemd`.
+A systemd service definition template is available in :repomasterlink:`dist/systemd`.
 Note that some of the options only work on recent versions of systemd.
 Any help & improvements are very welcome, see :issue:`145`.

endpoint/endpoint.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"fmt"
 	"path"
+	"sync"
 
 	"github.com/pkg/errors"
 
@@ -15,13 +16,39 @@ import (
 	"github.com/zrepl/zrepl/zfs"
 )
 
+type SenderConfig struct {
+	FSF     zfs.DatasetFilter
+	Encrypt *zfs.NilBool
+	JobID   JobID
+}
+
+func (c *SenderConfig) Validate() error {
+	c.JobID.MustValidate()
+	if err := c.Encrypt.Validate(); err != nil {
+		return errors.Wrap(err, "`Encrypt` field invalid")
+	}
+	if _, err := StepHoldTag(c.JobID); err != nil {
+		return fmt.Errorf("JobID cannot be used for hold tag: %s", err)
+	}
+	return nil
+}
+
 // Sender implements replication.ReplicationEndpoint for a sending side
 type Sender struct {
 	FSFilter zfs.DatasetFilter
+	encrypt  *zfs.NilBool
+	jobId    JobID
 }
 
-func NewSender(fsf zfs.DatasetFilter) *Sender {
-	return &Sender{FSFilter: fsf}
+func NewSender(conf SenderConfig) *Sender {
+	if err := conf.Validate(); err != nil {
+		panic("invalid config" + err.Error())
+	}
+	return &Sender{
+		FSFilter: conf.FSF,
+		encrypt:  conf.Encrypt,
+		jobId:    conf.JobID,
+	}
 }
 
 func (s *Sender) filterCheckFS(fs string) (*zfs.DatasetPath, error) {
@@ -49,10 +76,15 @@ func (s *Sender) ListFilesystems(ctx context.Context, r *pdu.ListFilesystemReq)
 	}
 	rfss := make([]*pdu.Filesystem, len(fss))
 	for i := range fss {
+		encEnabled, err := zfs.ZFSGetEncryptionEnabled(ctx, fss[i].ToString())
+		if err != nil {
+			return nil, errors.Wrap(err, "cannot get filesystem encryption status")
+		}
 		rfss[i] = &pdu.Filesystem{
 			Path: fss[i].ToString(),
-			// FIXME: not supporting ResumeToken yet
+			// ResumeToken does not make sense from Sender
 			IsPlaceholder: false, // sender FSs are never placeholders
+			IsEncrypted:   encEnabled,
 		}
 	}
 	res := &pdu.ListFilesystemRes{Filesystems: rfss}
@@ -64,7 +96,7 @@ func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesyst
 	if err != nil {
 		return nil, err
 	}
-	fsvs, err := zfs.ZFSListFilesystemVersions(lp, nil)
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, lp, zfs.ListFilesystemVersionsOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -77,13 +109,174 @@ func (s *Sender) ListFilesystemVersions(ctx context.Context, r *pdu.ListFilesyst
 
 }
 
+func (p *Sender) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+
+	fsp, err := p.filterCheckFS(r.GetFilesystem())
+	if err != nil {
+		return nil, err
+	}
+	fs := fsp.ToString()
+
+	log := getLogger(ctx).WithField("fs", fs).WithField("hinted_most_recent", fmt.Sprintf("%#v", r.GetSenderVersion()))
+
+	log.WithField("full_hint", r).Debug("full hint")
+
+	if r.GetSenderVersion() == nil {
+		// no common ancestor found, likely due to failed prior replication attempt
+		// => release stale step holds to prevent them from accumulating
+		//    (they can accumulate on initial replication because each inital replication step might hold a different `to`)
+		// => replication cursors cannot accumulate because we always _move_ the replication cursor
+		log.Debug("releasing all step holds on the filesystem")
+		TryReleaseStepStaleFS(ctx, fs, p.jobId)
+		return &pdu.HintMostRecentCommonAncestorRes{}, nil
+	}
+
+	// we were hinted a specific common ancestor
+
+	mostRecentVersion, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, r.GetSenderVersion())
+	if err != nil {
+		msg := "HintMostRecentCommonAncestor rpc with nonexistent most recent version"
+		log.Warn(msg)
+		return nil, errors.Wrap(err, msg)
+	}
+
+	// move replication cursor to this position
+	destroyedCursors, err := MoveReplicationCursor(ctx, fs, mostRecentVersion, p.jobId)
+	if err == zfs.ErrBookmarkCloningNotSupported {
+		log.Debug("not creating replication cursor from bookmark because ZFS does not support it")
+		// fallthrough
+	} else if err != nil {
+		return nil, errors.Wrap(err, "cannot set replication cursor to hinted version")
+	}
+
+	// take care of stale step holds
+	log.WithField("step-holds-cleanup-mode", senderHintMostRecentCommonAncestorStepCleanupMode).
+		Debug("taking care of possibly stale step holds")
+	doStepCleanup := false
+	var stepCleanupSince *CreateTXGRangeBound
+	switch senderHintMostRecentCommonAncestorStepCleanupMode {
+	case StepCleanupNoCleanup:
+		doStepCleanup = false
+	case StepCleanupRangeSinceUnbounded:
+		doStepCleanup = true
+		stepCleanupSince = nil
+	case StepCleanupRangeSinceReplicationCursor:
+		doStepCleanup = true
+		// Use the destroyed replication cursors as indicator how far the previous replication got.
+		// To be precise: We limit the amount of visisted snapshots to exactly those snapshots
+		// created since the last successful replication cursor movement (i.e. last successful replication step)
+		//
+		// If we crash now, we'll leak the step we are about to release, but the performance gain
+		// of limiting the amount of snapshots we visit makes up for that.
+		// Users have the `zrepl holds release-stale` command to cleanup leaked step holds.
+		for _, destroyed := range destroyedCursors {
+			if stepCleanupSince == nil {
+				stepCleanupSince = &CreateTXGRangeBound{
+					CreateTXG: destroyed.GetCreateTXG(),
+					Inclusive: &zfs.NilBool{B: true},
+				}
+			} else if destroyed.GetCreateTXG() < stepCleanupSince.CreateTXG {
+				stepCleanupSince.CreateTXG = destroyed.GetCreateTXG()
+			}
+		}
+	default:
+		panic(senderHintMostRecentCommonAncestorStepCleanupMode)
+	}
+	if !doStepCleanup {
+		log.Info("skipping cleanup of prior invocations' step holds due to environment variable setting")
+	} else {
+		if err := ReleaseStepCummulativeInclusive(ctx, fs, stepCleanupSince, mostRecentVersion, p.jobId); err != nil {
+			return nil, errors.Wrap(err, "cannot cleanup prior invocation's step holds and bookmarks")
+		} else {
+			log.Info("step hold cleanup done")
+		}
+	}
+
+	return &pdu.HintMostRecentCommonAncestorRes{}, nil
+}
+
+type HintMostRecentCommonAncestorStepCleanupMode struct{ string }
+
+var (
+	StepCleanupRangeSinceReplicationCursor = HintMostRecentCommonAncestorStepCleanupMode{"range-since-replication-cursor"}
+	StepCleanupRangeSinceUnbounded         = HintMostRecentCommonAncestorStepCleanupMode{"range-since-unbounded"}
+	StepCleanupNoCleanup                   = HintMostRecentCommonAncestorStepCleanupMode{"no-cleanup"}
+)
+
+func (m HintMostRecentCommonAncestorStepCleanupMode) String() string { return string(m.string) }
+func (m *HintMostRecentCommonAncestorStepCleanupMode) Set(s string) error {
+	switch s {
+	case StepCleanupRangeSinceReplicationCursor.String():
+		*m = StepCleanupRangeSinceReplicationCursor
+	case StepCleanupRangeSinceUnbounded.String():
+		*m = StepCleanupRangeSinceUnbounded
+	case StepCleanupNoCleanup.String():
+		*m = StepCleanupNoCleanup
+	default:
+		return fmt.Errorf("unknown step cleanup mode %q", s)
+	}
+	return nil
+}
+
+var senderHintMostRecentCommonAncestorStepCleanupMode = *envconst.Var("ZREPL_ENDPOINT_SENDER_HINT_MOST_RECENT_STEP_HOLD_CLEANUP_MODE", &StepCleanupRangeSinceReplicationCursor).(*HintMostRecentCommonAncestorStepCleanupMode)
+
 var maxConcurrentZFSSendSemaphore = semaphore.New(envconst.Int64("ZREPL_ENDPOINT_MAX_CONCURRENT_SEND", 10))
 
+func uncheckedSendArgsFromPDU(fsv *pdu.FilesystemVersion) *zfs.ZFSSendArgVersion {
+	if fsv == nil {
+		return nil
+	}
+	return &zfs.ZFSSendArgVersion{RelName: fsv.GetRelName(), GUID: fsv.Guid}
+}
+
+func sendArgsFromPDUAndValidateExistsAndGetVersion(ctx context.Context, fs string, fsv *pdu.FilesystemVersion) (v zfs.FilesystemVersion, err error) {
+	sendArgs := uncheckedSendArgsFromPDU(fsv)
+	if sendArgs == nil {
+		return v, errors.New("must not be nil")
+	}
+	version, err := sendArgs.ValidateExistsAndGetVersion(ctx, fs)
+	if err != nil {
+		return v, err
+	}
+	return version, nil
+}
+
 func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.StreamCopier, error) {
+
 	_, err := s.filterCheckFS(r.Filesystem)
 	if err != nil {
 		return nil, nil, err
 	}
+	switch r.Encrypted {
+	case pdu.Tri_DontCare:
+		// use s.encrypt setting
+		// ok, fallthrough outer
+	case pdu.Tri_False:
+		if s.encrypt.B {
+			return nil, nil, errors.New("only encrypted sends allowed (send -w + encryption!= off), but unencrypted send requested")
+		}
+		// fallthrough outer
+	case pdu.Tri_True:
+		if !s.encrypt.B {
+			return nil, nil, errors.New("only unencrypted sends allowed, but encrypted send requested")
+		}
+		// fallthrough outer
+	default:
+		return nil, nil, fmt.Errorf("unknown pdu.Tri variant %q", r.Encrypted)
+	}
+
+	sendArgsUnvalidated := zfs.ZFSSendArgsUnvalidated{
+		FS:          r.Filesystem,
+		From:        uncheckedSendArgsFromPDU(r.GetFrom()), // validated by zfs.ZFSSendDry / zfs.ZFSSend
+		To:          uncheckedSendArgsFromPDU(r.GetTo()),   // validated by zfs.ZFSSendDry / zfs.ZFSSend
+		Encrypted:   s.encrypt,
+		ResumeToken: r.ResumeToken, // nil or not nil, depending on decoding success
+	}
+
+	sendArgs, err := sendArgsUnvalidated.Validate(ctx)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "validate send arguments")
+	}
 
 	getLogger(ctx).Debug("acquire concurrent send semaphore")
 	// TODO use try-acquire and fail with resource-exhaustion rpc status
@@ -95,28 +288,155 @@ func (s *Sender) Send(ctx context.Context, r *pdu.SendReq) (*pdu.SendRes, zfs.St
 	}
 	defer guard.Release()
 
-	si, err := zfs.ZFSSendDry(r.Filesystem, r.From, r.To, "")
+	si, err := zfs.ZFSSendDry(ctx, sendArgs)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Wrap(err, "zfs send dry failed")
 	}
 
+	// From now on, assume that sendArgs has been validated by ZFSSendDry
+	// (because validation involves shelling out, it's actually a little expensive)
+
 	var expSize int64 = 0      // protocol says 0 means no estimate
 	if si.SizeEstimate != -1 { // but si returns -1 for no size estimate
 		expSize = si.SizeEstimate
 	}
-	res := &pdu.SendRes{ExpectedSize: expSize}
+	res := &pdu.SendRes{
+		ExpectedSize:    expSize,
+		UsedResumeToken: r.ResumeToken != "",
+	}
 
 	if r.DryRun {
 		return res, nil, nil
 	}
 
-	streamCopier, err := zfs.ZFSSend(ctx, r.Filesystem, r.From, r.To, "")
+	// update replication cursor
+	if sendArgs.From != nil {
+		// For all but the first replication, this should always be a no-op because SendCompleted already moved the cursor
+		_, err = MoveReplicationCursor(ctx, sendArgs.FS, sendArgs.FromVersion, s.jobId)
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			getLogger(ctx).Debug("not creating replication cursor from bookmark because ZFS does not support it")
+			// fallthrough
+		} else if err != nil {
+			return nil, nil, errors.Wrap(err, "cannot set replication cursor to `from` version before starting send")
+		}
+	}
+
+	// make sure `From` doesn't go away in order to make this step resumable
+	if sendArgs.From != nil {
+		_, err := HoldStep(ctx, sendArgs.FS, *sendArgs.FromVersion, s.jobId)
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			getLogger(ctx).Debug("not creating step bookmark because ZFS does not support it")
+			// fallthrough
+		} else if err != nil {
+			return nil, nil, errors.Wrapf(err, "cannot hold `from` version %q before starting send", *sendArgs.FromVersion)
+		}
+	}
+	// make sure `To` doesn't go away in order to make this step resumable
+	_, err = HoldStep(ctx, sendArgs.FS, sendArgs.ToVersion, s.jobId)
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, errors.Wrapf(err, "cannot hold `to` version %q before starting send", sendArgs.ToVersion)
+	}
+
+	// step holds & replication cursor released / moved forward in s.SendCompleted => s.moveCursorAndReleaseSendHolds
+
+	streamCopier, err := zfs.ZFSSend(ctx, sendArgs)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "zfs send failed")
 	}
 	return res, streamCopier, nil
 }
 
+func (p *Sender) SendCompleted(ctx context.Context, r *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+
+	orig := r.GetOriginalReq() // may be nil, always use proto getters
+	fsp, err := p.filterCheckFS(orig.GetFilesystem())
+	if err != nil {
+		return nil, err
+	}
+	fs := fsp.ToString()
+
+	var from *zfs.FilesystemVersion
+	if orig.GetFrom() != nil {
+		f, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, orig.GetFrom()) // no shadow
+		if err != nil {
+			return nil, errors.Wrap(err, "validate `from` exists")
+		}
+		from = &f
+	}
+	to, err := sendArgsFromPDUAndValidateExistsAndGetVersion(ctx, fs, orig.GetTo())
+	if err != nil {
+		return nil, errors.Wrap(err, "validate `to` exists")
+	}
+
+	log := getLogger(ctx).WithField("to_guid", to.Guid).
+		WithField("fs", fs).
+		WithField("to", to.RelName)
+	if from != nil {
+		log = log.WithField("from", from.RelName).WithField("from_guid", from.Guid)
+	}
+
+	log.Debug("move replication cursor to most recent common version")
+	destroyedCursors, err := MoveReplicationCursor(ctx, fs, to, p.jobId)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			log.Debug("not setting replication cursor, bookmark cloning not supported")
+		} else {
+			msg := "cannot move replication cursor, keeping hold on `to` until successful"
+			log.WithError(err).Error(msg)
+			err = errors.Wrap(err, msg)
+			// it is correct to not release the hold if we can't move the cursor!
+			return &pdu.SendCompletedRes{}, err
+		}
+	} else {
+		log.Info("successfully moved replication cursor")
+	}
+
+	// kick off releasing of step holds / bookmarks
+	// if we fail to release them, don't bother the caller:
+	// they are merely an implementation detail on the sender for better resumability
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
+		log.Debug("release step-hold of or step-bookmark on `to`")
+		err = ReleaseStep(ctx, fs, to, p.jobId)
+		if err != nil {
+			log.WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `to`")
+		} else {
+			log.Info("successfully released step-holds on or destroyed step-bookmark of `to`")
+		}
+
+	}()
+	go func() {
+		defer wg.Done()
+		if from == nil {
+			return
+		}
+		log.Debug("release step-hold of or step-bookmark on `from`")
+		err := ReleaseStep(ctx, fs, *from, p.jobId)
+		if err != nil {
+			if dne, ok := err.(*zfs.DatasetDoesNotExist); ok {
+				// If bookmark cloning is not supported, `from` might be the old replication cursor
+				// and thus have already been destroyed by MoveReplicationCursor above
+				// In that case, nonexistence of `from` is not an error, otherwise it is.
+				for _, c := range destroyedCursors {
+					if c.GetFullPath() == dne.Path {
+						log.Info("`from` was a replication cursor and has already been destroyed")
+						return
+					}
+				}
+				// fallthrough
+			}
+			log.WithError(err).Error("cannot release step-holds on or destroy step-bookmark of `from`")
+		} else {
+			log.Info("successfully released step-holds on or destroyed step-bookmark of `from`")
+		}
+	}()
+	wg.Wait()
+
+	return &pdu.SendCompletedRes{}, nil
+}
+
 func (p *Sender) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapshotsReq) (*pdu.DestroySnapshotsRes, error) {
 	dp, err := p.filterCheckFS(req.Filesystem)
 	if err != nil {
@@ -146,25 +466,14 @@ func (p *Sender) ReplicationCursor(ctx context.Context, req *pdu.ReplicationCurs
 		return nil, err
 	}
 
-	switch op := req.Op.(type) {
-	case *pdu.ReplicationCursorReq_Get:
-		cursor, err := zfs.ZFSGetReplicationCursor(dp)
-		if err != nil {
-			return nil, err
-		}
-		if cursor == nil {
-			return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
-		}
-		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: cursor.Guid}}, nil
-	case *pdu.ReplicationCursorReq_Set:
-		guid, err := zfs.ZFSSetReplicationCursor(dp, op.Set.Snapshot)
-		if err != nil {
-			return nil, err
-		}
-		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: guid}}, nil
-	default:
-		return nil, errors.Errorf("unknown op %T", op)
+	cursor, err := GetMostRecentReplicationCursorOfJob(ctx, dp.ToString(), p.jobId)
+	if err != nil {
+		return nil, err
 	}
+	if cursor == nil {
+		return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Notexist{Notexist: true}}, nil
+	}
+	return &pdu.ReplicationCursorRes{Result: &pdu.ReplicationCursorRes_Guid{Guid: cursor.Guid}}, nil
 }
 
 func (p *Sender) Receive(ctx context.Context, r *pdu.ReceiveReq, receive zfs.StreamCopier) (*pdu.ReceiveRes, error) {
@@ -183,22 +492,42 @@ type FSMap interface { // FIXME unused
 	AsFilter() FSFilter
 }
 
+type ReceiverConfig struct {
+	JobID JobID
+
+	RootWithoutClientComponent *zfs.DatasetPath // TODO use
+	AppendClientIdentity       bool
+
+	UpdateLastReceivedHold bool
+}
+
+func (c *ReceiverConfig) copyIn() {
+	c.RootWithoutClientComponent = c.RootWithoutClientComponent.Copy()
+}
+
+func (c *ReceiverConfig) Validate() error {
+	c.JobID.MustValidate()
+	if c.RootWithoutClientComponent.Length() <= 0 {
+		return errors.New("RootWithoutClientComponent must not be an empty dataset path")
+	}
+	return nil
+}
+
 // Receiver implements replication.ReplicationEndpoint for a receiving side
 type Receiver struct {
-	rootWithoutClientComponent *zfs.DatasetPath
-	appendClientIdentity       bool
+	conf ReceiverConfig // validated
 
 	recvParentCreationMtx *chainlock.L
 }
 
-func NewReceiver(rootDataset *zfs.DatasetPath, appendClientIdentity bool) *Receiver {
-	if rootDataset.Length() <= 0 {
-		panic(fmt.Sprintf("root dataset must not be an empty path: %v", rootDataset))
+func NewReceiver(config ReceiverConfig) *Receiver {
+	config.copyIn()
+	if err := config.Validate(); err != nil {
+		panic(err)
 	}
 	return &Receiver{
-		rootWithoutClientComponent: rootDataset.Copy(),
-		appendClientIdentity:       appendClientIdentity,
-		recvParentCreationMtx:      chainlock.New(),
+		conf:                  config,
+		recvParentCreationMtx: chainlock.New(),
 	}
 }
 
@@ -221,8 +550,8 @@ func clientRoot(rootFS *zfs.DatasetPath, clientIdentity string) (*zfs.DatasetPat
 }
 
 func (s *Receiver) clientRootFromCtx(ctx context.Context) *zfs.DatasetPath {
-	if !s.appendClientIdentity {
-		return s.rootWithoutClientComponent.Copy()
+	if !s.conf.AppendClientIdentity {
+		return s.conf.RootWithoutClientComponent.Copy()
 	}
 
 	clientIdentity, ok := ctx.Value(ClientIdentityKey).(string)
@@ -230,7 +559,7 @@ func (s *Receiver) clientRootFromCtx(ctx context.Context) *zfs.DatasetPath {
 		panic(fmt.Sprintf("ClientIdentityKey context value must be set"))
 	}
 
-	clientRoot, err := clientRoot(s.rootWithoutClientComponent, clientIdentity)
+	clientRoot, err := clientRoot(s.conf.RootWithoutClientComponent, clientIdentity)
 	if err != nil {
 		panic(fmt.Sprintf("ClientIdentityContextKey must have been validated before invoking Receiver: %s", err))
 	}
@@ -271,7 +600,7 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
 	fss := make([]*pdu.Filesystem, 0, len(filtered))
 	for _, a := range filtered {
 		l := getLogger(ctx).WithField("fs", a)
-		ph, err := zfs.ZFSGetFilesystemPlaceholderState(a)
+		ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, a)
 		if err != nil {
 			l.WithError(err).Error("error getting placeholder state")
 			return nil, errors.Wrapf(err, "cannot get placeholder state for fs %q", a)
@@ -282,8 +611,27 @@ func (s *Receiver) ListFilesystems(ctx context.Context, req *pdu.ListFilesystemR
 			err := errors.Errorf("inconsistent placeholder state: filesystem %q must exist in this context", a.ToString())
 			return nil, err
 		}
+		token, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, a)
+		if err != nil {
+			l.WithError(err).Error("cannot get receive resume token")
+			return nil, err
+		}
+		encEnabled, err := zfs.ZFSGetEncryptionEnabled(ctx, a.ToString())
+		if err != nil {
+			l.WithError(err).Error("cannot get encryption enabled status")
+			return nil, err
+		}
+		l.WithField("receive_resume_token", token).Debug("receive resume token")
+
 		a.TrimPrefix(root)
-		fss = append(fss, &pdu.Filesystem{Path: a.ToString(), IsPlaceholder: ph.IsPlaceholder})
+
+		fs := &pdu.Filesystem{
+			Path:          a.ToString(),
+			IsPlaceholder: ph.IsPlaceholder,
+			ResumeToken:   token,
+			IsEncrypted:   encEnabled,
+		}
+		fss = append(fss, fs)
 	}
 	if len(fss) == 0 {
 		getLogger(ctx).Debug("no filesystems found")
@@ -298,8 +646,9 @@ func (s *Receiver) ListFilesystemVersions(ctx context.Context, req *pdu.ListFile
 	if err != nil {
 		return nil, err
 	}
+	// TODO share following code with sender
 
-	fsvs, err := zfs.ZFSListFilesystemVersions(lp, nil)
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, lp, zfs.ListFilesystemVersionsOptions{})
 	if err != nil {
 		return nil, err
 	}
@@ -344,7 +693,15 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 	root := s.clientRootFromCtx(ctx)
 	lp, err := subroot{root}.MapToLocal(req.Filesystem)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrap(err, "`Filesystem` invalid")
+	}
+
+	to := uncheckedSendArgsFromPDU(req.GetTo())
+	if to == nil {
+		return nil, errors.New("`To` must not be nil")
+	}
+	if !to.IsSnapshot() {
+		return nil, errors.New("`To` must be a snapshot")
 	}
 
 	// create placeholder parent filesystems as appropriate
@@ -355,9 +712,9 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 	//       ZFS dataset hierarchy subtrees.
 	var visitErr error
 	func() {
-		getLogger(ctx).Debug("begin aquire recvParentCreationMtx")
+		getLogger(ctx).Debug("begin acquire recvParentCreationMtx")
 		defer s.recvParentCreationMtx.Lock().Unlock()
-		getLogger(ctx).Debug("end aquire recvParentCreationMtx")
+		getLogger(ctx).Debug("end acquire recvParentCreationMtx")
 		defer getLogger(ctx).Debug("release recvParentCreationMtx")
 
 		f := zfs.NewDatasetPathForest()
@@ -367,7 +724,7 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 			if v.Path.Equal(lp) {
 				return false
 			}
-			ph, err := zfs.ZFSGetFilesystemPlaceholderState(v.Path)
+			ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, v.Path)
 			getLogger(ctx).
 				WithField("fs", v.Path.ToString()).
 				WithField("placeholder_state", fmt.Sprintf("%#v", ph)).
@@ -380,18 +737,18 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 			}
 
 			if !ph.FSExists {
-				if s.rootWithoutClientComponent.HasPrefix(v.Path) {
+				if s.conf.RootWithoutClientComponent.HasPrefix(v.Path) {
 					if v.Path.Length() == 1 {
 						visitErr = fmt.Errorf("pool %q not imported", v.Path.ToString())
 					} else {
-						visitErr = fmt.Errorf("root_fs %q does not exist", s.rootWithoutClientComponent.ToString())
+						visitErr = fmt.Errorf("root_fs %q does not exist", s.conf.RootWithoutClientComponent.ToString())
 					}
 					getLogger(ctx).WithError(visitErr).Error("placeholders are only created automatically below root_fs")
 					return false
 				}
 				l := getLogger(ctx).WithField("placeholder_fs", v.Path)
 				l.Debug("create placeholder filesystem")
-				err := zfs.ZFSCreatePlaceholderFilesystem(v.Path)
+				err := zfs.ZFSCreatePlaceholderFilesystem(ctx, v.Path)
 				if err != nil {
 					l.WithError(err).Error("cannot create placeholder filesystem")
 					visitErr = err
@@ -411,17 +768,28 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 	// determine whether we need to rollback the filesystem / change its placeholder state
 	var clearPlaceholderProperty bool
 	var recvOpts zfs.RecvOptions
-	ph, err := zfs.ZFSGetFilesystemPlaceholderState(lp)
+	ph, err := zfs.ZFSGetFilesystemPlaceholderState(ctx, lp)
 	if err == nil && ph.FSExists && ph.IsPlaceholder {
 		recvOpts.RollbackAndForceRecv = true
 		clearPlaceholderProperty = true
 	}
 	if clearPlaceholderProperty {
-		if err := zfs.ZFSSetPlaceholder(lp, false); err != nil {
+		if err := zfs.ZFSSetPlaceholder(ctx, lp, false); err != nil {
 			return nil, fmt.Errorf("cannot clear placeholder property for forced receive: %s", err)
 		}
 	}
 
+	if req.ClearResumeToken && ph.FSExists {
+		if err := zfs.ZFSRecvClearResumeToken(ctx, lp.ToString()); err != nil {
+			return nil, errors.Wrap(err, "cannot clear resume token")
+		}
+	}
+
+	recvOpts.SavePartialRecvState, err = zfs.ResumeRecvSupported(ctx, lp)
+	if err != nil {
+		return nil, errors.Wrap(err, "cannot determine whether we can use resumable send & recv")
+	}
+
 	getLogger(ctx).Debug("acquire concurrent recv semaphore")
 	// TODO use try-acquire and fail with resource-exhaustion rpc status
 	// => would require handling on the client-side
@@ -434,13 +802,31 @@ func (s *Receiver) Receive(ctx context.Context, req *pdu.ReceiveReq, receive zfs
 
 	getLogger(ctx).WithField("opts", fmt.Sprintf("%#v", recvOpts)).Debug("start receive command")
 
-	if err := zfs.ZFSRecv(ctx, lp.ToString(), receive, recvOpts); err != nil {
+	snapFullPath := to.FullPath(lp.ToString())
+	if err := zfs.ZFSRecv(ctx, lp.ToString(), to, receive, recvOpts); err != nil {
 		getLogger(ctx).
 			WithError(err).
-			WithField("opts", recvOpts).
+			WithField("opts", fmt.Sprintf("%#v", recvOpts)).
 			Error("zfs receive failed")
 		return nil, err
 	}
+
+	// validate that we actually received what the sender claimed
+	toRecvd, err := to.ValidateExistsAndGetVersion(ctx, lp.ToString())
+	if err != nil {
+		msg := "receive request's `To` version does not match what we received in the stream"
+		getLogger(ctx).WithError(err).WithField("snap", snapFullPath).Error(msg)
+		getLogger(ctx).Error("aborting recv request, but keeping received snapshot for inspection")
+		return nil, errors.Wrap(err, msg)
+	}
+
+	if s.conf.UpdateLastReceivedHold {
+		getLogger(ctx).Debug("move last-received-hold")
+		if err := MoveLastReceivedHold(ctx, lp.ToString(), toRecvd, s.conf.JobID); err != nil {
+			return nil, errors.Wrap(err, "cannot move last-received-hold")
+		}
+	}
+
 	return &pdu.ReceiveRes{}, nil
 }
 
@@ -453,6 +839,19 @@ func (s *Receiver) DestroySnapshots(ctx context.Context, req *pdu.DestroySnapsho
 	return doDestroySnapshots(ctx, lp, req.Snapshots)
 }
 
+func (p *Receiver) HintMostRecentCommonAncestor(ctx context.Context, r *pdu.HintMostRecentCommonAncestorReq) (*pdu.HintMostRecentCommonAncestorRes, error) {
+	// we don't move last-received-hold as part of this hint
+	// because that wouldn't give us any benefit wrt resumability.
+	//
+	// Other reason: the replication logic that issues this RPC would require refactoring
+	// to include the receiver's FilesystemVersion in the request)
+	return &pdu.HintMostRecentCommonAncestorRes{}, nil
+}
+
+func (p *Receiver) SendCompleted(context.Context, *pdu.SendCompletedReq) (*pdu.SendCompletedRes, error) {
+	return &pdu.SendCompletedRes{}, nil
+}
+
 func doDestroySnapshots(ctx context.Context, lp *zfs.DatasetPath, snaps []*pdu.FilesystemVersion) (*pdu.DestroySnapshotsRes, error) {
 	reqs := make([]*zfs.DestroySnapOp, len(snaps))
 	ress := make([]*pdu.DestroySnapshotRes, len(snaps))

endpoint/endpoint_zfs_abstraction.go

@@ -0,0 +1,838 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"math"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/util/envconst"
+	"github.com/zrepl/zrepl/util/semaphore"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type AbstractionType string
+
+// Implementation note:
+// There are a lot of exhaustive switches on AbstractionType in the code base.
+// When adding a new abstraction type, make sure to search and update them!
+const (
+	AbstractionStepBookmark                AbstractionType = "step-bookmark"
+	AbstractionStepHold                    AbstractionType = "step-hold"
+	AbstractionLastReceivedHold            AbstractionType = "last-received-hold"
+	AbstractionReplicationCursorBookmarkV1 AbstractionType = "replication-cursor-bookmark-v1"
+	AbstractionReplicationCursorBookmarkV2 AbstractionType = "replication-cursor-bookmark-v2"
+)
+
+var AbstractionTypesAll = map[AbstractionType]bool{
+	AbstractionStepBookmark:                true,
+	AbstractionStepHold:                    true,
+	AbstractionLastReceivedHold:            true,
+	AbstractionReplicationCursorBookmarkV1: true,
+	AbstractionReplicationCursorBookmarkV2: true,
+}
+
+// Implementation Note:
+// Whenever you add a new accessor, adjust AbstractionJSON.MarshalJSON accordingly
+type Abstraction interface {
+	GetType() AbstractionType
+	GetFS() string
+	GetName() string
+	GetFullPath() string
+	GetJobID() *JobID // may return nil if the abstraction does not have a JobID
+	GetCreateTXG() uint64
+	GetFilesystemVersion() zfs.FilesystemVersion
+	String() string
+	// destroy the abstraction: either releases the hold or destroys the bookmark
+	Destroy(context.Context) error
+	json.Marshaler
+}
+
+func (t AbstractionType) Validate() error {
+	switch t {
+	case AbstractionStepBookmark:
+		return nil
+	case AbstractionStepHold:
+		return nil
+	case AbstractionLastReceivedHold:
+		return nil
+	case AbstractionReplicationCursorBookmarkV1:
+		return nil
+	case AbstractionReplicationCursorBookmarkV2:
+		return nil
+	default:
+		return errors.Errorf("unknown abstraction type %q", t)
+	}
+}
+
+func (t AbstractionType) MustValidate() error {
+	if err := t.Validate(); err != nil {
+		panic(err)
+	}
+	return nil
+}
+
+type AbstractionJSON struct{ Abstraction }
+
+var _ json.Marshaler = (*AbstractionJSON)(nil)
+
+func (a AbstractionJSON) MarshalJSON() ([]byte, error) {
+	type S struct {
+		Type              AbstractionType
+		FS                string
+		Name              string
+		FullPath          string
+		JobID             *JobID // may return nil if the abstraction does not have a JobID
+		CreateTXG         uint64
+		FilesystemVersion zfs.FilesystemVersion
+		String            string
+	}
+	v := S{
+		Type:              a.Abstraction.GetType(),
+		FS:                a.Abstraction.GetFS(),
+		Name:              a.Abstraction.GetName(),
+		FullPath:          a.Abstraction.GetFullPath(),
+		JobID:             a.Abstraction.GetJobID(),
+		CreateTXG:         a.Abstraction.GetCreateTXG(),
+		FilesystemVersion: a.Abstraction.GetFilesystemVersion(),
+		String:            a.Abstraction.String(),
+	}
+	return json.Marshal(v)
+}
+
+type AbstractionTypeSet map[AbstractionType]bool
+
+func AbstractionTypeSetFromStrings(sts []string) (AbstractionTypeSet, error) {
+	ats := make(map[AbstractionType]bool, len(sts))
+	for i, t := range sts {
+		at := AbstractionType(t)
+		if err := at.Validate(); err != nil {
+			return nil, errors.Wrapf(err, "invalid abstraction type #%d %q", i+1, t)
+		}
+		ats[at] = true
+	}
+	return ats, nil
+}
+
+func (s AbstractionTypeSet) ContainsAll(q AbstractionTypeSet) bool {
+	for k := range q {
+		if _, ok := s[k]; !ok {
+			return false
+		}
+	}
+	return true
+}
+
+func (s AbstractionTypeSet) ContainsAnyOf(q AbstractionTypeSet) bool {
+	for k := range q {
+		if _, ok := s[k]; ok {
+			return true
+		}
+	}
+	return false
+}
+
+func (s AbstractionTypeSet) String() string {
+	sts := make([]string, 0, len(s))
+	for i := range s {
+		sts = append(sts, string(i))
+	}
+	sts = sort.StringSlice(sts)
+	return strings.Join(sts, ",")
+}
+
+func (s AbstractionTypeSet) Validate() error {
+	for k := range s {
+		if err := k.Validate(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type BookmarkExtractor func(fs *zfs.DatasetPath, v zfs.FilesystemVersion) Abstraction
+
+// returns nil if the abstraction type is not bookmark-based
+func (t AbstractionType) BookmarkExtractor() BookmarkExtractor {
+	switch t {
+	case AbstractionStepBookmark:
+		return StepBookmarkExtractor
+	case AbstractionReplicationCursorBookmarkV1:
+		return ReplicationCursorV1Extractor
+	case AbstractionReplicationCursorBookmarkV2:
+		return ReplicationCursorV2Extractor
+	case AbstractionStepHold:
+		return nil
+	case AbstractionLastReceivedHold:
+		return nil
+	default:
+		panic(fmt.Sprintf("unimpl: %q", t))
+	}
+}
+
+type HoldExtractor = func(fs *zfs.DatasetPath, v zfs.FilesystemVersion, tag string) Abstraction
+
+// returns nil if the abstraction type is not hold-based
+func (t AbstractionType) HoldExtractor() HoldExtractor {
+	switch t {
+	case AbstractionStepBookmark:
+		return nil
+	case AbstractionReplicationCursorBookmarkV1:
+		return nil
+	case AbstractionReplicationCursorBookmarkV2:
+		return nil
+	case AbstractionStepHold:
+		return StepHoldExtractor
+	case AbstractionLastReceivedHold:
+		return LastReceivedHoldExtractor
+	default:
+		panic(fmt.Sprintf("unimpl: %q", t))
+	}
+}
+
+type ListZFSHoldsAndBookmarksQuery struct {
+	FS ListZFSHoldsAndBookmarksQueryFilesystemFilter
+	// What abstraction types should match (any contained in the set)
+	What AbstractionTypeSet
+
+	// The output for the query must satisfy _all_ (AND) requirements of all fields in this query struct.
+
+	// if not nil: JobID of the hold or bookmark in question must be equal
+	// else: JobID of the hold or bookmark can be any value
+	JobID *JobID
+
+	// zero-value means any CreateTXG is acceptable
+	CreateTXG CreateTXGRange
+
+	// Number of concurrently queried filesystems. Must be >= 1
+	Concurrency int64
+}
+
+type CreateTXGRangeBound struct {
+	CreateTXG uint64
+	Inclusive *zfs.NilBool // must not be nil
+}
+
+// A non-empty range of CreateTXGs
+//
+// If both Since and Until are nil, any CreateTXG is acceptable
+type CreateTXGRange struct {
+	// if not nil: The hold's snapshot or the bookmark's createtxg must be greater than (or equal) Since
+	// else: CreateTXG of the hold or bookmark can be any value accepted by Until
+	Since *CreateTXGRangeBound
+	// if not nil: The hold's snapshot or the bookmark's createtxg must be less than (or equal) Until
+	// else: CreateTXG of the hold or bookmark can be any value accepted by Since
+	Until *CreateTXGRangeBound
+}
+
+// FS == nil XOR Filter == nil
+type ListZFSHoldsAndBookmarksQueryFilesystemFilter struct {
+	FS     *string
+	Filter zfs.DatasetFilter
+}
+
+func (q *ListZFSHoldsAndBookmarksQuery) Validate() error {
+	if err := q.FS.Validate(); err != nil {
+		return errors.Wrap(err, "FS")
+	}
+	if q.JobID != nil {
+		q.JobID.MustValidate() // FIXME
+	}
+	if err := q.CreateTXG.Validate(); err != nil {
+		return errors.Wrap(err, "CreateTXGRange")
+	}
+	if err := q.What.Validate(); err != nil {
+		return err
+	}
+	if q.Concurrency < 1 {
+		return errors.New("Concurrency must be >= 1")
+	}
+	return nil
+}
+
+var createTXGRangeBoundAllowCreateTXG0 = envconst.Bool("ZREPL_ENDPOINT_LIST_ABSTRACTIONS_QUERY_CREATETXG_RANGE_BOUND_ALLOW_0", false)
+
+func (i *CreateTXGRangeBound) Validate() error {
+	if err := i.Inclusive.Validate(); err != nil {
+		return errors.Wrap(err, "Inclusive")
+	}
+	if i.CreateTXG == 0 && !createTXGRangeBoundAllowCreateTXG0 {
+		return errors.New("CreateTXG must be non-zero")
+	}
+	return nil
+
+}
+
+func (f *ListZFSHoldsAndBookmarksQueryFilesystemFilter) Validate() error {
+	if f == nil {
+		return nil
+	}
+	fsSet := f.FS != nil
+	filterSet := f.Filter != nil
+	if fsSet && filterSet || !fsSet && !filterSet {
+		return fmt.Errorf("must set FS or Filter field, but fsIsSet=%v and filterIsSet=%v", fsSet, filterSet)
+	}
+	if fsSet {
+		if err := zfs.EntityNamecheck(*f.FS, zfs.EntityTypeFilesystem); err != nil {
+			return errors.Wrap(err, "FS invalid")
+		}
+	}
+	return nil
+}
+
+func (f *ListZFSHoldsAndBookmarksQueryFilesystemFilter) Filesystems(ctx context.Context) ([]string, error) {
+	if err := f.Validate(); err != nil {
+		panic(err)
+	}
+	if f.FS != nil {
+		return []string{*f.FS}, nil
+	}
+	if f.Filter != nil {
+		dps, err := zfs.ZFSListMapping(ctx, f.Filter)
+		if err != nil {
+			return nil, err
+		}
+		fss := make([]string, len(dps))
+		for i, dp := range dps {
+			fss[i] = dp.ToString()
+		}
+		return fss, nil
+	}
+	panic("unreachable")
+}
+
+func (r *CreateTXGRange) Validate() error {
+	if r.Since != nil {
+		if err := r.Since.Validate(); err != nil {
+			return errors.Wrap(err, "Since")
+		}
+	}
+	if r.Until != nil {
+		if err := r.Until.Validate(); err != nil {
+			return errors.Wrap(err, "Until")
+		}
+	}
+	if _, err := r.effectiveBounds(); err != nil {
+		return errors.Wrapf(err, "specified range %s is semantically invalid", r)
+	}
+	return nil
+}
+
+// inclusive-inclusive bounds
+type effectiveBounds struct {
+	sinceInclusive uint64
+	sinceUnbounded bool
+	untilInclusive uint64
+	untilUnbounded bool
+}
+
+// callers must have validated r.Since and r.Until before calling this method
+func (r *CreateTXGRange) effectiveBounds() (bounds effectiveBounds, err error) {
+
+	bounds.sinceUnbounded = r.Since == nil
+	bounds.untilUnbounded = r.Until == nil
+
+	if r.Since == nil && r.Until == nil {
+		return bounds, nil
+	}
+
+	if r.Since != nil {
+		bounds.sinceInclusive = r.Since.CreateTXG
+		if !r.Since.Inclusive.B {
+			if r.Since.CreateTXG == math.MaxUint64 {
+				return bounds, errors.Errorf("Since-exclusive (%v) must be less than math.MaxUint64 (%v)",
+					r.Since.CreateTXG, uint64(math.MaxUint64))
+			}
+			bounds.sinceInclusive++
+		}
+	}
+
+	if r.Until != nil {
+		bounds.untilInclusive = r.Until.CreateTXG
+		if !r.Until.Inclusive.B {
+			if r.Until.CreateTXG == 0 {
+				return bounds, errors.Errorf("Until-exclusive (%v) must be greater than 0", r.Until.CreateTXG)
+			}
+			bounds.untilInclusive--
+		}
+	}
+
+	if !bounds.sinceUnbounded && !bounds.untilUnbounded {
+		if bounds.sinceInclusive >= bounds.untilInclusive {
+			return bounds, errors.Errorf("effective range bounds are [%v,%v] which is empty or invalid", bounds.sinceInclusive, bounds.untilInclusive)
+		}
+		// fallthrough
+	}
+
+	return bounds, nil
+}
+
+func (r *CreateTXGRange) String() string {
+	var buf strings.Builder
+	if r.Since == nil {
+		fmt.Fprintf(&buf, "~")
+	} else {
+		if err := r.Since.Inclusive.Validate(); err != nil {
+			fmt.Fprintf(&buf, "?")
+		} else if r.Since.Inclusive.B {
+			fmt.Fprintf(&buf, "[")
+		} else {
+			fmt.Fprintf(&buf, "(")
+		}
+		fmt.Fprintf(&buf, "%d", r.Since.CreateTXG)
+	}
+
+	fmt.Fprintf(&buf, ",")
+
+	if r.Until == nil {
+		fmt.Fprintf(&buf, "~")
+	} else {
+		fmt.Fprintf(&buf, "%d", r.Until.CreateTXG)
+		if err := r.Until.Inclusive.Validate(); err != nil {
+			fmt.Fprintf(&buf, "?")
+		} else if r.Until.Inclusive.B {
+			fmt.Fprintf(&buf, "]")
+		} else {
+			fmt.Fprintf(&buf, ")")
+		}
+	}
+
+	return buf.String()
+}
+
+// panics if not .Validate()
+func (r *CreateTXGRange) IsUnbounded() bool {
+	if err := r.Validate(); err != nil {
+		panic(err)
+	}
+	bounds, err := r.effectiveBounds()
+	if err != nil {
+		panic(err)
+	}
+	return bounds.sinceUnbounded && bounds.untilUnbounded
+}
+
+// panics if not .Validate()
+func (r *CreateTXGRange) Contains(qCreateTxg uint64) bool {
+	if err := r.Validate(); err != nil {
+		panic(err)
+	}
+
+	bounds, err := r.effectiveBounds()
+	if err != nil {
+		panic(err)
+	}
+
+	sinceMatches := bounds.sinceUnbounded || bounds.sinceInclusive <= qCreateTxg
+	untilMatches := bounds.untilUnbounded || qCreateTxg <= bounds.untilInclusive
+
+	return sinceMatches && untilMatches
+}
+
+type ListAbstractionsError struct {
+	FS   string
+	Snap string
+	What string
+	Err  error
+}
+
+func (e ListAbstractionsError) Error() string {
+	if e.FS == "" {
+		return fmt.Sprintf("list endpoint abstractions: %s: %s", e.What, e.Err)
+	} else {
+		v := e.FS
+		if e.Snap != "" {
+			v = fmt.Sprintf("%s@%s", e.FS, e.Snap)
+		}
+		return fmt.Sprintf("list endpoint abstractions on %q: %s: %s", v, e.What, e.Err)
+	}
+}
+
+type putListAbstractionErr func(err error, fs string, what string)
+type putListAbstraction func(a Abstraction)
+
+type ListAbstractionsErrors []ListAbstractionsError
+
+func (e ListAbstractionsErrors) Error() string {
+	if len(e) == 0 {
+		panic(e)
+	}
+	if len(e) == 1 {
+		return fmt.Sprintf("list endpoint abstractions: %s", e[0])
+	}
+	msgs := make([]string, len(e))
+	for i := range e {
+		msgs[i] = e[i].Error()
+	}
+	return fmt.Sprintf("list endpoint abstractions: multiple errors:\n%s", strings.Join(msgs, "\n"))
+}
+
+func ListAbstractions(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (out []Abstraction, outErrs []ListAbstractionsError, err error) {
+	outChan, outErrsChan, err := ListAbstractionsStreamed(ctx, query)
+	if err != nil {
+		return nil, nil, err
+	}
+	var wg sync.WaitGroup
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for a := range outChan {
+			out = append(out, a)
+		}
+	}()
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for err := range outErrsChan {
+			outErrs = append(outErrs, err)
+		}
+	}()
+	wg.Wait()
+	return out, outErrs, nil
+}
+
+// if err != nil, the returned channels are both nil
+// if err == nil, both channels must be fully drained by the caller to avoid leaking goroutines
+func ListAbstractionsStreamed(ctx context.Context, query ListZFSHoldsAndBookmarksQuery) (<-chan Abstraction, <-chan ListAbstractionsError, error) {
+
+	// impl note: structure the query processing in such a way that
+	// a minimum amount of zfs shell-outs needs to be done
+
+	if err := query.Validate(); err != nil {
+		return nil, nil, errors.Wrap(err, "validate query")
+	}
+
+	fss, err := query.FS.Filesystems(ctx)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "list filesystems")
+	}
+
+	outErrs := make(chan ListAbstractionsError)
+	out := make(chan Abstraction)
+
+	errCb := func(err error, fs string, what string) {
+		outErrs <- ListAbstractionsError{Err: err, FS: fs, What: what}
+	}
+	emitAbstraction := func(a Abstraction) {
+		jobIdMatches := query.JobID == nil || a.GetJobID() == nil || *a.GetJobID() == *query.JobID
+
+		createTXGMatches := query.CreateTXG.Contains(a.GetCreateTXG())
+
+		if jobIdMatches && createTXGMatches {
+			out <- a
+		}
+	}
+
+	sem := semaphore.New(int64(query.Concurrency))
+	go func() {
+		defer close(out)
+		defer close(outErrs)
+		var wg sync.WaitGroup
+		defer wg.Wait()
+		for i := range fss {
+			wg.Add(1)
+			go func(i int) {
+				defer wg.Done()
+				g, err := sem.Acquire(ctx)
+				if err != nil {
+					errCb(err, fss[i], err.Error())
+					return
+				}
+				func() {
+					defer g.Release()
+					listAbstractionsImplFS(ctx, fss[i], &query, emitAbstraction, errCb)
+				}()
+			}(i)
+		}
+	}()
+
+	return out, outErrs, nil
+}
+
+func listAbstractionsImplFS(ctx context.Context, fs string, query *ListZFSHoldsAndBookmarksQuery, emitCandidate putListAbstraction, errCb putListAbstractionErr) {
+	fsp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+
+	if len(query.What) == 0 {
+		return
+	}
+
+	whatTypes := zfs.VersionTypeSet{}
+	for what := range query.What {
+		if e := what.BookmarkExtractor(); e != nil {
+			whatTypes[zfs.Bookmark] = true
+		}
+		if e := what.HoldExtractor(); e != nil {
+			whatTypes[zfs.Snapshot] = true
+		}
+	}
+	fsvs, err := zfs.ZFSListFilesystemVersions(ctx, fsp, zfs.ListFilesystemVersionsOptions{
+		Types: whatTypes,
+	})
+	if err != nil {
+		errCb(err, fs, "list filesystem versions")
+		return
+	}
+
+	for at := range query.What {
+		bmE := at.BookmarkExtractor()
+		holdE := at.HoldExtractor()
+		if bmE == nil && holdE == nil || bmE != nil && holdE != nil {
+			panic("implementation error: extractors misconfigured for " + at)
+		}
+		for _, v := range fsvs {
+			var a Abstraction
+			if v.Type == zfs.Bookmark && bmE != nil {
+				a = bmE(fsp, v)
+			}
+			if v.Type == zfs.Snapshot && holdE != nil && query.CreateTXG.Contains(v.GetCreateTXG()) && (!v.UserRefs.Valid || v.UserRefs.Value > 0) {
+				holds, err := zfs.ZFSHolds(ctx, fsp.ToString(), v.Name)
+				if err != nil {
+					errCb(err, v.ToAbsPath(fsp), "get hold on snap")
+					continue
+				}
+				for _, tag := range holds {
+					a = holdE(fsp, v, tag)
+				}
+			}
+			if a != nil {
+				emitCandidate(a)
+			}
+		}
+	}
+}
+
+type BatchDestroyResult struct {
+	Abstraction
+	DestroyErr error
+}
+
+var _ json.Marshaler = (*BatchDestroyResult)(nil)
+
+func (r BatchDestroyResult) MarshalJSON() ([]byte, error) {
+	err := ""
+	if r.DestroyErr != nil {
+		err = r.DestroyErr.Error()
+	}
+	s := struct {
+		Abstraction AbstractionJSON
+		DestroyErr  string
+	}{
+		AbstractionJSON{r.Abstraction},
+		err,
+	}
+	return json.Marshal(s)
+}
+
+func BatchDestroy(ctx context.Context, abs []Abstraction) <-chan BatchDestroyResult {
+	// hold-based batching: per snapshot
+	// bookmark-based batching: none possible via CLI
+	// => not worth the trouble for now, will be worth it once we start using channel programs
+	// => TODO: actual batching using channel programs
+	res := make(chan BatchDestroyResult, len(abs))
+	go func() {
+		for _, a := range abs {
+			res <- BatchDestroyResult{
+				a,
+				a.Destroy(ctx),
+			}
+		}
+		close(res)
+	}()
+	return res
+}
+
+type StalenessInfo struct {
+	ConstructedWithQuery ListZFSHoldsAndBookmarksQuery
+	Live                 []Abstraction
+	Stale                []Abstraction
+}
+
+type fsAndJobId struct {
+	fs    string
+	jobId JobID
+}
+
+type ListStaleQueryError struct {
+	error
+}
+
+// returns *ListStaleQueryError if the given query cannot be used for determining staleness info
+func ListStale(ctx context.Context, q ListZFSHoldsAndBookmarksQuery) (*StalenessInfo, error) {
+	if !q.CreateTXG.IsUnbounded() {
+		// we must determine the most recent step per FS, can't allow that
+		return nil, &ListStaleQueryError{errors.New("ListStale cannot have Until != nil set on query")}
+	}
+
+	// if asking for step holds, must also as for step bookmarks (same kind of abstraction)
+	// as well as replication cursor bookmarks (for firstNotStale)
+	ifAnyThenAll := AbstractionTypeSet{
+		AbstractionStepHold:                    true,
+		AbstractionStepBookmark:                true,
+		AbstractionReplicationCursorBookmarkV2: true,
+	}
+	if q.What.ContainsAnyOf(ifAnyThenAll) && !q.What.ContainsAll(ifAnyThenAll) {
+		return nil, &ListStaleQueryError{errors.Errorf("ListStale requires query to ask for all of %s", ifAnyThenAll.String())}
+	}
+
+	// ----------------- done validating query for listStaleFiltering -----------------------
+
+	qAbs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, err
+	}
+	if len(absErr) > 0 {
+		// can't go on here because we can't determine the most recent step
+		return nil, ListAbstractionsErrors(absErr)
+	}
+
+	si := listStaleFiltering(qAbs, q.CreateTXG.Since)
+	si.ConstructedWithQuery = q
+	return si, nil
+}
+
+type fsAjobAtype struct {
+	fsAndJobId
+	Type AbstractionType
+}
+
+// For step holds and bookmarks, only those older than the most recent replication cursor
+// of their (filesystem,job) is considered because younger ones cannot be stale by definition
+// (if we destroy them, we might actually lose the hold on the `To` for an ongoing incremental replication)
+//
+// For replication cursors and last-received-holds, only the most recent one is kept.
+//
+// the returned StalenessInfo.ConstructedWithQuery is not set
+func listStaleFiltering(abs []Abstraction, sinceBound *CreateTXGRangeBound) *StalenessInfo {
+
+	var noJobId []Abstraction
+	by := make(map[fsAjobAtype][]Abstraction)
+	for _, a := range abs {
+		if a.GetJobID() == nil {
+			noJobId = append(noJobId, a)
+			continue
+		}
+		faj := fsAjobAtype{fsAndJobId{a.GetFS(), *a.GetJobID()}, a.GetType()}
+		l := by[faj]
+		l = append(l, a)
+		by[faj] = l
+	}
+
+	type stepFirstNotStaleCandidate struct {
+		cursor *Abstraction
+		step   *Abstraction
+	}
+	stepFirstNotStaleCandidates := make(map[fsAndJobId]stepFirstNotStaleCandidate) // empty map => will always return nil
+	for _, a := range abs {
+		if a.GetJobID() == nil {
+			continue // already put those into always-live list noJobId in above loop
+		}
+		key := fsAndJobId{a.GetFS(), *a.GetJobID()}
+		c := stepFirstNotStaleCandidates[key]
+
+		switch a.GetType() {
+		// stepFirstNotStaleCandidate.cursor
+		case AbstractionReplicationCursorBookmarkV2:
+			if c.cursor == nil || (*c.cursor).GetCreateTXG() < a.GetCreateTXG() {
+				a := a
+				c.cursor = &a
+			}
+
+		// stepFirstNotStaleCandidate.step
+		case AbstractionStepBookmark:
+			fallthrough
+		case AbstractionStepHold:
+			if c.step == nil || (*c.step).GetCreateTXG() < a.GetCreateTXG() {
+				a := a
+				c.step = &a
+			}
+
+		// not interested in the others
+		default:
+			continue // not relevant
+
+		}
+
+		stepFirstNotStaleCandidates[key] = c
+	}
+
+	ret := &StalenessInfo{
+		Live:  noJobId,
+		Stale: []Abstraction{},
+	}
+
+	for k := range by {
+		l := by[k]
+
+		if k.Type == AbstractionStepHold || k.Type == AbstractionStepBookmark {
+			// all older than the most recent cursor are stale, others are always live
+
+			// if we don't have a replication cursor yet, use untilBound = nil
+			// to consider all steps stale (...at first)
+			var untilBound *CreateTXGRangeBound
+			{
+				sfnsc := stepFirstNotStaleCandidates[k.fsAndJobId]
+
+				// if there's a replication cursor, use it as a cutoff between live and stale
+				// if there's none, we are in initial replication and only need to keep
+				// the most recent step hold live, since that's what our initial replication strategy
+				// uses (both initially and on resume)
+				// (FIXME hardcoded replication strategy)
+				if sfnsc.cursor != nil {
+					untilBound = &CreateTXGRangeBound{
+						CreateTXG: (*sfnsc.cursor).GetCreateTXG(),
+						// if we have a cursor, can throw away step hold on both From and To
+						Inclusive: &zfs.NilBool{B: true},
+					}
+				} else if sfnsc.step != nil {
+					untilBound = &CreateTXGRangeBound{
+						CreateTXG: (*sfnsc.step).GetCreateTXG(),
+						// if we don't have a cursor, the step most recent step hold is our
+						// initial replication cursor and it's possibly still live (interrupted initial replication)
+						Inclusive: &zfs.NilBool{B: false},
+					}
+				} else {
+					untilBound = nil // consider everything stale
+				}
+			}
+			staleRange := CreateTXGRange{
+				Since: sinceBound,
+				Until: untilBound,
+			}
+
+			// partition by staleRange
+			for _, a := range l {
+				if staleRange.Contains(a.GetCreateTXG()) {
+					ret.Stale = append(ret.Stale, a)
+				} else {
+					ret.Live = append(ret.Live, a)
+				}
+			}
+
+		} else if k.Type == AbstractionReplicationCursorBookmarkV2 || k.Type == AbstractionLastReceivedHold {
+			// all but the most recent are stale by definition (we always _move_ them)
+			// NOTE: must not use firstNotStale in this branch, not computed for these types
+
+			// sort descending (highest createtxg first), then cut off
+			sort.Slice(l, func(i, j int) bool {
+				return l[i].GetCreateTXG() > l[j].GetCreateTXG()
+			})
+			if len(l) > 0 {
+				ret.Live = append(ret.Live, l[0])
+				ret.Stale = append(ret.Stale, l[1:]...)
+			}
+		} else {
+			ret.Live = append(ret.Live, l...)
+		}
+	}
+
+	return ret
+
+}

endpoint/endpoint_zfs_abstraction_cursor_and_last_received_hold.go

@@ -0,0 +1,386 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"sort"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/util/errorarray"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+const replicationCursorBookmarkNamePrefix = "zrepl_CURSOR"
+
+func ReplicationCursorBookmarkName(fs string, guid uint64, id JobID) (string, error) {
+	return replicationCursorBookmarkNameImpl(fs, guid, id.String())
+}
+
+func replicationCursorBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
+	return makeJobAndGuidBookmarkName(replicationCursorBookmarkNamePrefix, fs, guid, jobid)
+}
+
+var ErrV1ReplicationCursor = fmt.Errorf("bookmark name is a v1-replication cursor")
+
+//err != nil always means that the bookmark is not a valid replication bookmark
+//
+// Returns ErrV1ReplicationCursor as error if the bookmark is a v1 replication cursor
+func ParseReplicationCursorBookmarkName(fullname string) (uint64, JobID, error) {
+
+	// check for legacy cursors
+	{
+		if err := zfs.EntityNamecheck(fullname, zfs.EntityTypeBookmark); err != nil {
+			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name")
+		}
+		_, _, name, err := zfs.DecomposeVersionString(fullname)
+		if err != nil {
+			return 0, JobID{}, errors.Wrap(err, "parse replication cursor bookmark name: decompose version string")
+		}
+		const V1ReplicationCursorBookmarkName = "zrepl_replication_cursor"
+		if name == V1ReplicationCursorBookmarkName {
+			return 0, JobID{}, ErrV1ReplicationCursor
+		}
+		// fallthrough to main parser
+	}
+
+	guid, jobID, err := parseJobAndGuidBookmarkName(fullname, replicationCursorBookmarkNamePrefix)
+	if err != nil {
+		err = errors.Wrap(err, "parse replication cursor bookmark name") // no shadow
+	}
+	return guid, jobID, err
+}
+
+// may return nil for both values, indicating there is no cursor
+func GetMostRecentReplicationCursorOfJob(ctx context.Context, fs string, jobID JobID) (*zfs.FilesystemVersion, error) {
+	fsp, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		return nil, err
+	}
+	candidates, err := GetReplicationCursors(ctx, fsp, jobID)
+	if err != nil || len(candidates) == 0 {
+		return nil, err
+	}
+
+	sort.Slice(candidates, func(i, j int) bool {
+		return candidates[i].CreateTXG < candidates[j].CreateTXG
+	})
+
+	mostRecent := candidates[len(candidates)-1]
+	return &mostRecent, nil
+}
+
+func GetReplicationCursors(ctx context.Context, dp *zfs.DatasetPath, jobID JobID) ([]zfs.FilesystemVersion, error) {
+
+	fs := dp.ToString()
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{FS: &fs},
+		What: map[AbstractionType]bool{
+			AbstractionReplicationCursorBookmarkV1: true,
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		JobID:       &jobID,
+		CreateTXG:   CreateTXGRange{},
+		Concurrency: 1,
+	}
+	abs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, errors.Wrap(err, "get replication cursor: list bookmarks and holds")
+	}
+	if len(absErr) > 0 {
+		return nil, ListAbstractionsErrors(absErr)
+	}
+
+	var v1, v2 []Abstraction
+	for _, a := range abs {
+		switch a.GetType() {
+		case AbstractionReplicationCursorBookmarkV1:
+			v1 = append(v1, a)
+		case AbstractionReplicationCursorBookmarkV2:
+			v2 = append(v2, a)
+		default:
+			panic("unexpected abstraction: " + a.GetType())
+		}
+	}
+
+	if len(v1) > 0 {
+		getLogger(ctx).WithField("bookmark", v1).
+			Warn("found v1-replication cursor bookmarks, consider running migration 'replication-cursor:v1-v2' after successful replication with this zrepl version")
+	}
+
+	candidates := make([]zfs.FilesystemVersion, 0)
+	for _, v := range v2 {
+		candidates = append(candidates, v.GetFilesystemVersion())
+	}
+
+	return candidates, nil
+}
+
+type ReplicationCursorTarget interface {
+	IsSnapshot() bool
+	GetGuid() uint64
+	GetCreateTXG() uint64
+	ToSendArgVersion() zfs.ZFSSendArgVersion
+}
+
+// `target` is validated before replication cursor is set. if validation fails, the cursor is not moved.
+//
+// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
+func MoveReplicationCursor(ctx context.Context, fs string, target ReplicationCursorTarget, jobID JobID) (destroyedCursors []Abstraction, err error) {
+
+	if !target.IsSnapshot() {
+		return nil, zfs.ErrBookmarkCloningNotSupported
+	}
+
+	bookmarkname, err := ReplicationCursorBookmarkName(fs, target.GetGuid(), jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "determine replication cursor name")
+	}
+
+	// idempotently create bookmark (guid is encoded in it, hence we'll most likely add a new one
+	// cleanup the old one afterwards
+
+	err = zfs.ZFSBookmark(ctx, fs, target.ToSendArgVersion(), bookmarkname)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			return nil, err // TODO go1.13 use wrapping
+		}
+		return nil, errors.Wrapf(err, "cannot create bookmark")
+	}
+
+	destroyedCursors, err = DestroyObsoleteReplicationCursors(ctx, fs, target, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "destroy obsolete replication cursors")
+	}
+
+	return destroyedCursors, nil
+}
+
+type ReplicationCursor interface {
+	GetCreateTXG() uint64
+}
+
+func DestroyObsoleteReplicationCursors(ctx context.Context, fs string, current ReplicationCursor, jobID JobID) (_ []Abstraction, err error) {
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		What: AbstractionTypeSet{
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: nil,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: current.GetCreateTXG(),
+				Inclusive: &zfs.NilBool{B: false},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErr, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return nil, errors.Wrap(err, "list abstractions")
+	}
+	if len(absErr) > 0 {
+		return nil, errors.Wrap(ListAbstractionsErrors(absErr), "list abstractions")
+	}
+
+	var destroyed []Abstraction
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("replication_cursor_bookmark", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot destroy obsolete replication cursor bookmark")
+		} else {
+			destroyed = append(destroyed, res.Abstraction)
+			log.Info("destroyed obsolete replication cursor bookmark")
+		}
+	}
+	if len(errs) == 0 {
+		return destroyed, nil
+	} else {
+		return destroyed, errorarray.Wrap(errs, "destroy obsolete replication cursor")
+	}
+}
+
+var lastReceivedHoldTagRE = regexp.MustCompile("^zrepl_last_received_J_(.+)$")
+
+// err != nil always means that the bookmark is not a step bookmark
+func ParseLastReceivedHoldTag(tag string) (JobID, error) {
+	match := lastReceivedHoldTagRE.FindStringSubmatch(tag)
+	if match == nil {
+		return JobID{}, errors.Errorf("parse last-received-hold tag: does not match regex %s", lastReceivedHoldTagRE.String())
+	}
+	jobId, err := MakeJobID(match[1])
+	if err != nil {
+		return JobID{}, errors.Wrap(err, "parse last-received-hold tag: invalid job id field")
+	}
+	return jobId, nil
+}
+
+func LastReceivedHoldTag(jobID JobID) (string, error) {
+	return lastReceivedHoldImpl(jobID.String())
+}
+
+func lastReceivedHoldImpl(jobid string) (string, error) {
+	tag := fmt.Sprintf("zrepl_last_received_J_%s", jobid)
+	if err := zfs.ValidHoldTag(tag); err != nil {
+		return "", err
+	}
+	return tag, nil
+}
+
+func MoveLastReceivedHold(ctx context.Context, fs string, to zfs.FilesystemVersion, jobID JobID) error {
+
+	if !to.IsSnapshot() {
+		return errors.Errorf("last-received-hold: target must be a snapshot: %s", to.FullPath(fs))
+	}
+
+	tag, err := LastReceivedHoldTag(jobID)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: hold tag")
+	}
+
+	// we never want to be without a hold
+	// => hold new one before releasing old hold
+
+	err = zfs.ZFSHold(ctx, fs, to, tag)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: hold newly received")
+	}
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		What: AbstractionTypeSet{
+			AbstractionLastReceivedHold: true,
+		},
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: nil,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: to.GetCreateTXG(),
+				Inclusive: &zfs.NilBool{B: false},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErrs, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return errors.Wrap(err, "last-received-hold: list")
+	}
+	if len(absErrs) > 0 {
+		return errors.Wrap(ListAbstractionsErrors(absErrs), "last-received-hold: list")
+	}
+
+	getLogger(ctx).WithField("last-received-holds", fmt.Sprintf("%s", abs)).Debug("releasing last-received-holds")
+
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("last-received-hold", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot release last-received-hold")
+		} else {
+			log.Info("released last-received-hold")
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	} else {
+		return errorarray.Wrap(errs, "last-received-hold: release")
+	}
+}
+
+func ReplicationCursorV2Extractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+	fullname := v.ToAbsPath(fs)
+	guid, jobid, err := ParseReplicationCursorBookmarkName(fullname)
+	if err == nil {
+		if guid != v.Guid {
+			// TODO log this possibly tinkered-with bookmark
+			return nil
+		}
+		return &bookmarkBasedAbstraction{
+			Type:              AbstractionReplicationCursorBookmarkV2,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			JobID:             jobid,
+		}
+	}
+	return nil
+}
+
+func ReplicationCursorV1Extractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+	fullname := v.ToAbsPath(fs)
+	_, _, err := ParseReplicationCursorBookmarkName(fullname)
+	if err == ErrV1ReplicationCursor {
+		return &ReplicationCursorV1{
+			Type:              AbstractionReplicationCursorBookmarkV1,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+		}
+	}
+	return nil
+}
+
+var _ HoldExtractor = LastReceivedHoldExtractor
+
+func LastReceivedHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
+	var err error
+
+	if v.Type != zfs.Snapshot {
+		panic("impl error")
+	}
+
+	jobID, err := ParseLastReceivedHoldTag(holdTag)
+	if err == nil {
+		return &holdBasedAbstraction{
+			Type:              AbstractionLastReceivedHold,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			Tag:               holdTag,
+			JobID:             jobID,
+		}
+	}
+	return nil
+}
+
+type ReplicationCursorV1 struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+}
+
+func (c ReplicationCursorV1) GetType() AbstractionType                    { return c.Type }
+func (c ReplicationCursorV1) GetFS() string                               { return c.FS }
+func (c ReplicationCursorV1) GetFullPath() string                         { return fmt.Sprintf("%s#%s", c.FS, c.GetName()) }
+func (c ReplicationCursorV1) GetJobID() *JobID                            { return nil }
+func (c ReplicationCursorV1) GetFilesystemVersion() zfs.FilesystemVersion { return c.FilesystemVersion }
+func (c ReplicationCursorV1) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{c})
+}
+func (c ReplicationCursorV1) String() string {
+	return fmt.Sprintf("%s %s", c.Type, c.GetFullPath())
+}
+func (c ReplicationCursorV1) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSDestroyIdempotent(ctx, c.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "destroy %s %s: zfs", c.Type, c.GetFullPath())
+	}
+	return nil
+}

endpoint/endpoint_zfs_abstraction_step.go

@@ -0,0 +1,286 @@
+package endpoint
+
+import (
+	"context"
+	"fmt"
+	"regexp"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/util/errorarray"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+var stepHoldTagRE = regexp.MustCompile("^zrepl_STEP_J_(.+)")
+
+func StepHoldTag(jobid JobID) (string, error) {
+	return stepHoldTagImpl(jobid.String())
+}
+
+func stepHoldTagImpl(jobid string) (string, error) {
+	t := fmt.Sprintf("zrepl_STEP_J_%s", jobid)
+	if err := zfs.ValidHoldTag(t); err != nil {
+		return "", err
+	}
+	return t, nil
+}
+
+// err != nil always means that the bookmark is not a step bookmark
+func ParseStepHoldTag(tag string) (JobID, error) {
+	match := stepHoldTagRE.FindStringSubmatch(tag)
+	if match == nil {
+		return JobID{}, fmt.Errorf("parse hold tag: match regex %q", stepHoldTagRE)
+	}
+	jobID, err := MakeJobID(match[1])
+	if err != nil {
+		return JobID{}, errors.Wrap(err, "parse hold tag: invalid job id field")
+	}
+	return jobID, nil
+}
+
+const stepBookmarkNamePrefix = "zrepl_STEP"
+
+// v must be validated by caller
+func StepBookmarkName(fs string, guid uint64, id JobID) (string, error) {
+	return stepBookmarkNameImpl(fs, guid, id.String())
+}
+
+func stepBookmarkNameImpl(fs string, guid uint64, jobid string) (string, error) {
+	return makeJobAndGuidBookmarkName(stepBookmarkNamePrefix, fs, guid, jobid)
+}
+
+// name is the full bookmark name, including dataset path
+//
+// err != nil always means that the bookmark is not a step bookmark
+func ParseStepBookmarkName(fullname string) (guid uint64, jobID JobID, err error) {
+	guid, jobID, err = parseJobAndGuidBookmarkName(fullname, stepBookmarkNamePrefix)
+	if err != nil {
+		err = errors.Wrap(err, "parse step bookmark name") // no shadow!
+	}
+	return guid, jobID, err
+}
+
+// idempotently hold / step-bookmark `version`
+//
+// returns ErrBookmarkCloningNotSupported if version is a bookmark and bookmarking bookmarks is not supported by ZFS
+func HoldStep(ctx context.Context, fs string, v zfs.FilesystemVersion, jobID JobID) (Abstraction, error) {
+	if v.IsSnapshot() {
+
+		tag, err := StepHoldTag(jobID)
+		if err != nil {
+			return nil, errors.Wrap(err, "step hold tag")
+		}
+
+		if err := zfs.ZFSHold(ctx, fs, v, tag); err != nil {
+			return nil, errors.Wrap(err, "step hold: zfs")
+		}
+
+		return &holdBasedAbstraction{
+			Type:              AbstractionStepHold,
+			FS:                fs,
+			Tag:               tag,
+			JobID:             jobID,
+			FilesystemVersion: v,
+		}, nil
+	}
+
+	if !v.IsBookmark() {
+		panic(fmt.Sprintf("version must bei either snapshot or bookmark, got %#v", v))
+	}
+
+	bmname, err := StepBookmarkName(fs, v.Guid, jobID)
+	if err != nil {
+		return nil, errors.Wrap(err, "create step bookmark: determine bookmark name")
+	}
+	// idempotently create bookmark
+	err = zfs.ZFSBookmark(ctx, fs, v.ToSendArgVersion(), bmname)
+	if err != nil {
+		if err == zfs.ErrBookmarkCloningNotSupported {
+			// TODO we could actually try to find a local snapshot that has the requested GUID
+			// 		however, the replication algorithm prefers snapshots anyways, so this quest
+			// 		is most likely not going to be successful. Also, there's the possibility that
+			//      the caller might want to filter what snapshots are eligibile, and this would
+			//      complicate things even further.
+			return nil, err // TODO go1.13 use wrapping
+		}
+		return nil, errors.Wrap(err, "create step bookmark: zfs")
+	}
+	return &bookmarkBasedAbstraction{
+		Type:              AbstractionStepBookmark,
+		FS:                fs,
+		FilesystemVersion: v,
+		JobID:             jobID,
+	}, nil
+}
+
+// idempotently release the step-hold on v if v is a snapshot
+// or idempotently destroy the step-bookmark  of v if v is a bookmark
+//
+// note that this operation leaves v itself untouched, unless v is the step-bookmark itself, in which case v is destroyed
+//
+// returns an instance of *zfs.DatasetDoesNotExist if `v` does not exist
+func ReleaseStep(ctx context.Context, fs string, v zfs.FilesystemVersion, jobID JobID) error {
+
+	if v.IsSnapshot() {
+		tag, err := StepHoldTag(jobID)
+		if err != nil {
+			return errors.Wrap(err, "step release tag")
+		}
+
+		if err := zfs.ZFSRelease(ctx, tag, v.FullPath(fs)); err != nil {
+			return errors.Wrap(err, "step release: zfs")
+		}
+
+		return nil
+	}
+	if !v.IsBookmark() {
+		panic(fmt.Sprintf("impl error: expecting version to be a bookmark, got %#v", v))
+	}
+
+	bmname, err := StepBookmarkName(fs, v.Guid, jobID)
+	if err != nil {
+		return errors.Wrap(err, "step release: determine bookmark name")
+	}
+	// idempotently destroy bookmark
+
+	if err := zfs.ZFSDestroyIdempotent(ctx, bmname); err != nil {
+		return errors.Wrap(err, "step release: bookmark destroy: zfs")
+	}
+
+	return nil
+}
+
+// release {step holds, step bookmarks} earlier and including `mostRecent`
+func ReleaseStepCummulativeInclusive(ctx context.Context, fs string, since *CreateTXGRangeBound, mostRecent zfs.FilesystemVersion, jobID JobID) error {
+	q := ListZFSHoldsAndBookmarksQuery{
+		What: AbstractionTypeSet{
+			AbstractionStepHold:     true,
+			AbstractionStepBookmark: true,
+		},
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		CreateTXG: CreateTXGRange{
+			Since: since,
+			Until: &CreateTXGRangeBound{
+				CreateTXG: mostRecent.CreateTXG,
+				Inclusive: &zfs.NilBool{B: true},
+			},
+		},
+		Concurrency: 1,
+	}
+	abs, absErrs, err := ListAbstractions(ctx, q)
+	if err != nil {
+		return errors.Wrap(err, "step release cummulative: list")
+	}
+	if len(absErrs) > 0 {
+		return errors.Wrap(ListAbstractionsErrors(absErrs), "step release cummulative: list")
+	}
+
+	getLogger(ctx).WithField("step_holds_and_bookmarks", fmt.Sprintf("%s", abs)).Debug("releasing step holds and bookmarks")
+
+	var errs []error
+	for res := range BatchDestroy(ctx, abs) {
+		log := getLogger(ctx).
+			WithField("step_hold_or_bookmark", res.Abstraction)
+		if res.DestroyErr != nil {
+			errs = append(errs, res.DestroyErr)
+			log.WithError(err).
+				Error("cannot release step hold or bookmark")
+		} else {
+			log.Info("released step hold or bookmark")
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	} else {
+		return errorarray.Wrap(errs, "step release cummulative: release")
+	}
+}
+
+func TryReleaseStepStaleFS(ctx context.Context, fs string, jobID JobID) {
+
+	q := ListZFSHoldsAndBookmarksQuery{
+		FS: ListZFSHoldsAndBookmarksQueryFilesystemFilter{
+			FS: &fs,
+		},
+		JobID: &jobID,
+		What: AbstractionTypeSet{
+			AbstractionStepHold:                    true,
+			AbstractionStepBookmark:                true,
+			AbstractionReplicationCursorBookmarkV2: true,
+		},
+		Concurrency: 1,
+	}
+	staleness, err := ListStale(ctx, q)
+	if _, ok := err.(*ListStaleQueryError); ok {
+		panic(err)
+	} else if err != nil {
+		getLogger(ctx).WithError(err).Error("cannot list stale step holds and bookmarks")
+		return
+	}
+	for _, s := range staleness.Stale {
+		getLogger(ctx).WithField("stale_step_hold_or_bookmark", s).Info("batch-destroying stale step hold or bookmark")
+	}
+	for res := range BatchDestroy(ctx, staleness.Stale) {
+		if res.DestroyErr != nil {
+			getLogger(ctx).
+				WithField("stale_step_hold_or_bookmark", res.Abstraction).
+				WithError(res.DestroyErr).
+				Error("cannot destroy stale step-hold or bookmark")
+		} else {
+			getLogger(ctx).
+				WithField("stale_step_hold_or_bookmark", res.Abstraction).
+				WithError(res.DestroyErr).
+				Info("destroyed stale step-hold or bookmark")
+		}
+	}
+
+}
+
+var _ BookmarkExtractor = StepBookmarkExtractor
+
+func StepBookmarkExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion) (_ Abstraction) {
+	if v.Type != zfs.Bookmark {
+		panic("impl error")
+	}
+
+	fullname := v.ToAbsPath(fs)
+
+	guid, jobid, err := ParseStepBookmarkName(fullname)
+	if guid != v.Guid {
+		// TODO log this possibly tinkered-with bookmark
+		return nil
+	}
+	if err == nil {
+		bm := &bookmarkBasedAbstraction{
+			Type:              AbstractionStepBookmark,
+			FS:                fs.ToString(),
+			FilesystemVersion: v,
+			JobID:             jobid,
+		}
+		return bm
+	}
+	return nil
+}
+
+var _ HoldExtractor = StepHoldExtractor
+
+func StepHoldExtractor(fs *zfs.DatasetPath, v zfs.FilesystemVersion, holdTag string) Abstraction {
+	if v.Type != zfs.Snapshot {
+		panic("impl error")
+	}
+
+	jobID, err := ParseStepHoldTag(holdTag)
+	if err == nil {
+		return &holdBasedAbstraction{
+			Type:              AbstractionStepHold,
+			FS:                fs.ToString(),
+			Tag:               holdTag,
+			FilesystemVersion: v,
+			JobID:             jobID,
+		}
+	}
+	return nil
+}

endpoint/endpoint_zfs_abstraction_test.go

@@ -0,0 +1,234 @@
+package endpoint
+
+import (
+	"fmt"
+	"math"
+	"runtime/debug"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func TestCreateTXGRange(t *testing.T) {
+
+	type testCaseExpectation struct {
+		input  uint64
+		expect bool
+	}
+	type testCase struct {
+		name                     string
+		config                   *CreateTXGRange
+		configAllowZeroCreateTXG bool
+		expectInvalid            bool
+		expectString             string
+		expect                   []testCaseExpectation
+	}
+
+	tcs := []testCase{
+		{
+			name:          "unbounded",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: nil,
+			},
+			expectString: "~,~",
+			expect: []testCaseExpectation{
+				{0, true},
+				{math.MaxUint64, true},
+				{1, true},
+				{math.MaxUint64 - 1, true},
+			},
+		},
+		{
+			name:          "wrong order obvious",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: true}},
+				Until: &CreateTXGRangeBound{20, &zfs.NilBool{B: true}},
+			},
+			expectString: "[23,20]",
+		},
+		{
+			name:          "wrong order edge-case could also be empty",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{23, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{22, &zfs.NilBool{B: true}},
+			},
+			expectString: "(23,22]",
+		},
+		{
+			name:          "empty",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+			},
+			expectString: "(2,2)",
+		},
+		{
+			name:          "inclusive-since-exclusive-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: true}},
+				Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: false}},
+			},
+			expectString: "[2,5)",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, true},
+				{3, true},
+				{4, true},
+				{5, false},
+				{6, false},
+			},
+		},
+		{
+			name:          "exclusive-since-inclusive-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: &CreateTXGRangeBound{5, &zfs.NilBool{B: true}},
+			},
+			expectString: "(2,5]",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, false},
+				{3, true},
+				{4, true},
+				{5, true},
+				{6, false},
+			},
+		},
+		{
+			name:          "zero-createtxg-not-allowed-because-likely-programmer-error",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
+			},
+			expectString: "~,0]",
+		},
+		{
+			name:          "half-open-no-until",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{2, &zfs.NilBool{B: false}},
+				Until: nil,
+			},
+			expectString: "(2,~",
+			expect: []testCaseExpectation{
+				{0, false},
+				{1, false},
+				{2, false},
+				{3, true},
+				{4, true},
+				{5, true},
+				{6, true},
+			},
+		},
+		{
+			name:          "half-open-no-since",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: nil,
+				Until: &CreateTXGRangeBound{4, &zfs.NilBool{B: true}},
+			},
+			expectString: "~,4]",
+			expect: []testCaseExpectation{
+				{0, true},
+				{1, true},
+				{2, true},
+				{3, true},
+				{4, true},
+				{5, false},
+			},
+		},
+		{
+			name:          "edgeSince",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: true}},
+				Until: nil,
+			},
+			expectString: "[18446744073709551615,~",
+			expect: []testCaseExpectation{
+				{math.MaxUint64, true},
+				{math.MaxUint64 - 1, false},
+				{0, false},
+				{1, false},
+			},
+		},
+		{
+			name:          "edgeSinceNegative",
+			expectInvalid: true,
+			config: &CreateTXGRange{
+				Since: &CreateTXGRangeBound{math.MaxUint64, &zfs.NilBool{B: false}},
+				Until: nil,
+			},
+			expectString: "(18446744073709551615,~",
+		},
+		{
+			name:          "edgeUntil",
+			expectInvalid: false,
+			config: &CreateTXGRange{
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: true}},
+			},
+			configAllowZeroCreateTXG: true,
+			expectString:             "~,0]",
+			expect: []testCaseExpectation{
+				{0, true},
+				{math.MaxUint64, false},
+				{1, false},
+			},
+		},
+		{
+			name:                     "edgeUntilNegative",
+			expectInvalid:            true,
+			configAllowZeroCreateTXG: true,
+			config: &CreateTXGRange{
+				Until: &CreateTXGRangeBound{0, &zfs.NilBool{B: false}},
+			},
+			expectString: "~,0)",
+		},
+	}
+
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			require.True(t, tc.expectInvalid != (len(tc.expect) > 0), "invalid test config: must either expect invalid or have expectations: %s", tc.name)
+			require.NotEmpty(t, tc.expectString)
+			assert.Equal(t, tc.expectString, tc.config.String())
+
+			save := createTXGRangeBoundAllowCreateTXG0
+			createTXGRangeBoundAllowCreateTXG0 = tc.configAllowZeroCreateTXG
+			defer func() {
+				createTXGRangeBoundAllowCreateTXG0 = save
+			}()
+
+			if tc.expectInvalid {
+				t.Run(tc.name, func(t *testing.T) {
+					assert.Error(t, tc.config.Validate())
+				})
+			} else {
+				for i, e := range tc.expect {
+					t.Run(fmt.Sprint(i), func(t *testing.T) {
+						defer func() {
+							v := recover()
+							if v != nil {
+								t.Fatalf("should not panic: %T %v\n%s", v, v, debug.Stack())
+							}
+						}()
+						assert.Equal(t, e.expect, tc.config.Contains(e.input))
+					})
+				}
+			}
+		})
+	}
+
+}

endpoint/endpoint_zfs_helpers_repr.go

@@ -0,0 +1,58 @@
+package endpoint
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// returns the short name (no fs# prefix)
+func makeJobAndGuidBookmarkName(prefix string, fs string, guid uint64, jobid string) (string, error) {
+	bmname := fmt.Sprintf(prefix+"_G_%016x_J_%s", guid, jobid)
+	if err := zfs.EntityNamecheck(fmt.Sprintf("%s#%s", fs, bmname), zfs.EntityTypeBookmark); err != nil {
+		return "", err
+	}
+	return bmname, nil
+}
+
+var jobAndGuidBookmarkRE = regexp.MustCompile(`(.+)_G_([0-9a-f]{16})_J_(.+)$`)
+
+func parseJobAndGuidBookmarkName(fullname string, prefix string) (guid uint64, jobID JobID, _ error) {
+
+	if len(prefix) == 0 {
+		panic("prefix must not be empty")
+	}
+
+	if err := zfs.EntityNamecheck(fullname, zfs.EntityTypeBookmark); err != nil {
+		return 0, JobID{}, err
+	}
+
+	_, _, name, err := zfs.DecomposeVersionString(fullname)
+	if err != nil {
+		return 0, JobID{}, errors.Wrap(err, "decompose bookmark name")
+	}
+
+	match := jobAndGuidBookmarkRE.FindStringSubmatch(name)
+	if match == nil {
+		return 0, JobID{}, errors.Errorf("bookmark name does not match regex %q", jobAndGuidBookmarkRE.String())
+	}
+	if match[1] != prefix {
+		return 0, JobID{}, errors.Errorf("prefix component does not match: expected %q, got %q", prefix, match[1])
+	}
+
+	guid, err = strconv.ParseUint(match[2], 16, 64)
+	if err != nil {
+		return 0, JobID{}, errors.Wrapf(err, "parse guid component: %q", match[2])
+	}
+
+	jobID, err = MakeJobID(match[3])
+	if err != nil {
+		return 0, JobID{}, errors.Wrapf(err, "parse jobid component: %q", match[3])
+	}
+
+	return guid, jobID, nil
+}

endpoint/endpoint_zfs_helpers_repr_test.go

@@ -0,0 +1,51 @@
+package endpoint
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestParseJobAndGuidBookmarkName(t *testing.T) {
+
+	type Case struct {
+		input     string
+		expectErr bool
+
+		guid  uint64
+		jobid string
+	}
+
+	cases := []Case{
+		{
+			`p1/sync#zrepl_CURSOR_G_932f3a7089080ce2_J_push with legitimate name`,
+			false, 0x932f3a7089080ce2, "push with legitimate name",
+		},
+		{
+			input:     `p1/sync#zrepl_CURSOR_G_932f3a7089_J_push with legitimate name`,
+			expectErr: true,
+		},
+		{
+			input:     `p1/sync#zrepl_CURSOR_G_932f3a7089080ce2_J_push with il\tlegitimate name`,
+			expectErr: true,
+		},
+		{
+			input:     `p1/sync#otherprefix_G_932f3a7089080ce2_J_push with legitimate name`,
+			expectErr: true,
+		},
+	}
+
+	for i := range cases {
+		t.Run(cases[i].input, func(t *testing.T) {
+			guid, jobid, err := parseJobAndGuidBookmarkName(cases[i].input, replicationCursorBookmarkNamePrefix)
+			if cases[i].expectErr {
+				assert.Error(t, err)
+			} else {
+				assert.NoError(t, err)
+				assert.Equal(t, cases[i].guid, guid)
+				assert.Equal(t, MustMakeJobID(cases[i].jobid), jobid)
+			}
+		})
+	}
+
+}

endpoint/endpoint_zfs_helpers_types.go

@@ -0,0 +1,74 @@
+package endpoint
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type bookmarkBasedAbstraction struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+	JobID JobID
+}
+
+func (b bookmarkBasedAbstraction) GetType() AbstractionType { return b.Type }
+func (b bookmarkBasedAbstraction) GetFS() string            { return b.FS }
+func (b bookmarkBasedAbstraction) GetJobID() *JobID         { return &b.JobID }
+func (b bookmarkBasedAbstraction) GetFullPath() string {
+	return fmt.Sprintf("%s#%s", b.FS, b.Name) // TODO use zfs.FilesystemVersion.ToAbsPath
+}
+func (b bookmarkBasedAbstraction) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{b})
+}
+func (b bookmarkBasedAbstraction) String() string {
+	return fmt.Sprintf("%s %s", b.Type, b.GetFullPath())
+}
+
+func (b bookmarkBasedAbstraction) GetFilesystemVersion() zfs.FilesystemVersion {
+	return b.FilesystemVersion
+}
+
+func (b bookmarkBasedAbstraction) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSDestroyIdempotent(ctx, b.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "destroy %s: zfs", b)
+	}
+	return nil
+}
+
+type holdBasedAbstraction struct {
+	Type AbstractionType
+	FS   string
+	zfs.FilesystemVersion
+	Tag   string
+	JobID JobID
+}
+
+func (h holdBasedAbstraction) GetType() AbstractionType { return h.Type }
+func (h holdBasedAbstraction) GetFS() string            { return h.FS }
+func (h holdBasedAbstraction) GetJobID() *JobID         { return &h.JobID }
+func (h holdBasedAbstraction) GetFullPath() string {
+	return fmt.Sprintf("%s@%s", h.FS, h.GetName()) // TODO use zfs.FilesystemVersion.ToAbsPath
+}
+func (h holdBasedAbstraction) MarshalJSON() ([]byte, error) {
+	return json.Marshal(AbstractionJSON{h})
+}
+func (h holdBasedAbstraction) String() string {
+	return fmt.Sprintf("%s %q on %s", h.Type, h.Tag, h.GetFullPath())
+}
+
+func (h holdBasedAbstraction) GetFilesystemVersion() zfs.FilesystemVersion {
+	return h.FilesystemVersion
+}
+
+func (h holdBasedAbstraction) Destroy(ctx context.Context) error {
+	if err := zfs.ZFSRelease(ctx, h.Tag, h.GetFullPath()); err != nil {
+		return errors.Wrapf(err, "release %s: zfs", h)
+	}
+	return nil
+}

endpoint/jobid.go

@@ -0,0 +1,78 @@
+package endpoint
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/pkg/errors"
+
+	"github.com/zrepl/zrepl/zfs"
+)
+
+// JobID instances returned by MakeJobID() guarantee their JobID.String()
+// can be used in ZFS dataset names and hold tags.
+type JobID struct {
+	jid string
+}
+
+func MakeJobID(s string) (JobID, error) {
+	if len(s) == 0 {
+		return JobID{}, fmt.Errorf("must not be empty string")
+	}
+
+	if err := zfs.ComponentNamecheck(s); err != nil {
+		return JobID{}, errors.Wrap(err, "must be usable as a dataset path component")
+	}
+
+	if _, err := stepBookmarkNameImpl("pool/ds", 0xface601d, s); err != nil {
+		// note that this might still fail due to total maximum name length, but we can't enforce that
+		return JobID{}, errors.Wrap(err, "must be usable for a step bookmark")
+	}
+
+	if _, err := stepHoldTagImpl(s); err != nil {
+		return JobID{}, errors.Wrap(err, "must be usable for a step hold tag")
+	}
+
+	if _, err := lastReceivedHoldImpl(s); err != nil {
+		return JobID{}, errors.Wrap(err, "must be usable as a last-received-hold tag")
+	}
+
+	// FIXME replication cursor bookmark name
+
+	_, err := zfs.NewDatasetPath(s)
+	if err != nil {
+		return JobID{}, fmt.Errorf("must be usable in a ZFS dataset path: %s", err)
+	}
+
+	return JobID{s}, nil
+}
+
+func MustMakeJobID(s string) JobID {
+	jid, err := MakeJobID(s)
+	if err != nil {
+		panic(err)
+	}
+	return jid
+}
+
+func (j JobID) expectInitialized() {
+	if j.jid == "" {
+		panic("use of uninitialized JobID")
+	}
+}
+
+func (j JobID) String() string {
+	j.expectInitialized()
+	return j.jid
+}
+
+var _ json.Marshaler = JobID{}
+var _ json.Unmarshaler = (*JobID)(nil)
+
+func (j JobID) MarshalJSON() ([]byte, error) { return json.Marshal(j.jid) }
+
+func (j *JobID) UnmarshalJSON(b []byte) error {
+	return json.Unmarshal(b, &j.jid)
+}
+
+func (j JobID) MustValidate() { j.expectInitialized() }

go.mod

@@ -3,80 +3,36 @@ module github.com/zrepl/zrepl
 go 1.12
 
 require (
-	github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 // indirect
-	github.com/alvaroloes/enumer v1.1.1
-	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/fatih/color v1.7.0
-	github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff // indirect
 	github.com/gdamore/tcell v1.2.0
-	github.com/go-critic/go-critic v0.3.4 // indirect
-	github.com/go-logfmt/logfmt v0.3.0
+	github.com/go-logfmt/logfmt v0.4.0
 	github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4
-	github.com/gogo/protobuf v1.2.1 // indirect
-	github.com/golang/mock v1.2.0 // indirect
-	github.com/golang/protobuf v1.2.0
-	github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 // indirect
-	github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 // indirect
-	github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d // indirect
-	github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 // indirect
-	github.com/golangci/golangci-lint v1.17.1
-	github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb // indirect
-	github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 // indirect
-	github.com/golangci/misspell v0.3.4 // indirect
-	github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 // indirect
+	github.com/golang/protobuf v1.3.2
 	github.com/google/uuid v1.1.1
 	github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8
-	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
-	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
-	github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 // indirect
 	github.com/kr/pretty v0.1.0
 	github.com/lib/pq v1.2.0
-	github.com/mattn/go-isatty v0.0.3
-	github.com/matttproud/golang_protobuf_extensions v1.0.0 // indirect
-	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mattn/go-colorable v0.1.4 // indirect
+	github.com/mattn/go-isatty v0.0.8
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // // go1.12 mod tidy adds this dependency as 'indirect', but go1.13 mod tidy removes it if the trailing comment is 'indirect' => add this comment to make the build work without changing go.mod on both go1.12 and go1.13
+	github.com/modern-go/reflect2 v1.0.1 // go1.12 mod tidy adds this dependency as 'indirect', but go1.13 mod tidy removes it if the trailing comment is 'indirect' => add this comment to make the build work without changing go.mod on both go1.12 and go1.13
 	github.com/montanaflynn/stats v0.5.0
-	github.com/pkg/errors v0.8.0
+	github.com/onsi/ginkgo v1.10.2 // indirect
+	github.com/onsi/gomega v1.7.0 // indirect
+	github.com/pkg/errors v0.8.1
 	github.com/pkg/profile v1.2.1
-	github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284
-	github.com/prometheus/client_golang v0.0.0-20180410130117-e11c6ff8170b
-	github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 // indirect
-	github.com/prometheus/common v0.0.0-20180413074202-d0f7cd64bda4 // indirect
-	github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d // indirect
-	github.com/sergi/go-diff v1.0.0 // indirect
-	github.com/sirupsen/logrus v1.4.0 // indirect
-	github.com/spf13/afero v1.2.2 // indirect
+	github.com/problame/go-netssh v0.0.0-20200601114649-26439f9f0dc5
+	github.com/prometheus/client_golang v1.2.1
+	github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 // go1.12 thinks it needs this
 	github.com/spf13/cobra v0.0.2
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
-	github.com/spf13/pflag v1.0.3
-	github.com/spf13/viper v1.3.2 // indirect
+	github.com/spf13/pflag v1.0.5
 	github.com/stretchr/testify v1.4.0
-	github.com/theckman/goconstraint v1.11.0 // indirect
 	github.com/yudai/gojsondiff v0.0.0-20170107030110-7b1b7adf999d
-	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect
-	github.com/yudai/pp v2.0.1+incompatible // indirect
-	github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f
-	golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c // indirect
-	golang.org/x/net v0.0.0-20190313220215-9f648a60d977
+	github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // go1.12 thinks it needs this
+	github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd
+	golang.org/x/net v0.0.0-20190613194153-d28f0bde5980
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58
-	golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756
-	golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b
-	google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 // indirect
+	golang.org/x/sys v0.0.0-20191026070338-33540a1f6037
+	gonum.org/v1/gonum v0.7.0 // indirect
 	google.golang.org/grpc v1.17.0
-	mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe // indirect
-	sourcegraph.com/sqs/pbtypes v1.0.0 // indirect
-)
-
-// invalid dates in transitive dependencies (first validated in Go 1.13, didn't fail in earlier Go versions)
-replace (
-	github.com/go-critic/go-critic v0.0.0-20181204210945-1df300866540 => github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540
-	github.com/go-critic/go-critic v0.0.0-20181204210945-ee9bf5809ead => github.com/go-critic/go-critic v0.3.5-0.20190210220443-ee9bf5809ead
-	github.com/golangci/errcheck v0.0.0-20181003203344-ef45e06d44b6 => github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6
-	github.com/golangci/go-tools v0.0.0-20180109140146-35a9f45a5db0 => github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0
-	github.com/golangci/go-tools v0.0.0-20180109140146-af6baa5dc196 => github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196
-	github.com/golangci/gofmt v0.0.0-20181105071733-0b8337e80d98 => github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98
-	github.com/golangci/gosec v0.0.0-20180901114220-66fb7fc33547 => github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547
-	github.com/golangci/ineffassign v0.0.0-20180808204949-42439a7714cc => github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc
-	github.com/golangci/lint-1 v0.0.0-20180610141402-ee948d087217 => github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217
-	golang.org/x/tools v0.0.0-20190125232054-379209517ffe => golang.org/x/tools v0.0.0-20190205201329-379209517ffe
-	mvdan.cc/unparam v0.0.0-20190124213536-fbb59629db34 => mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34
 )

go.sum

@@ -1,17 +1,24 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08=
 github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/OpenPeeDeeP/depguard v0.0.0-20180806142446-a69c782687b2/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
-github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810 h1:pFks+oaqVWDFq0KsLQZFB2pQGB5Us0HfMSBENtieXOs=
 github.com/OpenPeeDeeP/depguard v0.0.0-20181229194401-1f388ab2d810/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
-github.com/alvaroloes/enumer v1.1.1 h1:v+1scNqEhSFAPb9tLwblQegeTXJhnw8hf9O0amTbOvQ=
+github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alvaroloes/enumer v1.1.1/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.1.0 h1:yTUvW7Vhb89inJ+8irsUqiWjh8iT6sQPZiQzI6ReGkA=
+github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
@@ -22,6 +29,7 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ftrvxmtrx/fd v0.0.0-20150925145434-c6d800382fff h1:zk1wwii7uXmI0znwU+lqg+wFL9G5+vm5I+9rv2let60=
@@ -30,107 +38,83 @@ github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdk
 github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
 github.com/gdamore/tcell v1.2.0 h1:ikixzsxc8K8o3V2/CEmyoEW8mJZaNYQQ3NP3VIQdUe4=
 github.com/gdamore/tcell v1.2.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM=
-github.com/go-critic/go-critic v0.3.4 h1:FYaiaLjX0Nqei80KPhm4CyFQUBbmJwSrHxQ73taaGBc=
 github.com/go-critic/go-critic v0.3.4/go.mod h1:AHR42Lk/E/aOznsrYdMYeIQS5RH10HZHSqP+rD6AJrc=
 github.com/go-critic/go-critic v0.3.5-0.20190526074819-1df300866540/go.mod h1:+sE8vrLDS2M0pZkBk0wy6+nLdKexVDrl/jBqQOTDThA=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-lintpack/lintpack v0.5.1/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
-github.com/go-lintpack/lintpack v0.5.2 h1:DI5mA3+eKdWeJ40nU4d6Wc26qmdG8RCi/btYq0TuRN0=
 github.com/go-lintpack/lintpack v0.5.2/go.mod h1:NwZuYi2nUHho8XEIZ6SIxihrnPoqBTDqfpXvXAN0sXM=
 github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2ic=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8=
 github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4 h1:0suja/iKSDbEIYLbrS/8C7iArJiWpgCNcR+zwAHu7Ig=
 github.com/go-sql-driver/mysql v1.4.1-0.20190907122137-b2c03bcae3d4/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
-github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
-github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/go-toolsmith/astcast v0.0.0-20181028201508-b7a89ed70af1/go.mod h1:TEo3Ghaj7PsZawQHxT/oBvo4HK/sl1RcuUHDKTTju+o=
-github.com/go-toolsmith/astcast v1.0.0 h1:JojxlmI6STnFVG9yOImLeGREv8W2ocNUM+iOhR6jE7g=
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
 github.com/go-toolsmith/astcopy v0.0.0-20180903214859-79b422d080c4/go.mod h1:c9CPdq2AzM8oPomdlPniEfPAC6g1s7NqZzODt8y6ib8=
-github.com/go-toolsmith/astcopy v1.0.0 h1:OMgl1b1MEpjFQ1m5ztEO06rz5CUd3oBv9RF7+DyvdG8=
 github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
 github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
-github.com/go-toolsmith/astequal v1.0.0 h1:4zxD8j3JRFNyLN46lodQuqz3xdKSrur7U/sr0SDS/gQ=
 github.com/go-toolsmith/astequal v1.0.0/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
 github.com/go-toolsmith/astfmt v0.0.0-20180903215011-8f8ee99c3086/go.mod h1:mP93XdblcopXwlyN4X4uodxXQhldPGZbcEJIimQHrkg=
-github.com/go-toolsmith/astfmt v1.0.0 h1:A0vDDXt+vsvLEdbMFJAUBI/uTbRw1ffOPnxsILnFL6k=
 github.com/go-toolsmith/astfmt v1.0.0/go.mod h1:cnWmsOAuq4jJY6Ct5YWlVLmcmLMn1JUPuQIHCY7CJDw=
 github.com/go-toolsmith/astinfo v0.0.0-20180906194353-9809ff7efb21/go.mod h1:dDStQCHtmZpYOmjRP/8gHHnCCch3Zz3oEgCdZVdtweU=
 github.com/go-toolsmith/astp v0.0.0-20180903215135-0af7e3c24f30/go.mod h1:SV2ur98SGypH1UjcPpCatrV5hPazG6+IfNHbkDXBRrk=
-github.com/go-toolsmith/astp v1.0.0 h1:alXE75TXgcmupDsMK1fRAy0YUzLzqPVvBKoyWV+KPXg=
 github.com/go-toolsmith/astp v1.0.0/go.mod h1:RSyrtpVlfTFGDYRbrjyWP1pYu//tSFcvdYrA8meBmLI=
-github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8 h1:vVouagbdmqTVlCIAxpyYsNNTbkKZ3V66VpKOLU/s6W4=
 github.com/go-toolsmith/pkgload v0.0.0-20181119091011-e9e65178eee8/go.mod h1:WoMrjiy4zvdS+Bg6z9jZH82QXwkcgCBX6nOfnmdaHks=
-github.com/go-toolsmith/pkgload v1.0.0 h1:4DFWWMXVfbcN5So1sBNW9+yeiMqLFGl1wFLTL5R0Tgg=
 github.com/go-toolsmith/pkgload v1.0.0/go.mod h1:5eFArkbO80v7Z0kdngIxsRXRMTaX4Ilcwuh3clNrQJc=
 github.com/go-toolsmith/strparse v0.0.0-20180903215201-830b6daa1241/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
-github.com/go-toolsmith/strparse v1.0.0 h1:Vcw78DnpCAKlM20kSbAyO4mPfJn/lyYA4BJUDxe2Jb4=
 github.com/go-toolsmith/strparse v1.0.0/go.mod h1:YI2nUKP9YGZnL/L1/DLFBfixrcjslWct4wyljWhSRy8=
 github.com/go-toolsmith/typep v0.0.0-20181030061450-d63dc7650676/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
-github.com/go-toolsmith/typep v1.0.0 h1:zKymWyA1TRYvqYrYDrfEMZULyrhcnGY3x7LDKU2XQaA=
 github.com/go-toolsmith/typep v1.0.0/go.mod h1:JSQCQMUPdRlMZFswiq3TGpNp1GMktqkR2Ns5AIQkATU=
-github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y=
 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
 github.com/golang/mock v1.0.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/mock v1.2.0 h1:28o5sBqPkBsMGnC6b4MvE2TzSr5/AT4c/1fLqVGIwlk=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2 h1:23T5iq8rbUYlhpt5DB4XJkc6BU31uODLD1o1gKvZmD0=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golangci/check v0.0.0-20180506172741-cfe4005ccda2/go.mod h1:k9Qvh+8juN+UKMCS/3jFtGICgW8O96FVaZsaxdzDkR4=
-github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a h1:w8hkcTqaFpzKqonE9uMCefW1WDie15eSP/4MssdenaM=
 github.com/golangci/dupl v0.0.0-20180902072040-3e9179ac440a/go.mod h1:ryS0uhF+x9jgbj/N71xsEqODy9BN81/GonCZiOzirOk=
-github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6 h1:YYWNAGTKWhKpcLLt7aSj/odlKrSrelQwlovBpDuf19w=
 github.com/golangci/errcheck v0.0.0-20181223084120-ef45e06d44b6/go.mod h1:DbHgvLiFKX1Sh2T1w8Q/h4NAI8MHIpzCdnBUDTXU3I0=
-github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613 h1:9kfjN3AdxcbsZBf8NjltjWihK2QfBBBZuv91cMFfDHw=
 github.com/golangci/go-misc v0.0.0-20180628070357-927a3d87b613/go.mod h1:SyvUF2NxV+sN8upjjeVYr5W7tyxaT1JVtvhKhOn2ii8=
-github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0 h1:MRhC9XbUjE6XDOInSJ8pwHuPagqsyO89QDU9IdVhe3o=
 github.com/golangci/go-tools v0.0.0-20190124090046-35a9f45a5db0/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
 github.com/golangci/go-tools v0.0.0-20190318060251-af6baa5dc196/go.mod h1:unzUULGw35sjyOYjUt0jMTXqHlZPpPc6e+xfO4cd6mM=
-github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3 h1:pe9JHs3cHHDQgOFXJJdYkK6fLz2PWyYtP4hthoCMvs8=
 github.com/golangci/goconst v0.0.0-20180610141641-041c5f2b40f3/go.mod h1:JXrF4TWy4tXYn62/9x8Wm/K/dm06p8tCKwFRDPZG/1o=
 github.com/golangci/gocyclo v0.0.0-20180528134321-2becd97e67ee/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d h1:pXTK/gkVNs7Zyy7WKgLXmpQ5bHTrq5GDsp8R9Qs67g0=
 github.com/golangci/gocyclo v0.0.0-20180528144436-0a533e8fa43d/go.mod h1:ozx7R9SIwqmqf5pRP90DhR2Oay2UIjGuKheCBCNwAYU=
-github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98 h1:0OkFarm1Zy2CjCiDKfK9XHgmc2wbDlRMD2hD8anAJHU=
 github.com/golangci/gofmt v0.0.0-20181222123516-0b8337e80d98/go.mod h1:9qCChq59u/eW8im404Q2WWTrnBUQKjpNYKMbU4M7EFU=
-github.com/golangci/golangci-lint v1.17.1 h1:lc8Hf9GPCjIr0hg3S/xhvFT1+Hydass8F1xchr8jkME=
 github.com/golangci/golangci-lint v1.17.1/go.mod h1:+5sJSl2h3aly+fpmL2meSP8CaSKua2E4Twi9LPy7b1g=
-github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb h1:Bi7BYmZVg4C+mKGi8LeohcP2GGUl2XJD4xCkJoZSaYc=
 github.com/golangci/gosec v0.0.0-20180901114220-8afd9cbb6cfb/go.mod h1:ON/c2UR0VAAv6ZEAFKhjCLplESSmRFfZcDLASbI1GWo=
 github.com/golangci/gosec v0.0.0-20190211064107-66fb7fc33547/go.mod h1:0qUabqiIQgfmlAmulqxyiGkkyF6/tOGSnY2cnPVwrzU=
-github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc h1:gLLhTLMk2/SutryVJ6D4VZCU3CUqr8YloG7FPIBWFpI=
 github.com/golangci/ineffassign v0.0.0-20190609212857-42439a7714cc/go.mod h1:e5tpTHCfVze+7EpLEozzMB3eafxo2KT5veNg1k6byQU=
-github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219 h1:utua3L2IbQJmauC5IXdEA547bcoU5dozgQAfc8Onsg4=
 github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
 github.com/golangci/lint-1 v0.0.0-20190420132249-ee948d087217/go.mod h1:66R6K6P6VWk9I95jvqGxkqJxVWGFy9XlDwLwVz1RCFg=
-github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca h1:kNY3/svz5T29MYHubXix4aDDuE3RWHkPvopM/EDv/MA=
 github.com/golangci/maligned v0.0.0-20180506175553-b1d89398deca/go.mod h1:tvlJhZqDe4LMs4ZHD0oMUlt9G2LWuDGoisJTBzLMV9o=
 github.com/golangci/misspell v0.0.0-20180809174111-950f5d19e770/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/misspell v0.3.4 h1:kAD5J0611Zo2VirUn9KFP15RjEqkmfEfnFxyIVxZCYg=
 github.com/golangci/misspell v0.3.4/go.mod h1:dEbvlSfYbMQDtrpRMQU675gSDLDNa8sCPPChZ7PhiVA=
-github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21 h1:leSNB7iYzLYSSx3J/s5sVf4Drkc68W2wm4Ixh/mr0us=
 github.com/golangci/prealloc v0.0.0-20180630174525-215b22d4de21/go.mod h1:tf5+bzsHdTM0bsB7+8mt0GUMvjCgwLpTapNZHU8AajI=
 github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
-github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039 h1:XQKc8IYQOeRwVs36tDrEmTgDgP88d5iEURwpmtiAlOM=
 github.com/golangci/revgrep v0.0.0-20180812185044-276a5c0a1039/go.mod h1:qOQCunEYvmd/TLamH+7LlVccLvUH5kZNhbCgTHoBbp4=
-github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4 h1:zwtduBRr5SSWhqsYNgcuWO2kFlpdOZbP0+yRjmvPGys=
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
-github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3 h1:JVnpOZS+qxli+rgVl98ILOXVNbW+kb5wcxeGx8ShUIw=
 github.com/gostaticanalysis/analysisutil v0.0.0-20190318220348-4088753ea4d3/go.mod h1:eEOZF4jCKGi+aprrirO9e7WKB3beBRtWgqGunKl6pKE=
 github.com/hashicorp/hcl v0.0.0-20180404174102-ef8a98b0bbce/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w=
-github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
@@ -138,6 +122,10 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8 h1:+dKzeuiDYbD/Cfi/sEkkNrs6z4/WZ8ZIR8dAbUpjXbU=
 github.com/jinzhu/copier v0.0.0-20170922082739-db4671f3a9b8/go.mod h1:yL958EeXv8Ylng6IfnvG4oflryUi3vgA3xPs9hmII1s=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
@@ -149,7 +137,6 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
-github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -164,102 +151,128 @@ github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQ
 github.com/lucasb-eyer/go-colorful v1.0.2 h1:mCMFu6PgSozg9tDNMMK3g18oJBX7oYGrC09mS6CXfO4=
 github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
 github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
-github.com/matttproud/golang_protobuf_extensions v1.0.0 h1:YNOwxxSJzSUARoD9KRZLzM9Y858MNGCOACTvCW9TSAc=
 github.com/matttproud/golang_protobuf_extensions v1.0.0/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzOPt1ZSrGZWFoNhsAedKnEd6r9Np1+5blZCWk=
 github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
-github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/montanaflynn/stats v0.5.0 h1:2EkzeTSqBB4V4bJwWrt5gIIrZmpJBcoIRGS2kWLgzmk=
 github.com/montanaflynn/stats v0.5.0/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
 github.com/mozilla/tls-observatory v0.0.0-20180409132520-8791a200eb40/go.mod h1:SrKMQvPiws7F7iqYp8/TX+IhxCYhzr6N/1yb8cwHsGk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/nbutton23/zxcvbn-go v0.0.0-20160627004424-a22cb81b2ecd/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
-github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663 h1:Ri1EhipkbhWsffPJ3IPlrb4SkTOPa2PfRXp3jchBczw=
 github.com/nbutton23/zxcvbn-go v0.0.0-20171102151520-eafdab6b0663/go.mod h1:o96djdrsSGy3AWPyBgZMAGfxZNfgntdJG+11KU4QvbU=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94=
+github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.2 h1:3mYCb7aPxS/RU7TI1y4rkEn1oKmPRjNJLNEXgw7MH2I=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
-github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1 h1:/I3lTljEEDNYLho3/FUB7iD/oc2cEFgVmbHzV+O0PtU=
+github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
 github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
-github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE=
 github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284 h1:GX8YFBfSL6GlKymQZOsQllPbxVamfEYpis7oaJLXYZA=
 github.com/problame/go-netssh v0.0.0-20190110232351-09d6bc45d284/go.mod h1:XMR9fjP5/wMyFK0Ifk9ZHDFem4Rukm+ChvoYr2nxrgw=
-github.com/prometheus/client_golang v0.0.0-20180410130117-e11c6ff8170b h1:ymnAeobXUW0sfmt5n8euMzW4vgdQvZNTagdYMcpK63g=
+github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1 h1:HH8yzlaZq/A8xdJaSj/eu32yFFySWAJMTmOQPAGwfYg=
+github.com/problame/go-netssh v0.0.0-20191026123024-f34099f4f6b1/go.mod h1:JRs3nyBjvOv/9YHC+Vlw9z1Jfzl5s5t0BNnTzmclj1c=
+github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7 h1:Oik8tLrLhoMq4fduDRuNNOAQ+q0M0dXkjAYLUf4+mAc=
+github.com/problame/go-netssh v0.0.0-20191209123953-18d8aa6923c7/go.mod h1:Ba6RaFpK1+IHWs1wLZ6sCBuXkt4iAVLeOG5GinXHusM=
+github.com/problame/go-netssh v0.0.0-20200601114649-26439f9f0dc5 h1:1eSrO2WAeSXjMol8WRKW9LekL1252VIMaKQwWkmEdvs=
+github.com/problame/go-netssh v0.0.0-20200601114649-26439f9f0dc5/go.mod h1:Ba6RaFpK1+IHWs1wLZ6sCBuXkt4iAVLeOG5GinXHusM=
 github.com/prometheus/client_golang v0.0.0-20180410130117-e11c6ff8170b/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
-github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5 h1:cLL6NowurKLMfCeQy4tIeph12XNQWgANCNvdyrOYKV4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.2.1 h1:JnMpQc6ppsNgw9QPAGF6Dod479itz7lvlsMzzNayLOI=
+github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/common v0.0.0-20180413074202-d0f7cd64bda4 h1:rPmL0pCWefiqV7RjMYuBUrLbMJmNVjRKz82caELPTUQ=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180413074202-d0f7cd64bda4/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d h1:RCcsxyRr6+/pLg6wr0cUjPovhEhSNOtPh0SOz6u3hGU=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.7.0 h1:L+1lyG48J1zAQXA3RBX/nG/B3gjlHq0zTt2tlbJLyCY=
+github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA=
 github.com/prometheus/procfs v0.0.0-20180408092902-8b1c2da0d56d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
+github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.2.1/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44 h1:tB9NOR21++IjLyVx3/PCPhWMwqGNCMQEH96A6dMZ/gc=
+github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shirou/gopsutil v0.0.0-20180427012116-c95755e4bcd7/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA=
 github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc=
-github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e h1:MZM7FHLqUHYI0Y/mQAt3d2aYa0SiNms/hFqC9qJYolM=
 github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
-github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041 h1:llrF3Fs4018ePo4+G/HV/uQUqEI1HMDjCeOf2V6puPc=
 github.com/shurcooL/go-goon v0.0.0-20170922171312-37c2f522c041/go.mod h1:N5mDOmsrJOB+vfqUK+7DmDyjhSLIIBnXo9lvZJj3MWQ=
 github.com/sirupsen/logrus v1.0.5/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
-github.com/sirupsen/logrus v1.4.0 h1:yKenngtzGh+cUSSh6GWbxW2abRqhYUSR/t/6+2QqNvE=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
-github.com/sourcegraph/go-diff v0.5.1 h1:gO6i5zugwzo1RVTvgvfwCOSVegNuvnNi6bAD1QCmkHs=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sourcegraph/go-diff v0.5.1/go.mod h1:j2dHj3m8aZgQO8lMTcTnBcXkRRRqi34cd2MNlA9u1mE=
 github.com/spf13/afero v1.1.0/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
-github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
-github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.2 h1:NfkwRbgViGoyjBKsLI0QMDcuMnhM+SBg3T0cGfpvKDE=
 github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
-github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
-github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/theckman/goconstraint v1.11.0 h1:oBUwN5wpE4dwyPhRGraEgJsFTr+JtLWiDnaJZJeeXI0=
 github.com/theckman/goconstraint v1.11.0/go.mod h1:zkCR/f2kOULTk/h1ujgyB9BlCNLaqlQ6GN2Zl4mg81g=
-github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb h1:lI9ufgFfvuqRctP9Ny8lDDLbSWCMxBPletcSqrnyFYM=
 github.com/timakin/bodyclose v0.0.0-20190407043127-4a873e97b2bb/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
@@ -273,16 +286,21 @@ github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 h1:BHyfKlQyqbsFN5p3Ifn
 github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
 github.com/yudai/pp v2.0.1+incompatible h1:Q4//iY4pNF6yPLZIigmvcl7k/bPgrcTPIFIcmawg5bI=
 github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
-github.com/zrepl/yaml-config v0.0.0-20190903123849-809f4603401a h1:l37btErUqI1yNmt5mf8kzglbeQfo32+TRacnxthiuck=
-github.com/zrepl/yaml-config v0.0.0-20190903123849-809f4603401a/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
 github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f h1:3MuiGfgMHCSwKUcsuI7ODbi50j+evTB7SsoOBMNC5Fk=
 github.com/zrepl/yaml-config v0.0.0-20190928121844-af7ca3f8448f/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
+github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd h1:SSo67WLS+99QESvbW8Meibz7zCrxshP71U9dH5KOCXM=
+github.com/zrepl/yaml-config v0.0.0-20191220194647-cbb6b0cf4bdd/go.mod h1:JmNwisZzOvW4GfpfLvhZ+gtyKLsIiA+WC+wNKJGJaFg=
+github.com/zrepl/zrepl v0.2.0/go.mod h1:M3Zv2IGSO8iYpUjsZD6ayZ2LHy7zyMfzet9XatKOrZ8=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c h1:Vj5n4GlwjmQteupaxJ9+0FNOmBrHfq7vN4btdGoDZgI=
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU=
+golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -290,41 +308,59 @@ golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190313220215-9f648a60d977 h1:actzWV6iWn3GLqN8dZjzsB+CLt+gaV2+wsxroxiQI8I=
 golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756 h1:9nuHUbU8dRnRRfj9KjWUVrJeoexdbeMjttk6Oh1rD10=
 golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.0.0-20170915090833-1cbadb444a80/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181205014116-22934f0fdb62/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190121143147-24cd39ecf745/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8 h1:b0PLhFjEMqrIqsD4rS5sp0YxBYgdqKzX0KkkGGKLV8I=
+golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190213192042-740235f6c0d8/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190311215038-5c2858a9cfe5/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190322203728-c1a832b0ad89/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b h1:iEAPfYPbYbxG/2lNN4cMOHkmgKNsCuUwkxlDCK46UlU=
 golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo=
+gonum.org/v1/gonum v0.7.0 h1:Hdks0L0hgznZLG9nzXb8vZ0rRvqNvAcgAp84y7Mwkgw=
+gonum.org/v1/gonum v0.7.0/go.mod h1:L02bwd0sqlsvRv41G7wGWFCsVNZFv/k1xzGIxeANHGM=
+gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw=
+gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc=
+google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898 h1:yvw+zsSmSM02Z5H3ZdEV7B7Ql7eFrjQTnmByJvK+3J8=
 google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
@@ -332,6 +368,8 @@ google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9M
 google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
 google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -346,13 +384,10 @@ gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed h1:WX1yoOaKQfddO/mLzdV4wptyWgoH/6hwLs7QHTixo0I=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
-mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b h1:DxJ5nJdkhDlLok9K6qO+5290kphDJbHOQO1DFFFTeBo=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
-mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe h1:Ekmnp+NcP2joadI9pbK4Bva87QKZSeY7le//oiMrc9g=
 mvdan.cc/unparam v0.0.0-20190310220240-1b9ccfa71afe/go.mod h1:BnhuWBAqxH3+J5bDybdxgw5ZfS+DsVd4iylsKQePN8o=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 sourcegraph.com/sqs/pbtypes v0.0.0-20180604144634-d3ebe8f20ae4/go.mod h1:ketZ/q3QxT9HOBeFhu6RdvsftgpsbFHBF5Cas6cDKZ0=
-sourcegraph.com/sqs/pbtypes v1.0.0 h1:f7lAwqviDEGvON4kRv0o5V7FT/IQK+tbkF664XMbP3o=
 sourcegraph.com/sqs/pbtypes v1.0.0/go.mod h1:3AciMUv4qUuRHRHhOG4TZOB+72GdPVz5k+c648qsFS4=

lazy.sh

@@ -27,7 +27,8 @@ fi
 CHECKOUTPATH="${GOPATH}/src/github.com/zrepl/zrepl"
 
 godep() {
-    step "install build dependencies (versions pinned using go.mod and tools.go)"
+    step "install build dependencies (versions pinned in build/go.mod and build/tools.go)"
+    pushd "$(dirname "${BASH_SOURCE[0]}")"/build
     set -x
     export GO111MODULE=on # otherwise, a checkout of this repo in GOPATH will disable modules on Go 1.12 and earlier
     go build -v -mod=readonly -o "$GOPATH/bin/stringer"      golang.org/x/tools/cmd/stringer
@@ -36,6 +37,7 @@ godep() {
     go build -v -mod=readonly -o "$GOPATH/bin/goimports"     golang.org/x/tools/cmd/goimports
     go build -v -mod=readonly -o "$GOPATH/bin/golangci-lint" github.com/golangci/golangci-lint/cmd/golangci-lint
     set +x
+    popd
     if ! type stringer || ! type protoc-gen-go || ! type enumer || ! type goimports || ! type golangci-lint; then
         echo "Installed dependencies but can't find them in \$PATH, adjust it to contain \$GOPATH/bin" 1>&2
         exit 1

main.go

@@ -17,6 +17,7 @@ func init() {
 	cli.AddSubcommand(client.PprofCmd)
 	cli.AddSubcommand(client.TestCmd)
 	cli.AddSubcommand(client.MigrateCmd)
+	cli.AddSubcommand(client.ZFSAbstractionsCmd)
 }
 
 func main() {

platformtest/harness/harness.go

@@ -4,11 +4,14 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"os"
 	"path/filepath"
+	"regexp"
 	"time"
 
 	"github.com/fatih/color"
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/config"
 	"github.com/zrepl/zrepl/daemon/logging"
 	"github.com/zrepl/zrepl/logger"
@@ -16,17 +19,38 @@ import (
 	"github.com/zrepl/zrepl/platformtest/tests"
 )
 
-func main() {
+var bold = color.New(color.Bold)
+var boldRed = color.New(color.Bold, color.FgHiRed)
+var boldGreen = color.New(color.Bold, color.FgHiGreen)
 
-	var args struct {
-		createArgs platformtest.ZpoolCreateArgs
+var args struct {
+	createArgs            platformtest.ZpoolCreateArgs
+	stopAndKeepPoolOnFail bool
+
+	run   string
+	runRE *regexp.Regexp
+}
+
+func main() {
+	if err := doMain(); err != nil {
+		os.Exit(1)
 	}
+}
+
+var exitWithErr = fmt.Errorf("exit with error")
+
+func doMain() error {
+
 	flag.StringVar(&args.createArgs.PoolName, "poolname", "", "")
 	flag.StringVar(&args.createArgs.ImagePath, "imagepath", "", "")
-	flag.Int64Var(&args.createArgs.ImageSize, "imagesize", 100*(1<<20), "")
-	flag.StringVar(&args.createArgs.Mountpoint, "mountpoint", "none", "")
+	flag.Int64Var(&args.createArgs.ImageSize, "imagesize", 200*(1<<20), "")
+	flag.StringVar(&args.createArgs.Mountpoint, "mountpoint", "", "")
+	flag.BoolVar(&args.stopAndKeepPoolOnFail, "failure.stop-and-keep-pool", false, "if a test case fails, stop test execution and keep pool as it was when the test failed")
+	flag.StringVar(&args.run, "run", "", "")
 	flag.Parse()
 
+	args.runRE = regexp.MustCompile(args.run)
+
 	outlets := logger.NewOutlets()
 	outlet, level, err := logging.ParseOutlet(config.LoggingOutletEnum{Ret: &config.StdoutLoggingOutlet{
 		LoggingOutletCommon: config.LoggingOutletCommon{
@@ -47,33 +71,132 @@ func main() {
 	ctx := platformtest.WithLogger(context.Background(), logger)
 	ex := platformtest.NewEx(logger)
 
-	bold := color.New(color.Bold)
+	type invocation struct {
+		runFunc tests.Case
+		result  *testCaseResult
+	}
+
+	invocations := make([]*invocation, 0, len(tests.Cases))
 	for _, c := range tests.Cases {
-		// ATTENTION future parallelism must pass c by value into closure!
-		err := func() error {
-			bold.Printf("BEGIN TEST CASE %s\n", c)
-			pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.createArgs)
-			if err != nil {
-				return errors.Wrap(err, "create test pool")
-			}
-			defer func() {
-				if err := pool.Destroy(ctx, ex); err != nil {
-					fmt.Printf("error destroying test pool: %s", err)
-				}
-			}()
-			ctx := &platformtest.Context{
-				Context:     ctx,
-				RootDataset: filepath.Join(pool.Name(), "rootds"),
-			}
-			c(ctx)
-			bold.Printf("DONE  TEST CASE %s\n", c)
-			fmt.Println()
-			return nil
-		}()
-		if err != nil {
-			bold.Printf("TEST CASE FAILED WITH ERROR:\n")
-			fmt.Println(err)
+		if args.runRE.MatchString(c.String()) {
+			invocations = append(invocations, &invocation{runFunc: c})
 		}
 	}
 
+	for _, inv := range invocations {
+
+		bold.Printf("BEGIN TEST CASE %s\n", inv.runFunc.String())
+
+		pool, err := platformtest.CreateOrReplaceZpool(ctx, ex, args.createArgs)
+		if err != nil {
+			panic(errors.Wrap(err, "create test pool"))
+		}
+
+		ctx := &platformtest.Context{
+			Context:     ctx,
+			RootDataset: filepath.Join(pool.Name(), "rootds"),
+		}
+
+		res := runTestCase(ctx, ex, inv.runFunc)
+		inv.result = res
+		if res.failed {
+			fmt.Printf("%+v\n", res.failedStack) // print with stack trace
+		}
+
+		if res.failed && args.stopAndKeepPoolOnFail {
+			boldRed.Printf("STOPPING TEST RUN AT FAILING TEST PER USER REQUEST\n")
+			return exitWithErr
+		}
+
+		if err := pool.Destroy(ctx, ex); err != nil {
+			panic(fmt.Sprintf("error destroying test pool: %s", err))
+		}
+
+		if res.failed {
+			boldRed.Printf("TEST FAILED\n")
+		} else if res.skipped {
+			bold.Printf("TEST SKIPPED\n")
+		} else if res.succeeded {
+			boldGreen.Printf("TEST PASSED\n")
+		} else {
+			panic("unreachable")
+		}
+
+		fmt.Println()
+	}
+
+	var summary struct {
+		succ, fail, skip []*invocation
+	}
+	for _, inv := range invocations {
+		var bucket *[]*invocation
+		if inv.result.failed {
+			bucket = &summary.fail
+		} else if inv.result.skipped {
+			bucket = &summary.skip
+		} else if inv.result.succeeded {
+			bucket = &summary.succ
+		} else {
+			panic("unreachable")
+		}
+		*bucket = append(*bucket, inv)
+	}
+	printBucket := func(bucketName string, c *color.Color, bucket []*invocation) {
+		c.Printf("%s:", bucketName)
+		if len(bucket) == 0 {
+			fmt.Printf(" []\n")
+			return
+		}
+		fmt.Printf("\n")
+		for _, inv := range bucket {
+			fmt.Printf("  %s\n", inv.runFunc.String())
+		}
+	}
+	printBucket("PASSING TESTS", boldGreen, summary.succ)
+	printBucket("SKIPPED TESTS", bold, summary.skip)
+	printBucket("FAILED TESTS", boldRed, summary.fail)
+
+	if len(summary.fail) > 0 {
+		return errors.New("at least one test failed")
+	}
+	return nil
+}
+
+type testCaseResult struct {
+	// oneof
+	failed, skipped, succeeded bool
+
+	failedStack error // has stack inside, valid if failed=true
+}
+
+func runTestCase(ctx *platformtest.Context, ex platformtest.Execer, c tests.Case) *testCaseResult {
+
+	// run case
+	var panicked = false
+	var panicValue interface{} = nil
+	var panicStack error
+	func() {
+		defer func() {
+			if item := recover(); item != nil {
+				panicValue = item
+				panicked = true
+				panicStack = errors.Errorf("panic while running test: %v", panicValue)
+			}
+		}()
+		c(ctx)
+	}()
+
+	if panicked {
+		switch panicValue {
+		case platformtest.SkipNowSentinel:
+			return &testCaseResult{skipped: true}
+		case platformtest.FailNowSentinel:
+			return &testCaseResult{failed: true, failedStack: panicStack}
+		default:
+			return &testCaseResult{failed: true, failedStack: panicStack}
+		}
+	} else {
+		return &testCaseResult{succeeded: true}
+	}
+
 }

platformtest/platformtest.go

@@ -2,6 +2,7 @@ package platformtest
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -12,13 +13,27 @@ type Context struct {
 	RootDataset string
 }
 
+var FailNowSentinel = fmt.Errorf("platformtest: FailNow called on context")
+
+var SkipNowSentinel = fmt.Errorf("platformtest: SkipNow called on context")
+
 var _ assert.TestingT = (*Context)(nil)
 var _ require.TestingT = (*Context)(nil)
 
+func (c *Context) Logf(format string, args ...interface{}) {
+	msg := fmt.Sprintf(format, args...)
+	GetLog(c).Info(msg)
+}
+
 func (c *Context) Errorf(format string, args ...interface{}) {
-	getLog(c).Printf(format, args...)
+	GetLog(c).Printf(format, args...)
+	c.FailNow()
 }
 
 func (c *Context) FailNow() {
-	panic(nil)
+	panic(FailNowSentinel)
+}
+
+func (c *Context) SkipNow() {
+	panic(SkipNowSentinel)
 }

platformtest/platformtest_exec.go

@@ -32,7 +32,7 @@ func (e *ex) runNoOutput(expectSuccess bool, ctx context.Context, cmd string, ar
 	buf, _ := circlog.NewCircularLog(32 << 10)
 	ecmd.Stdout, ecmd.Stderr = buf, buf
 	err := ecmd.Run()
-	log.Printf("command output: %s", buf.String())
+	log.WithField("output", buf.String()).Debug("command output")
 	if _, ok := err.(*exec.ExitError); err != nil && !ok {
 		panic(err)
 	}

platformtest/platformtest_logging.go

@@ -19,6 +19,6 @@ func WithLogger(ctx context.Context, logger Logger) context.Context {
 	return ctx
 }
 
-func getLog(ctx context.Context) Logger {
+func GetLog(ctx context.Context) Logger {
 	return ctx.Value(contextKeyLogger).(Logger)
 }

platformtest/platformtest_ops.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
@@ -23,6 +24,7 @@ type Stmt interface {
 type Op string
 
 const (
+	Comment         Op = "#"
 	AssertExists    Op = "!E"
 	AssertNotExists Op = "!N"
 	Add             Op = "+"
@@ -39,15 +41,16 @@ type DestroyRootOp struct {
 func (o *DestroyRootOp) Run(ctx context.Context, e Execer) error {
 	// early-exit if it doesn't exist
 	if err := e.RunExpectSuccessNoOutput(ctx, "zfs", "get", "-H", "name", o.Path); err != nil {
-		getLog(ctx).WithField("root_ds", o.Path).Info("assume root ds doesn't exist")
+		GetLog(ctx).WithField("root_ds", o.Path).Info("assume root ds doesn't exist")
 		return nil
 	}
 	return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", "-r", o.Path)
 }
 
 type FSOp struct {
-	Op   Op
-	Path string
+	Op        Op
+	Path      string
+	Encrypted bool // only for Op=Add
 }
 
 func (o *FSOp) Run(ctx context.Context, e Execer) error {
@@ -57,7 +60,22 @@ func (o *FSOp) Run(ctx context.Context, e Execer) error {
 	case AssertNotExists:
 		return e.RunExpectFailureNoOutput(ctx, "zfs", "get", "-H", "name", o.Path)
 	case Add:
-		return e.RunExpectSuccessNoOutput(ctx, "zfs", "create", o.Path)
+		opts := []string{"create"}
+		if o.Encrypted {
+			const passphraseFilePath = "/tmp/zreplplatformtest.encryption.passphrase"
+			const passphrase = "foobar2342"
+			err := ioutil.WriteFile(passphraseFilePath, []byte(passphrase), 0600)
+			if err != nil {
+				panic(err)
+			}
+			opts = append(opts,
+				"-o", "encryption=on",
+				"-o", "keylocation=file:///"+passphraseFilePath,
+				"-o", "keyformat=passphrase",
+			)
+		}
+		opts = append(opts, o.Path)
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", opts...)
 	case Del:
 		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Path)
 	default:
@@ -85,6 +103,26 @@ func (o *SnapOp) Run(ctx context.Context, e Execer) error {
 	}
 }
 
+type BookmarkOp struct {
+	Op       Op
+	Existing string
+	Bookmark string
+}
+
+func (o *BookmarkOp) Run(ctx context.Context, e Execer) error {
+	switch o.Op {
+	case Add:
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "bookmark", o.Existing, o.Bookmark)
+	case Del:
+		if o.Existing != "" {
+			panic("existing must be empty for destroy, got " + o.Existing)
+		}
+		return e.RunExpectSuccessNoOutput(ctx, "zfs", "destroy", o.Bookmark)
+	default:
+		panic(o.Op)
+	}
+}
+
 type RunOp struct {
 	RootDS string
 	Script string
@@ -94,7 +132,7 @@ func (o *RunOp) Run(ctx context.Context, e Execer) error {
 	cmd := exec.CommandContext(ctx, "/usr/bin/env", "bash", "-c", o.Script)
 	cmd.Env = os.Environ()
 	cmd.Env = append(cmd.Env, fmt.Sprintf("ROOTDS=%s", o.RootDS))
-	log := getLog(ctx).WithField("script", o.Script)
+	log := GetLog(ctx).WithField("script", o.Script)
 	log.Info("start script")
 	defer log.Info("script done")
 	output, err := cmd.CombinedOutput()
@@ -126,7 +164,7 @@ func Run(ctx context.Context, rk RunKind, rootds string, stmtsStr string) {
 	if err != nil {
 		panic(err)
 	}
-	execer := NewEx(getLog(ctx))
+	execer := NewEx(GetLog(ctx))
 	for _, s := range stmt {
 		err := s.Run(ctx, execer)
 		if err == nil {
@@ -163,8 +201,8 @@ func splitQuotedWords(data []byte, atEOF bool) (advance int, token []byte, err e
 				// unescaped quote, end of this string
 				// remove backslash-escapes
 				withBackslash := data[begin+1 : end]
-				withoutBaskslash := bytes.Replace(withBackslash, []byte("\\\""), []byte("\""), -1)
-				return end + 1, withoutBaskslash, nil
+				withoutBackslash := bytes.Replace(withBackslash, []byte("\\\""), []byte("\""), -1)
+				return end + 1, withoutBackslash, nil
 			} else {
 				// continue to next quote
 				end += 1
@@ -238,18 +276,46 @@ nextLine:
 			op = AssertExists
 		case string(AssertNotExists):
 			op = AssertNotExists
+		case string(Comment):
+			op = Comment
+			continue
 		default:
 			return nil, &LineError{scan.Text(), fmt.Sprintf("invalid op %q", comps.Text())}
 		}
 
-		// FS / SNAP
+		// FS / SNAP / BOOKMARK
 		if err := expectMoreTokens(); err != nil {
 			return nil, err
 		}
 		if strings.ContainsAny(comps.Text(), "@") {
 			stmts = append(stmts, &SnapOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+		} else if strings.ContainsAny(comps.Text(), "#") {
+			bookmark := fmt.Sprintf("%s/%s", rootds, comps.Text())
+			if err := expectMoreTokens(); err != nil {
+				return nil, err
+			}
+			existing := fmt.Sprintf("%s/%s", rootds, comps.Text())
+			stmts = append(stmts, &BookmarkOp{Op: op, Existing: existing, Bookmark: bookmark})
 		} else {
-			stmts = append(stmts, &FSOp{Op: op, Path: fmt.Sprintf("%s/%s", rootds, comps.Text())})
+			// FS
+			fs := comps.Text()
+			var encrypted bool = false
+			if op == Add {
+				if comps.Scan() {
+					t := comps.Text()
+					switch t {
+					case "encrypted":
+						encrypted = true
+					default:
+						panic(fmt.Sprintf("unexpected token %q", t))
+					}
+				}
+			}
+			stmts = append(stmts, &FSOp{
+				Op:        op,
+				Path:      fmt.Sprintf("%s/%s", rootds, fs),
+				Encrypted: encrypted,
+			})
 		}
 
 		if comps.Scan() {

platformtest/platformtest_zpool.go

@@ -2,10 +2,12 @@ package platformtest
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 
 	"github.com/pkg/errors"
+
 	"github.com/zrepl/zrepl/zfs"
 )
 
@@ -28,11 +30,11 @@ func (a ZpoolCreateArgs) Validate() error {
 	if a.ImageSize < minImageSize {
 		return errors.Errorf("ImageSize must be > %v, got %v", minImageSize, a.ImageSize)
 	}
-	if a.Mountpoint != "none" {
-		return errors.Errorf("Mountpoint must be \"none\"")
+	if a.Mountpoint == "" || a.Mountpoint[0] != '/' {
+		return errors.Errorf("Mountpoint must be an absolute path to a directory")
 	}
 	if a.PoolName == "" {
-		return errors.Errorf("PoolName must not be emtpy")
+		return errors.Errorf("PoolName must not be empty")
 	}
 	return nil
 }
@@ -43,7 +45,7 @@ func CreateOrReplaceZpool(ctx context.Context, e Execer, args ZpoolCreateArgs) (
 	}
 
 	// export pool if it already exists (idempotence)
-	if _, err := zfs.ZFSGetRawAnySource(args.PoolName, []string{"name"}); err != nil {
+	if _, err := zfs.ZFSGetRawAnySource(ctx, args.PoolName, []string{"name"}); err != nil {
 		if _, ok := err.(*zfs.DatasetDoesNotExist); ok {
 			// we'll create it shortly
 		} else {
@@ -68,7 +70,10 @@ func CreateOrReplaceZpool(ctx context.Context, e Execer, args ZpoolCreateArgs) (
 	image.Close()
 
 	// create the pool
-	err = e.RunExpectSuccessNoOutput(ctx, "zpool", "create", "-O", "mountpoint=none", args.PoolName, args.ImagePath)
+	err = e.RunExpectSuccessNoOutput(ctx, "zpool", "create", "-f",
+		"-O", fmt.Sprintf("mountpoint=%s", args.Mountpoint),
+		args.PoolName, args.ImagePath,
+	)
 	if err != nil {
 		return nil, errors.Wrap(err, "zpool create")
 	}

platformtest/tests/getNonexistent.go

@@ -15,19 +15,16 @@ func GetNonexistent(ctx *platformtest.Context) {
 		+  "foo bar"
 		+  "foo bar@1"
 	`)
-	defer platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
-		DESTROYROOT
-	`)
 
 	// test raw
-	_, err := zfs.ZFSGetRawAnySource(fmt.Sprintf("%s/foo bar", ctx.RootDataset), []string{"name"})
+	_, err := zfs.ZFSGetRawAnySource(ctx, fmt.Sprintf("%s/foo bar", ctx.RootDataset), []string{"name"})
 	if err != nil {
 		panic(err)
 	}
 
 	// test nonexistent filesystem
 	nonexistent := fmt.Sprintf("%s/nonexistent filesystem", ctx.RootDataset)
-	props, err := zfs.ZFSGetRawAnySource(nonexistent, []string{"name"})
+	props, err := zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
 	if err == nil {
 		panic(props)
 	}
@@ -40,7 +37,20 @@ func GetNonexistent(ctx *platformtest.Context) {
 
 	// test nonexistent snapshot
 	nonexistent = fmt.Sprintf("%s/foo bar@non existent", ctx.RootDataset)
-	props, err = zfs.ZFSGetRawAnySource(nonexistent, []string{"name"})
+	props, err = zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
+	if err == nil {
+		panic(props)
+	}
+	dsne, ok = err.(*zfs.DatasetDoesNotExist)
+	if !ok {
+		panic(err)
+	} else if dsne.Path != nonexistent {
+		panic(err)
+	}
+
+	// test nonexistent bookmark
+	nonexistent = fmt.Sprintf("%s/foo bar#non existent", ctx.RootDataset)
+	props, err = zfs.ZFSGetRawAnySource(ctx, nonexistent, []string{"name"})
 	if err == nil {
 		panic(props)
 	}

platformtest/tests/helpers.go

@@ -0,0 +1,176 @@
+package tests
+
+import (
+	"io"
+	"math/rand"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/util/limitio"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func sendArgVersion(ctx *platformtest.Context, fs, relName string) zfs.ZFSSendArgVersion {
+	guid, err := zfs.ZFSGetGUID(ctx, fs, relName)
+	if err != nil {
+		panic(err)
+	}
+	return zfs.ZFSSendArgVersion{
+		RelName: relName,
+		GUID:    guid,
+	}
+}
+
+func fsversion(ctx *platformtest.Context, fs, relname string) zfs.FilesystemVersion {
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, fs+relname)
+	if err != nil {
+		panic(err)
+	}
+	return v
+}
+
+func mustDatasetPath(fs string) *zfs.DatasetPath {
+	p, err := zfs.NewDatasetPath(fs)
+	if err != nil {
+		panic(err)
+	}
+	return p
+}
+
+func mustSnapshot(ctx *platformtest.Context, snap string) {
+	if err := zfs.EntityNamecheck(snap, zfs.EntityTypeSnapshot); err != nil {
+		panic(err)
+	}
+	comps := strings.Split(snap, "@")
+	if len(comps) != 2 {
+		panic(comps)
+	}
+	err := zfs.ZFSSnapshot(ctx, mustDatasetPath(comps[0]), comps[1], false)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func mustGetFilesystemVersion(ctx *platformtest.Context, snapOrBookmark string) zfs.FilesystemVersion {
+	v, err := zfs.ZFSGetFilesystemVersion(ctx, snapOrBookmark)
+	check(err)
+	return v
+}
+
+func check(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+var dummyDataRand = rand.New(rand.NewSource(99))
+
+func writeDummyData(path string, numBytes int64) {
+	r := io.LimitReader(dummyDataRand, numBytes)
+	d, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666)
+	check(err)
+	defer d.Close()
+	_, err = io.Copy(d, r)
+	check(err)
+}
+
+type dummySnapshotSituation struct {
+	sendFS       string
+	dummyDataLen int64
+	snapA        *zfs.ZFSSendArgVersion
+	snapB        *zfs.ZFSSendArgVersion
+}
+
+type resumeSituation struct {
+	sendArgs         zfs.ZFSSendArgsUnvalidated
+	recvOpts         zfs.RecvOptions
+	sendErr, recvErr error
+	recvErrDecoded   *zfs.RecvFailedWithResumeTokenErr
+}
+
+func makeDummyDataSnapshots(ctx *platformtest.Context, sendFS string) (situation dummySnapshotSituation) {
+
+	situation.sendFS = sendFS
+	sendFSMount, err := zfs.ZFSGetMountpoint(ctx, sendFS)
+	require.NoError(ctx, err)
+	require.True(ctx, sendFSMount.Mounted)
+
+	const dummyLen = int64(10 * (1 << 20))
+	situation.dummyDataLen = dummyLen
+
+	writeDummyData(path.Join(sendFSMount.Mountpoint, "dummy_data"), dummyLen)
+	mustSnapshot(ctx, sendFS+"@a snapshot")
+	snapA := sendArgVersion(ctx, sendFS, "@a snapshot")
+	situation.snapA = &snapA
+
+	writeDummyData(path.Join(sendFSMount.Mountpoint, "dummy_data"), dummyLen)
+	mustSnapshot(ctx, sendFS+"@b snapshot")
+	snapB := sendArgVersion(ctx, sendFS, "@b snapshot")
+	situation.snapB = &snapB
+
+	return situation
+}
+
+func makeResumeSituation(ctx *platformtest.Context, src dummySnapshotSituation, recvFS string, sendArgs zfs.ZFSSendArgsUnvalidated, recvOptions zfs.RecvOptions) *resumeSituation {
+
+	situation := &resumeSituation{}
+
+	situation.sendArgs = sendArgs
+	situation.recvOpts = recvOptions
+	require.True(ctx, recvOptions.SavePartialRecvState, "this method would be pointless otherwise")
+	require.Equal(ctx, sendArgs.FS, src.sendFS)
+	sendArgsValidated, err := sendArgs.Validate(ctx)
+	situation.sendErr = err
+	if err != nil {
+		return situation
+	}
+
+	copier, err := zfs.ZFSSend(ctx, sendArgsValidated)
+	situation.sendErr = err
+	if err != nil {
+		return situation
+	}
+
+	limitedCopier := zfs.NewReadCloserCopier(limitio.ReadCloser(copier, src.dummyDataLen/2))
+	defer limitedCopier.Close()
+
+	require.NotNil(ctx, sendArgs.To)
+	err = zfs.ZFSRecv(ctx, recvFS, sendArgs.To, limitedCopier, recvOptions)
+	situation.recvErr = err
+	ctx.Logf("zfs recv exit with %T %s", err, err)
+
+	require.NotNil(ctx, err)
+	resumeErr, ok := err.(*zfs.RecvFailedWithResumeTokenErr)
+	require.True(ctx, ok)
+	situation.recvErrDecoded = resumeErr
+
+	return situation
+}
+
+func versionRelnamesSorted(versions []zfs.FilesystemVersion) []string {
+	var vstrs []string
+	for _, v := range versions {
+		vstrs = append(vstrs, v.RelName())
+	}
+	sort.Strings(vstrs)
+	return vstrs
+}
+
+func datasetToStringSortedTrimPrefix(prefix *zfs.DatasetPath, paths []*zfs.DatasetPath) []string {
+	var pstrs []string
+	for _, p := range paths {
+		trimmed := p.Copy()
+		trimmed.TrimPrefix(prefix)
+		if trimmed.Length() == 0 {
+			continue
+		}
+		pstrs = append(pstrs, trimmed.ToString())
+	}
+	sort.Strings(pstrs)
+	return pstrs
+}

platformtest/tests/idempotentBookmark.go

@@ -0,0 +1,59 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentBookmark(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+		+  "foo bar@another snap"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+
+	asnap := sendArgVersion(ctx, fs, "@a snap")
+	anotherSnap := sendArgVersion(ctx, fs, "@another snap")
+
+	err := zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	// do it again, should be idempotent
+	err = zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	// should fail for another snapshot
+	err = zfs.ZFSBookmark(ctx, fs, anotherSnap, "a bookmark")
+	if err == nil {
+		panic(err)
+	}
+	if _, ok := err.(*zfs.BookmarkExists); !ok {
+		panic(fmt.Sprintf("has type %T", err))
+	}
+
+	// destroy the snapshot
+	if err := zfs.ZFSDestroy(ctx, fmt.Sprintf("%s@a snap", fs)); err != nil {
+		panic(err)
+	}
+
+	// do it again, should fail with special error type
+	err = zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err == nil {
+		panic(err)
+	}
+	if _, ok := err.(*zfs.DatasetDoesNotExist); !ok {
+		panic(fmt.Sprintf("has type %T", err))
+	}
+
+}

platformtest/tests/idempotentDestroy.go

@@ -0,0 +1,75 @@
+package tests
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentDestroy(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@a snap"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	asnap := sendArgVersion(ctx, fs, "@a snap")
+	err := zfs.ZFSBookmark(ctx, fs, asnap, "a bookmark")
+	if err != nil {
+		panic(err)
+	}
+
+	type testCase struct {
+		description, path string
+	}
+
+	cases := []testCase{
+		{"snapshot", fmt.Sprintf("%s@a snap", fs)},
+		{"bookmark", fmt.Sprintf("%s#a bookmark", fs)},
+		{"filesystem", fs},
+	}
+
+	for i := range cases {
+		func() {
+			c := cases[i]
+
+			log.Printf("SUBBEGIN testing idempotent destroy %q for path %q", c.description, c.path)
+
+			log.Println("destroy existing")
+			err = zfs.ZFSDestroy(ctx, c.path)
+			if err != nil {
+				panic(err)
+			}
+			log.Println("destroy again, non-idempotently, must error")
+			err = zfs.ZFSDestroy(ctx, c.path)
+			if _, ok := err.(*zfs.DatasetDoesNotExist); !ok {
+				panic(fmt.Sprintf("%T: %s", err, err))
+			}
+			log.Println("destroy again, idempotently, must not error")
+			err = zfs.ZFSDestroyIdempotent(ctx, c.path)
+			if err != nil {
+				panic(err)
+			}
+
+			log.Println("SUBEND")
+
+		}()
+	}
+
+	// also test idempotent destroy for cases where the parent dataset does not exist
+	err = zfs.ZFSDestroyIdempotent(ctx, fmt.Sprintf("%s/not foo bar@nonexistent snapshot", ctx.RootDataset))
+	if err != nil {
+		panic(err)
+	}
+
+	err = zfs.ZFSDestroyIdempotent(ctx, fmt.Sprintf("%s/not foo bar#nonexistent bookmark", ctx.RootDataset))
+	if err != nil {
+		panic(err)
+	}
+
+}

platformtest/tests/idempotentHold.go

@@ -0,0 +1,37 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func IdempotentHold(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1"
+	`)
+	defer platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		R zfs release zrepl_platformtest "${ROOTDS}/foo bar@1"
+		-  "foo bar@1"
+		-  "foo bar"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", ctx.RootDataset)
+	v1 := fsversion(ctx, fs, "@1")
+
+	tag := "zrepl_platformtest"
+	err := zfs.ZFSHold(ctx, fs, v1, tag)
+	if err != nil {
+		panic(err)
+	}
+
+	err = zfs.ZFSHold(ctx, fs, v1, tag)
+	if err != nil {
+		panic(err)
+	}
+}

platformtest/tests/listFilesystemVersions.go

@@ -0,0 +1,179 @@
+package tests
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func ListFilesystemVersionsTypeFilteringAndPrefix(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@foo 1"
+		+  "foo bar#foo 1" "foo bar@foo 1"
+		+  "foo bar#bookfoo 1" "foo bar@foo 1"
+		+  "foo bar@foo 2"
+		+  "foo bar#foo 2" "foo bar@foo 2"
+		+  "foo bar#bookfoo 2" "foo bar@foo 2"
+		+  "foo bar@blup 1"
+		+  "foo bar#blup 1" "foo bar@blup 1"
+		+  "foo bar@ foo with leading whitespace"
+
+		# repeat the whole thing for a child dataset to make sure we disable recursion
+
+		+  "foo bar/child dataset"
+		+  "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset#foo 1" "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset#bookfoo 1" "foo bar/child dataset@foo 1"
+		+  "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset#foo 2" "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset#bookfoo 2" "foo bar/child dataset@foo 2"
+		+  "foo bar/child dataset@blup 1"
+		+  "foo bar/child dataset#blup 1" "foo bar/child dataset@blup 1"
+		+  "foo bar/child dataset@ foo with leading whitespace"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	// no options := all types
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.NoError(t, err)
+	require.Equal(t, []string{
+		"#blup 1", "#bookfoo 1", "#bookfoo 2", "#foo 1", "#foo 2",
+		"@ foo with leading whitespace", "@blup 1", "@foo 1", "@foo 2",
+	}, versionRelnamesSorted(vs))
+
+	// just snapshots
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Snapshots,
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"@ foo with leading whitespace", "@blup 1", "@foo 1", "@foo 2"}, versionRelnamesSorted(vs))
+
+	// just bookmarks
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		Types: zfs.Bookmarks,
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"#blup 1", "#bookfoo 1", "#bookfoo 2", "#foo 1", "#foo 2"}, versionRelnamesSorted(vs))
+
+	// just with prefix foo
+	vs, err = zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{
+		ShortnamePrefix: "foo",
+	})
+	require.NoError(t, err)
+	require.Equal(t, []string{"#foo 1", "#foo 2", "@foo 1", "@foo 2"}, versionRelnamesSorted(vs))
+
+}
+
+func ListFilesystemVersionsZeroExistIsNotAnError(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+ "foo bar"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.Empty(t, vs)
+	require.NoError(t, err)
+	dsne, ok := err.(*zfs.DatasetDoesNotExist)
+	require.True(t, ok)
+	require.Equal(t, fs, dsne.Path)
+}
+
+func ListFilesystemVersionsFilesystemNotExist(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	`)
+
+	nonexistentFS := fmt.Sprintf("%s/not existent", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(nonexistentFS), zfs.ListFilesystemVersionsOptions{})
+	require.Empty(t, vs)
+	require.Error(t, err)
+	t.Logf("err = %T\n%s", err, err)
+	dsne, ok := err.(*zfs.DatasetDoesNotExist)
+	require.True(t, ok)
+	require.Equal(t, nonexistentFS, dsne.Path)
+}
+
+func ListFilesystemVersionsUserrefs(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+ "foo bar"
+	+ "foo bar@snap 1"
+	+ "foo bar#snap 1" "foo bar@snap 1"
+	+ "foo bar@snap 2"
+	+ "foo bar#snap 2" "foo bar@snap 2"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@snap 2"
+	+ "foo bar@snap 3"
+	+ "foo bar#snap 3" "foo bar@snap 3"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar@snap 3"
+	R zfs hold zrepl_platformtest_second_hold "${ROOTDS}/foo bar@snap 3"
+	+ "foo bar@snap 4"
+	+ "foo bar#snap 4" "foo bar@snap 4"
+
+
+	+ "foo bar/child datset"
+	+ "foo bar/child datset@snap 1"
+	+ "foo bar/child datset#snap 1" "foo bar/child datset@snap 1"
+	+ "foo bar/child datset@snap 2"
+	+ "foo bar/child datset#snap 2" "foo bar/child datset@snap 2"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar/child datset@snap 2"
+	+ "foo bar/child datset@snap 3"
+	+ "foo bar/child datset#snap 3" "foo bar/child datset@snap 3"
+	R zfs hold zrepl_platformtest "${ROOTDS}/foo bar/child datset@snap 3"
+	R zfs hold zrepl_platformtest_second_hold "${ROOTDS}/foo bar/child datset@snap 3"
+	+ "foo bar/child datset@snap 4"
+	+ "foo bar/child datset#snap 4" "foo bar/child datset@snap 4"
+	`)
+
+	fs := fmt.Sprintf("%s/foo bar", t.RootDataset)
+
+	vs, err := zfs.ZFSListFilesystemVersions(t, mustDatasetPath(fs), zfs.ListFilesystemVersionsOptions{})
+	require.NoError(t, err)
+
+	type expectation struct {
+		relName  string
+		userrefs zfs.OptionUint64
+	}
+
+	expect := []expectation{
+		{"#snap 1", zfs.OptionUint64{Valid: false}},
+		{"#snap 2", zfs.OptionUint64{Valid: false}},
+		{"#snap 3", zfs.OptionUint64{Valid: false}},
+		{"#snap 4", zfs.OptionUint64{Valid: false}},
+		{"@snap 1", zfs.OptionUint64{Value: 0, Valid: true}},
+		{"@snap 2", zfs.OptionUint64{Value: 1, Valid: true}},
+		{"@snap 3", zfs.OptionUint64{Value: 2, Valid: true}},
+		{"@snap 4", zfs.OptionUint64{Value: 0, Valid: true}},
+	}
+
+	sort.Slice(vs, func(i, j int) bool {
+		return strings.Compare(vs[i].RelName(), vs[j].RelName()) < 0
+	})
+
+	var expectRelNames []string
+	for _, e := range expect {
+		expectRelNames = append(expectRelNames, e.relName)
+	}
+
+	require.Equal(t, expectRelNames, versionRelnamesSorted(vs))
+
+	for i, e := range expect {
+		require.Equal(t, e.relName, vs[i].RelName())
+		require.Equal(t, e.userrefs, vs[i].UserRefs)
+	}
+
+}

platformtest/tests/listFilesystems.go

@@ -0,0 +1,34 @@
+package tests
+
+import (
+	"strings"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func ListFilesystemsNoFilter(t *platformtest.Context) {
+	platformtest.Run(t, platformtest.PanicErr, t.RootDataset, `
+		DESTROYROOT
+		CREATEROOT
+		R  zfs create -V 10M "${ROOTDS}/bar baz"
+		+  "foo bar"
+		+  "foo bar/bar blup"
+		+  "foo bar/blah"
+		R  zfs create -V 10M "${ROOTDS}/foo bar/blah/a volume"
+	`)
+
+	fss, err := zfs.ZFSListMapping(t, zfs.NoFilter())
+	require.NoError(t, err)
+	var onlyTestPool []*zfs.DatasetPath
+	for _, fs := range fss {
+		if strings.HasPrefix(fs.ToString(), t.RootDataset) {
+			onlyTestPool = append(onlyTestPool, fs)
+		}
+	}
+	onlyTestPoolStr := datasetToStringSortedTrimPrefix(mustDatasetPath(t.RootDataset), onlyTestPool)
+	require.Equal(t, []string{"bar baz", "foo bar", "foo bar/bar blup", "foo bar/blah", "foo bar/blah/a volume"}, onlyTestPoolStr)
+
+}

platformtest/tests/replicationCursor.go

@@ -0,0 +1,69 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/endpoint"
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func ReplicationCursor(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+		CREATEROOT
+		+  "foo bar"
+		+  "foo bar@1 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@1 with space" "${ROOTDS}/foo bar#1 with space"
+		+  "foo bar@2 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@1 with space" "${ROOTDS}/foo bar#2 with space"
+		+  "foo bar@3 with space"
+		R  zfs bookmark "${ROOTDS}/foo bar@1 with space" "${ROOTDS}/foo bar#3 with space"
+	`)
+
+	jobid := endpoint.MustMakeJobID("zreplplatformtest")
+
+	ds, err := zfs.NewDatasetPath(ctx.RootDataset + "/foo bar")
+	if err != nil {
+		panic(err)
+	}
+
+	fs := ds.ToString()
+	snap := fsversion(ctx, fs, "@1 with space")
+
+	destroyed, err := endpoint.MoveReplicationCursor(ctx, fs, &snap, jobid)
+	if err != nil {
+		panic(err)
+	}
+	assert.Empty(ctx, destroyed)
+
+	snapProps, err := zfs.ZFSGetFilesystemVersion(ctx, snap.FullPath(fs))
+	if err != nil {
+		panic(err)
+	}
+
+	bm, err := endpoint.GetMostRecentReplicationCursorOfJob(ctx, fs, jobid)
+	if err != nil {
+		panic(err)
+	}
+	if bm.CreateTXG != snapProps.CreateTXG {
+		panic(fmt.Sprintf("createtxgs do not match: %v != %v", bm.CreateTXG, snapProps.CreateTXG))
+	}
+	if bm.Guid != snapProps.Guid {
+		panic(fmt.Sprintf("guids do not match: %v != %v", bm.Guid, snapProps.Guid))
+	}
+
+	// try moving
+	cursor1BookmarkName, err := endpoint.ReplicationCursorBookmarkName(fs, snap.Guid, jobid)
+	require.NoError(ctx, err)
+
+	snap2 := fsversion(ctx, fs, "@2 with space")
+	destroyed, err = endpoint.MoveReplicationCursor(ctx, fs, &snap2, jobid)
+	require.NoError(ctx, err)
+	require.Equal(ctx, 1, len(destroyed))
+	require.Equal(ctx, endpoint.AbstractionReplicationCursorBookmarkV2, destroyed[0].GetType())
+	require.Equal(ctx, cursor1BookmarkName, destroyed[0].GetName())
+}

platformtest/tests/resumableRecvAndTokenHandling.go

@@ -0,0 +1,64 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func ResumableRecvAndTokenHandling(ctx *platformtest.Context) {
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er"
+	`)
+
+	sendFS := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	recvFS := fmt.Sprintf("%s/recv er", ctx.RootDataset)
+
+	supported, err := zfs.ResumeRecvSupported(ctx, mustDatasetPath(sendFS))
+	check(err)
+
+	src := makeDummyDataSnapshots(ctx, sendFS)
+
+	s := makeResumeSituation(ctx, src, recvFS, zfs.ZFSSendArgsUnvalidated{
+		FS:          sendFS,
+		To:          src.snapA,
+		Encrypted:   &zfs.NilBool{B: false},
+		ResumeToken: "",
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false, // doesnt' exist yet
+		SavePartialRecvState: true,
+	})
+
+	if !supported {
+		_, ok := s.recvErr.(*zfs.ErrRecvResumeNotSupported)
+		require.True(ctx, ok)
+
+		// we know that support on sendFS implies support on recvFS
+		// => assert that if we don't support resumed recv, the method returns ""
+		tok, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, mustDatasetPath(recvFS))
+		check(err)
+		require.Equal(ctx, "", tok)
+
+		return // nothing more to test for recv that doesn't support -s
+	}
+
+	getTokenRaw, err := zfs.ZFSGetReceiveResumeTokenOrEmptyStringIfNotSupported(ctx, mustDatasetPath(recvFS))
+	check(err)
+
+	require.NotEmpty(ctx, getTokenRaw)
+	decodedToken, err := zfs.ParseResumeToken(ctx, getTokenRaw)
+	check(err)
+
+	require.True(ctx, decodedToken.HasToGUID)
+	require.Equal(ctx, s.sendArgs.To.GUID, decodedToken.ToGUID)
+
+	recvErr := s.recvErr.(*zfs.RecvFailedWithResumeTokenErr)
+	require.Equal(ctx, recvErr.ResumeTokenRaw, getTokenRaw)
+	require.Equal(ctx, recvErr.ResumeTokenParsed, decodedToken)
+}

platformtest/tests/resumeTokenParsing.go

@@ -0,0 +1,105 @@
+package tests
+
+import (
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+type resumeTokenTest struct {
+	Msg         string
+	Token       string
+	ExpectToken *zfs.ResumeToken
+	ExpectError error
+}
+
+func (rtt *resumeTokenTest) Test(t *platformtest.Context) {
+
+	resumeSendSupported, err := zfs.ResumeSendSupported(t)
+	if err != nil {
+		t.Errorf("cannot determine whether resume supported: %T %s", err, err)
+		t.FailNow()
+		return
+	}
+
+	res, err := zfs.ParseResumeToken(t, rtt.Token)
+
+	// if decoding is not supported, don't bother with the expectations
+	if !resumeSendSupported {
+		require.Error(t, err)
+		require.Equal(t, zfs.ResumeTokenDecodingNotSupported, err)
+		return
+	}
+
+	if rtt.ExpectError != nil {
+		require.EqualValues(t, rtt.ExpectError, err)
+		return
+	}
+	if rtt.ExpectToken != nil {
+		require.Nil(t, err)
+		require.EqualValues(t, rtt.ExpectToken, res)
+		return
+	}
+}
+
+func ResumeTokenParsing(ctx *platformtest.Context) {
+
+	// cases generated using resumeTokensGenerate.bash on ZoL 0.8.1
+	cases := []resumeTokenTest{
+		{
+			Msg:   "zreplplatformtest/dst/full",
+			Token: "1-b338b54f3-c0-789c636064000310a500c4ec50360710e72765a52697303030419460caa7a515a796806474e0f26c48f2499525a9c540ba42430fabfe92fcf4d2cc140686c88a76d578ae45530c90e439c1f27989b9a90c0c5545a905390539892569f945b940234bf48b8b921d12c1660200c61a1aba",
+			ExpectToken: &zfs.ResumeToken{
+				HasToGUID: true,
+				ToGUID:    0x94a20a5f25877859,
+				ToName:    "zreplplatformtest/src@a",
+			},
+		},
+		{
+			Msg:   "zreplplatformtest/dst/full_raw",
+			Token: "1-e3f40c323-f8-789c636064000310a500c4ec50360710e72765a52697303030419460caa7a515a796806474e0f26c48f2499525a9c540da454f0fabfe92fcf4d2cc140686c88a76d578ae45530c90e439c1f27989b9a90c0c5545a905390539892569f945b940234bf48b8b921d12c1e6713320dc9f9c9f5b50945a5c9c9f0d119380ba07265f94580e936200004ff12141",
+			ExpectToken: &zfs.ResumeToken{
+				HasToGUID:     true,
+				ToGUID:        0x94a20a5f25877859,
+				ToName:        "zreplplatformtest/src@a",
+				HasCompressOK: true, CompressOK: true,
+				HasRawOk: true, RawOK: true,
+			},
+		},
+
+		{
+			Msg:   "zreplplatformtest/dst/inc",
+			Token: "1-eadabb296-e8-789c636064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b518483ba8ea61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b09900d43e2191",
+			ExpectToken: &zfs.ResumeToken{
+				HasFromGUID: true, FromGUID: 0x94a20a5f25877859,
+				HasToGUID: true, ToGUID: 0xf784e1004f460f7a,
+				ToName: "zreplplatformtest/src@b",
+			},
+		},
+		{
+			Msg:   "zreplplatformtest/dst/inc_raw",
+			Token: "1-1164f8d409-120-789c636064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b51848f368eb61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b079dc0c087f26e7e71614a51617e76743c424a0ee81c9172596c3a41800dd2c2818",
+			ExpectToken: &zfs.ResumeToken{
+				HasFromGUID: true, FromGUID: 0x94a20a5f25877859,
+				HasToGUID: true, ToGUID: 0xf784e1004f460f7a,
+				ToName:        "zreplplatformtest/src@b",
+				HasCompressOK: true, CompressOK: true,
+				HasRawOk: true, RawOK: true,
+			},
+		},
+
+		// manual test csaes
+		{
+			Msg:         "corrupted",
+			Token:       "1-1164f8d409-120-badf00d064000310a501c49c50360710a715e5e7a69766a63040416445bb6a3cd7a2290a40363b92bafca4acd4e412060626a83a0cf9b4b4e2d412908c0e5c9e0d493ea9b224b51848f368eb61d55f920f714515bf9b3fc3c396ef0648f29c60f9bcc4dc54a07c516a414e414e62495a7e512ed0c812fde2a2648724b079dc0c087f26e7e71614a51617e76743c424a0ee81c9172596c3a41800dd2c2818",
+			ExpectError: zfs.ResumeTokenCorruptError,
+		},
+	}
+
+	for _, test := range cases {
+		ctx.Logf("BEGIN SUBTEST: %s", test.Msg)
+		test.Test(ctx)
+		ctx.Logf("COMPLETE SUBTEST: %s", test.Msg)
+	}
+}

platformtest/tests/resumeTokensGenerate.bash

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -euo pipefail
+set -x
+
+POOLNAME="zreplplatformtest"
+POOLIMG="$1"
+
+if zpool status "$POOLNAME"; then
+    exit 1
+fi
+
+if [ -e "$POOLIMG" ]; then
+    exit 1
+fi
+fallocate -l 500M "$POOLIMG"
+zpool create "$POOLNAME" "$POOLIMG"
+
+SRC_DATASET="$POOLNAME/src"
+DST_ROOT="$POOLNAME/dst"
+
+keylocation="$(mktemp /tmp/ZREPL_PLATFORMTEST_GENERATE_TOKENS_KEYFILE_XXX)"
+echo "foobar123" > "$keylocation"
+zfs create -o encryption=on -o keylocation="file:///$keylocation" -o keyformat=passphrase "$SRC_DATASET"
+rm "$keylocation"
+
+SRC_MOUNT="$(zfs get -H -o value mountpoint "$SRC_DATASET")"
+test -d "$SRC_MOUNT"
+
+A="$SRC_DATASET"@a
+B="$SRC_DATASET"@b
+dd if=/dev/urandom of="$SRC_MOUNT"/dummy_data bs=1M count=5
+zfs snapshot "$A"
+dd if=/dev/urandom of="$SRC_MOUNT"/dummy_data bs=1M count=5
+zfs snapshot "$B"
+
+zfs create "$DST_ROOT"
+
+cutoff="dd bs=1024 count=3K"
+
+set +e
+zfs send    "$A"         | $cutoff | zfs recv -s "$DST_ROOT"/full
+
+zfs send "$A" | zfs recv "$DST_ROOT"/inc
+zfs send    -i "$A" "$B" | $cutoff | zfs recv -s "$DST_ROOT"/inc
+
+zfs send -w "$A"         | $cutoff | zfs recv -s "$DST_ROOT"/full_raw
+
+zfs send -w "$A" | zfs recv "$DST_ROOT"/inc_raw
+zfs send -w -i "$A" "$B" | $cutoff | zfs recv -s "$DST_ROOT"/inc_raw
+set -e
+
+TOKENS="$(zfs list -H -o name,receive_resume_token -r "$DST_ROOT")"
+
+echo "$TOKENS" | awk -e '//{ if ($2 == "-") {  } else { system("zfs send -nvt " + $2);} }'
+
+zpool destroy "$POOLNAME"
+rm "$POOLIMG"

platformtest/tests/sendArgsValidation.go

@@ -0,0 +1,198 @@
+package tests
+
+import (
+	"fmt"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/zrepl/zrepl/platformtest"
+	"github.com/zrepl/zrepl/zfs"
+)
+
+func SendArgsValidationEncryptedSendOfUnencryptedDatasetForbidden(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	check(err)
+	expectNotSupportedErr := !supported
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er"
+	+   "send er@a snap"
+	`)
+
+	fs := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	props := mustGetFilesystemVersion(ctx, fs+"@a snap")
+
+	sendArgs, err := zfs.ZFSSendArgsUnvalidated{
+		FS: fs,
+		To: &zfs.ZFSSendArgVersion{
+			RelName: "@a snap",
+			GUID:    props.Guid,
+		},
+		Encrypted:   &zfs.NilBool{B: true},
+		ResumeToken: "",
+	}.Validate(ctx)
+
+	var stream *zfs.ReadCloserCopier
+	if err == nil {
+		stream, err = zfs.ZFSSend(ctx, sendArgs) // no shadow
+		if err == nil {
+			defer stream.Close()
+		}
+		// fallthrough
+	}
+
+	if expectNotSupportedErr {
+		require.Error(ctx, err)
+		require.Equal(ctx, zfs.ErrEncryptedSendNotSupported, err)
+		return
+	}
+	require.Error(ctx, err)
+	ctx.Logf("send err: %T %s", err, err)
+	validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+	require.True(ctx, ok)
+	require.True(ctx, validationErr.What == zfs.ZFSSendArgsEncryptedSendRequestedButFSUnencrypted)
+}
+
+func SendArgsValidationResumeTokenEncryptionMismatchForbidden(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+	supported, err = zfs.ResumeSendSupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er" encrypted
+	`)
+
+	sendFS := fmt.Sprintf("%s/send er", ctx.RootDataset)
+	unencRecvFS := fmt.Sprintf("%s/unenc recv", ctx.RootDataset)
+	encRecvFS := fmt.Sprintf("%s/enc recv", ctx.RootDataset)
+
+	src := makeDummyDataSnapshots(ctx, sendFS)
+
+	unencS := makeResumeSituation(ctx, src, unencRecvFS, zfs.ZFSSendArgsUnvalidated{
+		FS:        sendFS,
+		To:        src.snapA,
+		Encrypted: &zfs.NilBool{B: false}, // !
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	encS := makeResumeSituation(ctx, src, encRecvFS, zfs.ZFSSendArgsUnvalidated{
+		FS:        sendFS,
+		To:        src.snapA,
+		Encrypted: &zfs.NilBool{B: true}, // !
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	// threat model: use of a crafted resume token that requests an unencrypted send
+	//               but send args require encrypted send
+	{
+		var maliciousSend zfs.ZFSSendArgsUnvalidated = encS.sendArgs
+		maliciousSend.ResumeToken = unencS.recvErrDecoded.ResumeTokenRaw
+
+		_, err := maliciousSend.Validate(ctx)
+		validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+		require.True(ctx, ok)
+		require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+		ctx.Logf("%s", validationErr)
+
+		mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+		require.True(ctx, ok)
+		require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchEncryptionNotSet)
+	}
+
+	// threat model: use of a crafted resume token that requests an encrypted send
+	//               but send args require unencrypted send
+	{
+		var maliciousSend zfs.ZFSSendArgsUnvalidated = unencS.sendArgs
+		maliciousSend.ResumeToken = encS.recvErrDecoded.ResumeTokenRaw
+
+		_, err := maliciousSend.Validate(ctx)
+		require.Error(ctx, err)
+		ctx.Logf("send err: %T %s", err, err)
+		validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+		require.True(ctx, ok)
+		require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+		ctx.Logf("%s", validationErr)
+
+		mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+		require.True(ctx, ok)
+		require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchEncryptionSet)
+	}
+
+}
+
+func SendArgsValidationResumeTokenDifferentFilesystemForbidden(ctx *platformtest.Context) {
+
+	supported, err := zfs.EncryptionCLISupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+	supported, err = zfs.ResumeSendSupported(ctx)
+	check(err)
+	if !supported {
+		ctx.SkipNow()
+	}
+
+	platformtest.Run(ctx, platformtest.PanicErr, ctx.RootDataset, `
+	DESTROYROOT
+	CREATEROOT
+	+	"send er1"
+	+	"send er2"
+	`)
+
+	sendFS1 := fmt.Sprintf("%s/send er1", ctx.RootDataset)
+	sendFS2 := fmt.Sprintf("%s/send er2", ctx.RootDataset)
+	recvFS := fmt.Sprintf("%s/unenc recv", ctx.RootDataset)
+
+	src1 := makeDummyDataSnapshots(ctx, sendFS1)
+	src2 := makeDummyDataSnapshots(ctx, sendFS2)
+
+	rs := makeResumeSituation(ctx, src1, recvFS, zfs.ZFSSendArgsUnvalidated{
+		FS:        sendFS1,
+		To:        src1.snapA,
+		Encrypted: &zfs.NilBool{B: false},
+	}, zfs.RecvOptions{
+		RollbackAndForceRecv: false,
+		SavePartialRecvState: true,
+	})
+
+	// threat model: forged resume token tries to steal a full send of snapA on fs2 by
+	//               presenting a resume token for full send of snapA on fs1
+	var maliciousSend zfs.ZFSSendArgsUnvalidated = zfs.ZFSSendArgsUnvalidated{
+		FS: sendFS2,
+		To: &zfs.ZFSSendArgVersion{
+			RelName: src2.snapA.RelName,
+			GUID:    src2.snapA.GUID,
+		},
+		Encrypted:   &zfs.NilBool{B: false},
+		ResumeToken: rs.recvErrDecoded.ResumeTokenRaw,
+	}
+	_, err = maliciousSend.Validate(ctx)
+	require.Error(ctx, err)
+	ctx.Logf("send err: %T %s", err, err)
+	validationErr, ok := err.(*zfs.ZFSSendArgsValidationError)
+	require.True(ctx, ok)
+	require.Equal(ctx, validationErr.What, zfs.ZFSSendArgsResumeTokenMismatch)
+	ctx.Logf("%s", validationErr)
+
+	mismatchError, ok := validationErr.Msg.(*zfs.ZFSSendArgsResumeTokenMismatchError)
+	require.True(ctx, ok)
+	require.Equal(ctx, mismatchError.What, zfs.ZFSSendArgsResumeTokenMismatchFilesystem)
+}